Update fine-tuned model

adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "down_proj",
+    "q_proj",
     "k_proj",
-    "v_proj",
-    "o_proj",
     "gate_proj",
+    "down_proj",
     "up_proj",
-    "q_proj"
+    "v_proj",
+    "o_proj"
   ],
   "task_type": " CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d254208220452efb5ba0aedb7d5be500c3209e4b506c5c6c4ff320e14666e2db
+oid sha256:3d054996c8870886154c89eb4b321cffae3782503afa3093805cae5b92e89d63
 size 159967880

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:769fa125c712bf48bb3a15782602e5476cdad85b05282b3d5ca1fef5c0c7a0c8
-size 852866272
+oid sha256:75b0df8cc32c0c3fdd058abda370de0bd80f3dfa6026c5b8d7404204aa249974
+size 852876198

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:313eafef9cf4a8331b910ce797cfc97b5dbb488c88178d802041cdba21c8bb1e
+oid sha256:34e497cead5b5d8cb6e8ac2be926d8e9aecf8f729139a14383abc9f0da4763cc
 size 14244

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e69e2b49ea642509f0c688c16fb190b7cf27dac0a18903a5e2d1467d0343d8b8
+oid sha256:4521b8db9cc205e54aa606d85e707c024abd2d8ad4a20bec4b2cff365dc59cdf
 size 1064

trainer_state.json

@@ -3,364 +3,189 @@
   "best_model_checkpoint": null,
   "epoch": 5.0,
   "eval_steps": 500,
-  "global_step": 50,
+  "global_step": 25,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
-    {
-      "epoch": 0.1,
-      "grad_norm": 0.6260854601860046,
-      "learning_rate": 6.666666666666667e-05,
-      "loss": 0.6709,
-      "step": 1
-    },
     {
       "epoch": 0.2,
-      "grad_norm": 0.6020879149436951,
-      "learning_rate": 0.00013333333333333334,
-      "loss": 0.6739,
-      "step": 2
-    },
-    {
-      "epoch": 0.3,
-      "grad_norm": 0.4678296446800232,
-      "learning_rate": 0.0002,
-      "loss": 0.6687,
-      "step": 3
+      "grad_norm": 0.6263333559036255,
+      "learning_rate": 0.0001,
+      "loss": 0.6512,
+      "step": 1
     },
     {
       "epoch": 0.4,
-      "grad_norm": 0.40054887533187866,
-      "learning_rate": 0.00019977668786231534,
-      "loss": 0.6272,
-      "step": 4
-    },
-    {
-      "epoch": 0.5,
-      "grad_norm": 0.39390796422958374,
-      "learning_rate": 0.000199107748815478,
-      "loss": 0.6232,
-      "step": 5
+      "grad_norm": 0.659807562828064,
+      "learning_rate": 0.0002,
+      "loss": 0.6388,
+      "step": 2
     },
     {
       "epoch": 0.6,
-      "grad_norm": 0.3897330164909363,
-      "learning_rate": 0.0001979961705036587,
-      "loss": 0.6218,
-      "step": 6
-    },
-    {
-      "epoch": 0.7,
-      "grad_norm": 0.33647778630256653,
-      "learning_rate": 0.00019644691750543767,
-      "loss": 0.6658,
-      "step": 7
+      "grad_norm": 0.4718220829963684,
+      "learning_rate": 0.00019906859460363307,
+      "loss": 0.6469,
+      "step": 3
     },
     {
       "epoch": 0.8,
-      "grad_norm": 0.3042166233062744,
-      "learning_rate": 0.0001944669091607919,
-      "loss": 0.6237,
-      "step": 8
-    },
-    {
-      "epoch": 0.9,
-      "grad_norm": 0.2846579849720001,
-      "learning_rate": 0.00019206498866764288,
-      "loss": 0.6481,
-      "step": 9
+      "grad_norm": 0.3931182324886322,
+      "learning_rate": 0.00019629172873477995,
+      "loss": 0.6385,
+      "step": 4
     },
     {
       "epoch": 1.0,
-      "grad_norm": 0.30414876341819763,
-      "learning_rate": 0.00018925188358598813,
-      "loss": 0.6247,
-      "step": 10
-    },
-    {
-      "epoch": 1.1,
-      "grad_norm": 0.3322124183177948,
-      "learning_rate": 0.00018604015792601396,
-      "loss": 0.521,
-      "step": 11
+      "grad_norm": 0.394342839717865,
+      "learning_rate": 0.00019172113015054532,
+      "loss": 0.6754,
+      "step": 5
     },
     {
       "epoch": 1.2,
-      "grad_norm": 0.31530916690826416,
-      "learning_rate": 0.00018244415603417603,
-      "loss": 0.5385,
-      "step": 12
-    },
-    {
-      "epoch": 1.3,
-      "grad_norm": 0.32330620288848877,
-      "learning_rate": 0.0001784799385278661,
-      "loss": 0.5396,
-      "step": 13
+      "grad_norm": 0.4089839458465576,
+      "learning_rate": 0.00018544194045464886,
+      "loss": 0.5292,
+      "step": 6
     },
     {
       "epoch": 1.4,
-      "grad_norm": 0.33883941173553467,
-      "learning_rate": 0.00017416521056479577,
-      "loss": 0.4918,
-      "step": 14
-    },
-    {
-      "epoch": 1.5,
-      "grad_norm": 0.30997005105018616,
-      "learning_rate": 0.00016951924276746425,
-      "loss": 0.5089,
-      "step": 15
+      "grad_norm": 0.39185038208961487,
+      "learning_rate": 0.000177571129070442,
+      "loss": 0.5756,
+      "step": 7
     },
     {
       "epoch": 1.6,
-      "grad_norm": 0.3036053478717804,
-      "learning_rate": 0.00016456278515588024,
-      "loss": 0.4947,
-      "step": 16
-    },
-    {
-      "epoch": 1.7,
-      "grad_norm": 0.27217769622802734,
-      "learning_rate": 0.00015931797447293552,
-      "loss": 0.485,
-      "step": 17
+      "grad_norm": 0.35863593220710754,
+      "learning_rate": 0.00016825531432186543,
+      "loss": 0.5311,
+      "step": 8
     },
     {
       "epoch": 1.8,
-      "grad_norm": 0.27569136023521423,
-      "learning_rate": 0.00015380823531633729,
-      "loss": 0.5081,
-      "step": 18
-    },
-    {
-      "epoch": 1.9,
-      "grad_norm": 0.27971240878105164,
-      "learning_rate": 0.00014805817551866838,
-      "loss": 0.4818,
-      "step": 19
+      "grad_norm": 0.3710576891899109,
+      "learning_rate": 0.00015766803221148673,
+      "loss": 0.5056,
+      "step": 9
     },
     {
       "epoch": 2.0,
-      "grad_norm": 0.2714477777481079,
-      "learning_rate": 0.0001420934762428335,
-      "loss": 0.4738,
-      "step": 20
-    },
-    {
-      "epoch": 2.1,
-      "grad_norm": 0.245743066072464,
-      "learning_rate": 0.00013594077728375128,
-      "loss": 0.4526,
-      "step": 21
+      "grad_norm": 0.365957647562027,
+      "learning_rate": 0.00014600650377311522,
+      "loss": 0.4892,
+      "step": 10
     },
     {
       "epoch": 2.2,
-      "grad_norm": 0.26513662934303284,
-      "learning_rate": 0.00012962755808856342,
-      "loss": 0.4306,
-      "step": 22
-    },
-    {
-      "epoch": 2.3,
-      "grad_norm": 0.26220983266830444,
-      "learning_rate": 0.00012318201502675285,
-      "loss": 0.3857,
-      "step": 23
+      "grad_norm": 0.34360572695732117,
+      "learning_rate": 0.00013348796121709862,
+      "loss": 0.4838,
+      "step": 11
     },
     {
       "epoch": 2.4,
-      "grad_norm": 0.2714617848396301,
-      "learning_rate": 0.00011663293545831302,
-      "loss": 0.4123,
-      "step": 24
-    },
-    {
-      "epoch": 2.5,
-      "grad_norm": 0.2738859951496124,
-      "learning_rate": 0.00011000956916240985,
-      "loss": 0.4085,
-      "step": 25
+      "grad_norm": 0.33910414576530457,
+      "learning_rate": 0.0001203456013052634,
+      "loss": 0.4295,
+      "step": 12
     },
     {
       "epoch": 2.6,
-      "grad_norm": 0.2666757106781006,
-      "learning_rate": 0.00010334149770076747,
-      "loss": 0.4161,
-      "step": 26
-    },
-    {
-      "epoch": 2.7,
-      "grad_norm": 0.27604955434799194,
-      "learning_rate": 9.665850229923258e-05,
-      "loss": 0.4053,
-      "step": 27
+      "grad_norm": 0.31048351526260376,
+      "learning_rate": 0.0001068242413364671,
+      "loss": 0.4072,
+      "step": 13
     },
     {
       "epoch": 2.8,
-      "grad_norm": 0.3039579689502716,
-      "learning_rate": 8.999043083759017e-05,
-      "loss": 0.3988,
-      "step": 28
-    },
-    {
-      "epoch": 2.9,
-      "grad_norm": 0.27524131536483765,
-      "learning_rate": 8.336706454168701e-05,
-      "loss": 0.4112,
-      "step": 29
+      "grad_norm": 0.3141072988510132,
+      "learning_rate": 9.317575866353292e-05,
+      "loss": 0.4244,
+      "step": 14
     },
     {
       "epoch": 3.0,
-      "grad_norm": 0.2864067256450653,
-      "learning_rate": 7.681798497324716e-05,
-      "loss": 0.3834,
-      "step": 30
-    },
-    {
-      "epoch": 3.1,
-      "grad_norm": 0.2553528845310211,
-      "learning_rate": 7.037244191143661e-05,
-      "loss": 0.381,
-      "step": 31
+      "grad_norm": 0.31501471996307373,
+      "learning_rate": 7.965439869473664e-05,
+      "loss": 0.4128,
+      "step": 15
     },
     {
       "epoch": 3.2,
-      "grad_norm": 0.2796800434589386,
-      "learning_rate": 6.405922271624874e-05,
-      "loss": 0.3569,
-      "step": 32
-    },
-    {
-      "epoch": 3.3,
-      "grad_norm": 0.27441126108169556,
-      "learning_rate": 5.790652375716652e-05,
-      "loss": 0.341,
-      "step": 33
+      "grad_norm": 0.2909337878227234,
+      "learning_rate": 6.651203878290139e-05,
+      "loss": 0.3821,
+      "step": 16
     },
     {
       "epoch": 3.4,
-      "grad_norm": 0.2751557528972626,
-      "learning_rate": 5.1941824481331626e-05,
-      "loss": 0.3154,
-      "step": 34
-    },
-    {
-      "epoch": 3.5,
-      "grad_norm": 0.28258493542671204,
-      "learning_rate": 4.6191764683662744e-05,
-      "loss": 0.346,
-      "step": 35
+      "grad_norm": 0.26387819647789,
+      "learning_rate": 5.399349622688479e-05,
+      "loss": 0.3776,
+      "step": 17
     },
     {
       "epoch": 3.6,
-      "grad_norm": 0.2823265492916107,
-      "learning_rate": 4.0682025527064486e-05,
-      "loss": 0.3615,
-      "step": 36
-    },
-    {
-      "epoch": 3.7,
-      "grad_norm": 0.2728078067302704,
-      "learning_rate": 3.543721484411976e-05,
-      "loss": 0.3512,
-      "step": 37
+      "grad_norm": 0.26343247294425964,
+      "learning_rate": 4.2331967788513295e-05,
+      "loss": 0.3619,
+      "step": 18
     },
     {
       "epoch": 3.8,
-      "grad_norm": 0.28614407777786255,
-      "learning_rate": 3.0480757232535772e-05,
-      "loss": 0.3426,
-      "step": 38
-    },
-    {
-      "epoch": 3.9,
-      "grad_norm": 0.28317952156066895,
-      "learning_rate": 2.5834789435204243e-05,
-      "loss": 0.3557,
-      "step": 39
+      "grad_norm": 0.26532021164894104,
+      "learning_rate": 3.174468567813461e-05,
+      "loss": 0.3715,
+      "step": 19
     },
     {
       "epoch": 4.0,
-      "grad_norm": 0.2815261781215668,
-      "learning_rate": 2.1520061472133902e-05,
-      "loss": 0.3308,
-      "step": 40
-    },
-    {
-      "epoch": 4.1,
-      "grad_norm": 0.27806347608566284,
-      "learning_rate": 1.7555843965823992e-05,
-      "loss": 0.3504,
-      "step": 41
+      "grad_norm": 0.27087175846099854,
+      "learning_rate": 2.242887092955801e-05,
+      "loss": 0.3683,
+      "step": 20
     },
     {
       "epoch": 4.2,
-      "grad_norm": 0.2629810571670532,
-      "learning_rate": 1.3959842073986085e-05,
-      "loss": 0.3188,
-      "step": 42
-    },
-    {
-      "epoch": 4.3,
-      "grad_norm": 0.26613494753837585,
-      "learning_rate": 1.0748116414011888e-05,
-      "loss": 0.3116,
-      "step": 43
+      "grad_norm": 0.25199756026268005,
+      "learning_rate": 1.4558059545351143e-05,
+      "loss": 0.3523,
+      "step": 21
     },
     {
       "epoch": 4.4,
-      "grad_norm": 0.27727553248405457,
-      "learning_rate": 7.935011332357112e-06,
-      "loss": 0.3273,
-      "step": 44
-    },
-    {
-      "epoch": 4.5,
-      "grad_norm": 0.27071133255958557,
-      "learning_rate": 5.533090839208133e-06,
-      "loss": 0.3067,
-      "step": 45
+      "grad_norm": 0.2494262307882309,
+      "learning_rate": 8.278869849454718e-06,
+      "loss": 0.333,
+      "step": 22
     },
     {
       "epoch": 4.6,
-      "grad_norm": 0.27544400095939636,
-      "learning_rate": 3.5530824945623542e-06,
-      "loss": 0.3217,
-      "step": 46
-    },
-    {
-      "epoch": 4.7,
-      "grad_norm": 0.2573586106300354,
-      "learning_rate": 2.003829496341325e-06,
-      "loss": 0.2935,
-      "step": 47
+      "grad_norm": 0.25421398878097534,
+      "learning_rate": 3.7082712652200867e-06,
+      "loss": 0.3289,
+      "step": 23
     },
     {
       "epoch": 4.8,
-      "grad_norm": 0.2781751751899719,
-      "learning_rate": 8.922511845219971e-07,
-      "loss": 0.3137,
-      "step": 48
-    },
-    {
-      "epoch": 4.9,
-      "grad_norm": 0.27297401428222656,
-      "learning_rate": 2.2331213768468363e-07,
-      "loss": 0.3279,
-      "step": 49
+      "grad_norm": 0.26278430223464966,
+      "learning_rate": 9.314053963669245e-07,
+      "loss": 0.3438,
+      "step": 24
     },
     {
       "epoch": 5.0,
-      "grad_norm": 0.2672978639602661,
+      "grad_norm": 0.2513781785964966,
       "learning_rate": 0.0,
-      "loss": 0.3248,
-      "step": 50
+      "loss": 0.379,
+      "step": 25
     }
   ],
   "logging_steps": 1,
-  "max_steps": 50,
+  "max_steps": 25,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 5,
   "save_steps": 500,
@@ -376,7 +201,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 6.004775691603149e+16,
+  "total_flos": 3.082693846592717e+16,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:173743c1b39166bcbb089ece0637a2f11fd365a569d3db5307fd0e6f00eb54e3
+oid sha256:bdb67f7e08d706c1aba378a44137116de683468a035998cac2d26dc2c2a5fd26
 size 5240