Update fine-tuned model

adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "o_proj",
     "k_proj",
-    "q_proj",
-    "v_proj",
-    "down_proj",
+    "gate_proj",
     "up_proj",
-    "gate_proj"
+    "o_proj",
+    "down_proj",
+    "q_proj",
+    "v_proj"
   ],
   "task_type": " CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:430a11259d17a49541e9f31c838ea163b82b162b916c4215762ffab0b4917a07
+oid sha256:9a83e9a0016f9da67a243e4a1e37823c5fd485eab0afee3026b8ef7de3841085
 size 159967880

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e45b6af16f366e2479a1fda5784a26168e5d866ba82ed55f4eb013e34c3ceee
+oid sha256:8ecccb0d08fadf6a1ad6f7ab16c4ac9074e62eccf977a1c221a8fb9b0f83894a
 size 852876198

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b0654b1e5962e1d264ec17c40dcea6c3de9060e283aa95a5fea4d3b6d181505f
+oid sha256:c09ae3d43feaff4e0394c4352b4f7f3a126522099688de7cea344ac7d55250d9
 size 14244

trainer_state.json

@@ -10,352 +10,352 @@
   "log_history": [
     {
       "epoch": 0.1,
-      "grad_norm": 0.886165976524353,
+      "grad_norm": 0.7374217510223389,
       "learning_rate": 6.666666666666667e-05,
-      "loss": 0.8569,
+      "loss": 0.828,
       "step": 1
     },
     {
       "epoch": 0.2,
-      "grad_norm": 0.9352289438247681,
+      "grad_norm": 0.7326979041099548,
       "learning_rate": 0.00013333333333333334,
-      "loss": 0.9157,
+      "loss": 0.8692,
       "step": 2
     },
     {
       "epoch": 0.3,
-      "grad_norm": 0.6887379288673401,
+      "grad_norm": 0.5996476411819458,
       "learning_rate": 0.0002,
-      "loss": 0.8368,
+      "loss": 0.7978,
       "step": 3
     },
     {
       "epoch": 0.4,
-      "grad_norm": 0.5438244938850403,
+      "grad_norm": 0.4960692524909973,
       "learning_rate": 0.00019977668786231534,
-      "loss": 0.7759,
+      "loss": 0.7581,
       "step": 4
     },
     {
       "epoch": 0.5,
-      "grad_norm": 0.4049173593521118,
+      "grad_norm": 0.38662832975387573,
       "learning_rate": 0.000199107748815478,
-      "loss": 0.7108,
+      "loss": 0.7099,
       "step": 5
     },
     {
       "epoch": 0.6,
-      "grad_norm": 0.3374454379081726,
+      "grad_norm": 0.47017937898635864,
       "learning_rate": 0.0001979961705036587,
-      "loss": 0.6949,
+      "loss": 0.6783,
       "step": 6
     },
     {
       "epoch": 0.7,
-      "grad_norm": 0.31860846281051636,
+      "grad_norm": 0.39107415080070496,
       "learning_rate": 0.00019644691750543767,
-      "loss": 0.6725,
+      "loss": 0.6699,
       "step": 7
     },
     {
       "epoch": 0.8,
-      "grad_norm": 0.3116016089916229,
+      "grad_norm": 0.31766048073768616,
       "learning_rate": 0.0001944669091607919,
-      "loss": 0.6814,
+      "loss": 0.6986,
       "step": 8
     },
     {
       "epoch": 0.9,
-      "grad_norm": 0.3020254969596863,
+      "grad_norm": 0.28557154536247253,
       "learning_rate": 0.00019206498866764288,
-      "loss": 0.6628,
+      "loss": 0.6539,
       "step": 9
     },
     {
       "epoch": 1.0,
-      "grad_norm": 0.3751624524593353,
+      "grad_norm": 0.2970712184906006,
       "learning_rate": 0.00018925188358598813,
-      "loss": 0.6491,
+      "loss": 0.6672,
       "step": 10
     },
     {
       "epoch": 1.1,
-      "grad_norm": 0.33873409032821655,
+      "grad_norm": 0.33635565638542175,
       "learning_rate": 0.00018604015792601396,
-      "loss": 0.5887,
+      "loss": 0.5643,
       "step": 11
     },
     {
       "epoch": 1.2,
-      "grad_norm": 0.3233942985534668,
+      "grad_norm": 0.34821704030036926,
       "learning_rate": 0.00018244415603417603,
-      "loss": 0.5414,
+      "loss": 0.5525,
       "step": 12
     },
     {
       "epoch": 1.3,
-      "grad_norm": 0.33184298872947693,
+      "grad_norm": 0.3633900582790375,
       "learning_rate": 0.0001784799385278661,
-      "loss": 0.5534,
+      "loss": 0.5256,
       "step": 13
     },
     {
       "epoch": 1.4,
-      "grad_norm": 0.3251623511314392,
+      "grad_norm": 0.3432920575141907,
       "learning_rate": 0.00017416521056479577,
-      "loss": 0.5303,
+      "loss": 0.5193,
       "step": 14
     },
     {
       "epoch": 1.5,
-      "grad_norm": 0.30667614936828613,
+      "grad_norm": 0.3452214002609253,
       "learning_rate": 0.00016951924276746425,
-      "loss": 0.5261,
+      "loss": 0.5046,
       "step": 15
     },
     {
       "epoch": 1.6,
-      "grad_norm": 0.30262428522109985,
+      "grad_norm": 0.29285934567451477,
       "learning_rate": 0.00016456278515588024,
-      "loss": 0.5101,
+      "loss": 0.4897,
       "step": 16
     },
     {
       "epoch": 1.7,
-      "grad_norm": 0.30721279978752136,
+      "grad_norm": 0.30729544162750244,
       "learning_rate": 0.00015931797447293552,
-      "loss": 0.5059,
+      "loss": 0.501,
       "step": 17
     },
     {
       "epoch": 1.8,
-      "grad_norm": 0.2939946949481964,
+      "grad_norm": 0.2720145583152771,
       "learning_rate": 0.00015380823531633729,
-      "loss": 0.4949,
+      "loss": 0.4759,
       "step": 18
     },
     {
       "epoch": 1.9,
-      "grad_norm": 0.28288447856903076,
+      "grad_norm": 0.2651268541812897,
       "learning_rate": 0.00014805817551866838,
-      "loss": 0.4869,
+      "loss": 0.4527,
       "step": 19
     },
     {
       "epoch": 2.0,
-      "grad_norm": 0.2906552255153656,
+      "grad_norm": 0.2597902715206146,
       "learning_rate": 0.0001420934762428335,
-      "loss": 0.4605,
+      "loss": 0.4479,
       "step": 20
     },
     {
       "epoch": 2.1,
-      "grad_norm": 0.2544306218624115,
+      "grad_norm": 0.2531144917011261,
       "learning_rate": 0.00013594077728375128,
-      "loss": 0.4436,
+      "loss": 0.4056,
       "step": 21
     },
     {
       "epoch": 2.2,
-      "grad_norm": 0.2552658021450043,
+      "grad_norm": 0.2363481968641281,
       "learning_rate": 0.00012962755808856342,
-      "loss": 0.4192,
+      "loss": 0.3951,
       "step": 22
     },
     {
       "epoch": 2.3,
-      "grad_norm": 0.2546628713607788,
+      "grad_norm": 0.23172461986541748,
       "learning_rate": 0.00012318201502675285,
-      "loss": 0.4106,
+      "loss": 0.3863,
       "step": 23
     },
     {
       "epoch": 2.4,
-      "grad_norm": 0.23741765320301056,
+      "grad_norm": 0.24559740722179413,
       "learning_rate": 0.00011663293545831302,
-      "loss": 0.3938,
+      "loss": 0.3809,
       "step": 24
     },
     {
       "epoch": 2.5,
-      "grad_norm": 0.2525660991668701,
+      "grad_norm": 0.24797773361206055,
       "learning_rate": 0.00011000956916240985,
-      "loss": 0.4017,
+      "loss": 0.3768,
       "step": 25
     },
     {
       "epoch": 2.6,
-      "grad_norm": 0.2451285570859909,
+      "grad_norm": 0.25474536418914795,
       "learning_rate": 0.00010334149770076747,
-      "loss": 0.3794,
+      "loss": 0.3628,
       "step": 26
     },
     {
       "epoch": 2.7,
-      "grad_norm": 0.2524901032447815,
+      "grad_norm": 0.25486743450164795,
       "learning_rate": 9.665850229923258e-05,
-      "loss": 0.3855,
+      "loss": 0.3603,
       "step": 27
     },
     {
       "epoch": 2.8,
-      "grad_norm": 0.24502427875995636,
+      "grad_norm": 0.26718005537986755,
       "learning_rate": 8.999043083759017e-05,
-      "loss": 0.3616,
+      "loss": 0.3584,
       "step": 28
     },
     {
       "epoch": 2.9,
-      "grad_norm": 0.2539559602737427,
+      "grad_norm": 0.2610970139503479,
       "learning_rate": 8.336706454168701e-05,
-      "loss": 0.3763,
+      "loss": 0.3546,
       "step": 29
     },
     {
       "epoch": 3.0,
-      "grad_norm": 0.24636436998844147,
+      "grad_norm": 0.2513510584831238,
       "learning_rate": 7.681798497324716e-05,
-      "loss": 0.3517,
+      "loss": 0.3507,
       "step": 30
     },
     {
       "epoch": 3.1,
-      "grad_norm": 0.23345255851745605,
+      "grad_norm": 0.23601773381233215,
       "learning_rate": 7.037244191143661e-05,
-      "loss": 0.3284,
+      "loss": 0.3212,
       "step": 31
     },
     {
       "epoch": 3.2,
-      "grad_norm": 0.2510325014591217,
+      "grad_norm": 0.2521430253982544,
       "learning_rate": 6.405922271624874e-05,
-      "loss": 0.3301,
+      "loss": 0.3399,
       "step": 32
     },
     {
       "epoch": 3.3,
-      "grad_norm": 0.24276795983314514,
+      "grad_norm": 0.24177978932857513,
       "learning_rate": 5.790652375716652e-05,
-      "loss": 0.3339,
+      "loss": 0.2891,
       "step": 33
     },
     {
       "epoch": 3.4,
-      "grad_norm": 0.23698894679546356,
+      "grad_norm": 0.25264158844947815,
       "learning_rate": 5.1941824481331626e-05,
-      "loss": 0.3273,
+      "loss": 0.3047,
       "step": 34
     },
     {
       "epoch": 3.5,
-      "grad_norm": 0.24454469978809357,
+      "grad_norm": 0.23934145271778107,
       "learning_rate": 4.6191764683662744e-05,
-      "loss": 0.3073,
+      "loss": 0.2896,
       "step": 35
     },
     {
       "epoch": 3.6,
-      "grad_norm": 0.25007879734039307,
+      "grad_norm": 0.25319987535476685,
       "learning_rate": 4.0682025527064486e-05,
-      "loss": 0.3149,
+      "loss": 0.2977,
       "step": 36
     },
     {
       "epoch": 3.7,
-      "grad_norm": 0.25587430596351624,
+      "grad_norm": 0.23155006766319275,
       "learning_rate": 3.543721484411976e-05,
-      "loss": 0.3081,
+      "loss": 0.2793,
       "step": 37
     },
     {
       "epoch": 3.8,
-      "grad_norm": 0.2517322599887848,
+      "grad_norm": 0.2498036026954651,
       "learning_rate": 3.0480757232535772e-05,
-      "loss": 0.3131,
+      "loss": 0.3165,
       "step": 38
     },
     {
       "epoch": 3.9,
-      "grad_norm": 0.23721425235271454,
+      "grad_norm": 0.23940016329288483,
       "learning_rate": 2.5834789435204243e-05,
-      "loss": 0.2987,
+      "loss": 0.2938,
       "step": 39
     },
     {
       "epoch": 4.0,
-      "grad_norm": 0.23976309597492218,
+      "grad_norm": 0.23618489503860474,
       "learning_rate": 2.1520061472133902e-05,
-      "loss": 0.3148,
+      "loss": 0.2713,
       "step": 40
     },
     {
       "epoch": 4.1,
-      "grad_norm": 0.22691340744495392,
+      "grad_norm": 0.22089381515979767,
       "learning_rate": 1.7555843965823992e-05,
-      "loss": 0.2777,
+      "loss": 0.2618,
       "step": 41
     },
     {
       "epoch": 4.2,
-      "grad_norm": 0.23945631086826324,
+      "grad_norm": 0.2343159019947052,
       "learning_rate": 1.3959842073986085e-05,
-      "loss": 0.2954,
+      "loss": 0.2668,
       "step": 42
     },
     {
       "epoch": 4.3,
-      "grad_norm": 0.23303616046905518,
+      "grad_norm": 0.2323874831199646,
       "learning_rate": 1.0748116414011888e-05,
-      "loss": 0.2882,
+      "loss": 0.2645,
       "step": 43
     },
     {
       "epoch": 4.4,
-      "grad_norm": 0.24104540050029755,
+      "grad_norm": 0.24557726085186005,
       "learning_rate": 7.935011332357112e-06,
-      "loss": 0.2939,
+      "loss": 0.2728,
       "step": 44
     },
     {
       "epoch": 4.5,
-      "grad_norm": 0.23101571202278137,
+      "grad_norm": 0.24076133966445923,
       "learning_rate": 5.533090839208133e-06,
-      "loss": 0.2789,
+      "loss": 0.2729,
       "step": 45
     },
     {
       "epoch": 4.6,
-      "grad_norm": 0.2310316413640976,
+      "grad_norm": 0.22625155746936798,
       "learning_rate": 3.5530824945623542e-06,
-      "loss": 0.2925,
+      "loss": 0.2929,
       "step": 46
     },
     {
       "epoch": 4.7,
-      "grad_norm": 0.23140206933021545,
+      "grad_norm": 0.23357822000980377,
       "learning_rate": 2.003829496341325e-06,
-      "loss": 0.2824,
+      "loss": 0.2691,
       "step": 47
     },
     {
       "epoch": 4.8,
-      "grad_norm": 0.22320057451725006,
+      "grad_norm": 0.22757555544376373,
       "learning_rate": 8.922511845219971e-07,
-      "loss": 0.2975,
+      "loss": 0.2713,
       "step": 48
     },
     {
       "epoch": 4.9,
-      "grad_norm": 0.22732405364513397,
+      "grad_norm": 0.23513054847717285,
       "learning_rate": 2.2331213768468363e-07,
-      "loss": 0.2891,
+      "loss": 0.2777,
       "step": 49
     },
     {
       "epoch": 5.0,
-      "grad_norm": 0.2486155927181244,
+      "grad_norm": 0.23401321470737457,
       "learning_rate": 0.0,
-      "loss": 0.2815,
+      "loss": 0.2611,
       "step": 50
     }
   ],
@@ -376,7 +376,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 7.457968683122688e+16,
+  "total_flos": 7.430306788466688e+16,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:379e9b75008263e92f24f632d566b21e4ea482a4e680b4f98fef9cb95afe9d88
+oid sha256:e78e03cdc8dd492bf3e2caa50f5fa2767b7f8b075cdb59edee3e79f289bb3410
 size 5240