update

quant_strategy.json

@@ -1,8 +1,8 @@
 {
     "measurement": {
         "model.layers.0": {
-            "accuracy": 0.9305233955383301,
-            "total_bits": 47656992,
+            "accuracy": 0.8778786659240723,
+            "total_bits": 41980320,
             "q_proj": {
                 "group_size": {
                     "2": 64
@@ -65,10 +65,10 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -89,14 +89,14 @@
             }
         },
         "model.layers.1": {
-            "accuracy": 0.961686372756958,
-            "total_bits": 51784992,
+            "accuracy": 0.8126821517944336,
+            "total_bits": 30627648,
             "q_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -105,10 +105,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -141,10 +141,10 @@
             },
             "up_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -153,10 +153,10 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -165,10 +165,10 @@
             },
             "down_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -177,14 +177,14 @@
             }
         },
         "model.layers.2": {
-            "accuracy": 0.9639403820037842,
-            "total_bits": 51784992,
+            "accuracy": 0.8082990646362305,
+            "total_bits": 30627648,
             "q_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -193,10 +193,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -229,10 +229,10 @@
             },
             "up_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -241,10 +241,10 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -253,10 +253,10 @@
             },
             "down_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -265,8 +265,8 @@
             }
         },
         "model.layers.3": {
-            "accuracy": 0.9068508148193359,
-            "total_bits": 41980320,
+            "accuracy": 0.8619828224182129,
+            "total_bits": 34239648,
             "q_proj": {
                 "group_size": {
                     "2": 64
@@ -305,10 +305,10 @@
             },
             "o_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -317,10 +317,10 @@
             },
             "up_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -353,8 +353,8 @@
             }
         },
         "model.layers.4": {
-            "accuracy": 0.9586918354034424,
-            "total_bits": 49720992,
+            "accuracy": 0.8780388832092285,
+            "total_bits": 36303648,
             "q_proj": {
                 "group_size": {
                     "2": 64
@@ -369,10 +369,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -405,10 +405,10 @@
             },
             "up_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -417,10 +417,10 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -529,8 +529,8 @@
             }
         },
         "model.layers.6": {
-            "accuracy": 0.9293514490127563,
-            "total_bits": 30627648,
+            "accuracy": 0.9222159385681152,
+            "total_bits": 28563648,
             "q_proj": {
                 "group_size": {
                     "2": 64
@@ -569,10 +569,10 @@
             },
             "o_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -617,8 +617,8 @@
             }
         },
         "model.layers.7": {
-            "accuracy": 0.9222983121871948,
-            "total_bits": 30627648,
+            "accuracy": 0.9137954711914062,
+            "total_bits": 28563648,
             "q_proj": {
                 "group_size": {
                     "2": 64
@@ -633,10 +633,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -705,8 +705,8 @@
             }
         },
         "model.layers.8": {
-            "accuracy": 0.9248720407485962,
-            "total_bits": 30627648,
+            "accuracy": 0.9160647392272949,
+            "total_bits": 28563648,
             "q_proj": {
                 "group_size": {
                     "2": 64
@@ -721,10 +721,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -881,8 +881,8 @@
             }
         },
         "model.layers.10": {
-            "accuracy": 0.9260375499725342,
-            "total_bits": 30627648,
+            "accuracy": 0.9132890701293945,
+            "total_bits": 28563648,
             "q_proj": {
                 "group_size": {
                     "2": 64
@@ -921,10 +921,10 @@
             },
             "o_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -969,8 +969,8 @@
             }
         },
         "model.layers.11": {
-            "accuracy": 0.9440933465957642,
-            "total_bits": 38367648,
+            "accuracy": 0.908735990524292,
+            "total_bits": 28563648,
             "q_proj": {
                 "group_size": {
                     "2": 64
@@ -985,10 +985,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -1009,10 +1009,10 @@
             },
             "o_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -1045,10 +1045,10 @@
             },
             "down_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -1057,8 +1057,8 @@
             }
         },
         "model.layers.12": {
-            "accuracy": 0.9411404132843018,
-            "total_bits": 36303648,
+            "accuracy": 0.9236173629760742,
+            "total_bits": 30627648,
             "q_proj": {
                 "group_size": {
                     "2": 64
@@ -1133,10 +1133,10 @@
             },
             "down_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -1321,8 +1321,8 @@
             }
         },
         "model.layers.15": {
-            "accuracy": 0.9073450565338135,
-            "total_bits": 30627648,
+            "accuracy": 0.9489641189575195,
+            "total_bits": 41980320,
             "q_proj": {
                 "group_size": {
                     "2": 64
@@ -1373,10 +1373,10 @@
             },
             "up_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1397,10 +1397,10 @@
             },
             "down_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1409,8 +1409,8 @@
             }
         },
         "model.layers.16": {
-            "accuracy": 0.9269454479217529,
-            "total_bits": 36303648,
+            "accuracy": 0.9460692405700684,
+            "total_bits": 41980320,
             "q_proj": {
                 "group_size": {
                     "2": 64
@@ -1461,10 +1461,10 @@
             },
             "up_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1497,14 +1497,14 @@
             }
         },
         "model.layers.17": {
-            "accuracy": 0.9374923706054688,
-            "total_bits": 41980320,
+            "accuracy": 0.97892165184021,
+            "total_bits": 51784992,
             "q_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1513,10 +1513,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1561,10 +1561,10 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1585,14 +1585,14 @@
             }
         },
         "model.layers.18": {
-            "accuracy": 0.9396567344665527,
-            "total_bits": 41980320,
+            "accuracy": 0.9773706197738647,
+            "total_bits": 51784992,
             "q_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1601,10 +1601,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1649,10 +1649,10 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1673,14 +1673,14 @@
             }
         },
         "model.layers.19": {
-            "accuracy": 0.9197125434875488,
-            "total_bits": 36303648,
+            "accuracy": 0.9783408641815186,
+            "total_bits": 51784992,
             "q_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1689,10 +1689,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1725,10 +1725,10 @@
             },
             "up_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1737,10 +1737,10 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1761,14 +1761,14 @@
             }
         },
         "model.layers.20": {
-            "accuracy": 0.9247608184814453,
-            "total_bits": 38367648,
+            "accuracy": 0.9780358076095581,
+            "total_bits": 51784992,
             "q_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1813,10 +1813,10 @@
             },
             "up_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1825,10 +1825,10 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1849,14 +1849,14 @@
             }
         },
         "model.layers.21": {
-            "accuracy": 0.9209287166595459,
-            "total_bits": 38367648,
+            "accuracy": 0.9766302108764648,
+            "total_bits": 51784992,
             "q_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1901,10 +1901,10 @@
             },
             "up_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1913,10 +1913,10 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1937,14 +1937,14 @@
             }
         },
         "model.layers.22": {
-            "accuracy": 0.9368994235992432,
-            "total_bits": 41980320,
+            "accuracy": 0.9679172039031982,
+            "total_bits": 51784992,
             "q_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1953,10 +1953,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -2001,10 +2001,10 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -2025,14 +2025,14 @@
             }
         },
         "model.layers.23": {
-            "accuracy": 0.950645923614502,
-            "total_bits": 47656992,
+            "accuracy": 0.9569640159606934,
+            "total_bits": 51784992,
             "q_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -2041,10 +2041,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1