diff --git "a/ndarray-cache.json" "b/ndarray-cache.json" new file mode 100644--- /dev/null +++ "b/ndarray-cache.json" @@ -0,0 +1,2929 @@ +{ + "metadata": { + "ParamSize": 267, + "ParamBytes": 227273728.0, + "BitsPerParam": 3.6803020806911335 + }, + "records": [ + { + "dataPath": "params_shard_0.bin", + "format": "raw-shard", + "nbytes": 55912448, + "records": [ + { + "name": "model.embed_tokens.q_weight", + "shape": [ + 151936, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 55912448, + "byteOffset": 0 + } + ], + "md5sum": "698b490b21d1ff19742071511b583e5d" + }, + { + "dataPath": "params_shard_1.bin", + "format": "raw-shard", + "nbytes": 33536256, + "records": [ + { + "name": "model.embed_tokens.q_scale", + "shape": [ + 151936, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6989056, + "byteOffset": 0 + }, + { + "name": "model.layers.0.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 6989056 + }, + { + "name": "model.layers.0.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 6990848 + }, + { + "name": "model.layers.0.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 8739840 + }, + { + "name": "model.layers.0.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 8958464 + }, + { + "name": "model.layers.0.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 12538368 + }, + { + "name": "model.layers.0.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 12985856 + }, + { + "name": "model.layers.0.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 12987648 + }, + { + "name": "model.layers.0.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 12989952 + }, + { + "name": "model.layers.0.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 13413888 + }, + { + "name": "model.layers.0.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 13466880 + }, + { + "name": "model.layers.0.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 13796608 + }, + { + "name": "model.layers.1.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 13837824 + }, + { + "name": "model.layers.1.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 13839616 + }, + { + "name": "model.layers.1.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 15588608 + }, + { + "name": "model.layers.1.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 15807232 + }, + { + "name": "model.layers.1.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 19387136 + }, + { + "name": "model.layers.1.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 19834624 + }, + { + "name": "model.layers.1.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 19836416 + }, + { + "name": "model.layers.1.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 19838720 + }, + { + "name": "model.layers.1.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 20262656 + }, + { + "name": "model.layers.1.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 20315648 + }, + { + "name": "model.layers.1.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 20645376 + }, + { + "name": "model.layers.10.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 20686592 + }, + { + "name": "model.layers.10.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 20688384 + }, + { + "name": "model.layers.10.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 22437376 + }, + { + "name": "model.layers.10.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 22656000 + }, + { + "name": "model.layers.10.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 26235904 + }, + { + "name": "model.layers.10.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 26683392 + }, + { + "name": "model.layers.10.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 26685184 + }, + { + "name": "model.layers.10.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 26687488 + }, + { + "name": "model.layers.10.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 27111424 + }, + { + "name": "model.layers.10.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 27164416 + }, + { + "name": "model.layers.10.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 27494144 + }, + { + "name": "model.layers.11.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 27535360 + }, + { + "name": "model.layers.11.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 27537152 + }, + { + "name": "model.layers.11.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 29286144 + }, + { + "name": "model.layers.11.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 29504768 + }, + { + "name": "model.layers.11.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 33084672 + }, + { + "name": "model.layers.11.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 33532160 + }, + { + "name": "model.layers.11.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 33533952 + } + ], + "md5sum": "89e35bc57d8e3fe53080c99d6a330e9e" + }, + { + "dataPath": "params_shard_2.bin", + "format": "raw-shard", + "nbytes": 30212352, + "records": [ + { + "name": "model.layers.11.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 0 + }, + { + "name": "model.layers.11.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 423936 + }, + { + "name": "model.layers.11.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 476928 + }, + { + "name": "model.layers.11.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 806656 + }, + { + "name": "model.layers.12.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 847872 + }, + { + "name": "model.layers.12.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 849664 + }, + { + "name": "model.layers.12.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 2598656 + }, + { + "name": "model.layers.12.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 2817280 + }, + { + "name": "model.layers.12.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 6397184 + }, + { + "name": "model.layers.12.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 6844672 + }, + { + "name": "model.layers.12.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 6846464 + }, + { + "name": "model.layers.12.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 6848768 + }, + { + "name": "model.layers.12.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 7272704 + }, + { + "name": "model.layers.12.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 7325696 + }, + { + "name": "model.layers.12.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 7655424 + }, + { + "name": "model.layers.13.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 7696640 + }, + { + "name": "model.layers.13.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 7698432 + }, + { + "name": "model.layers.13.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 9447424 + }, + { + "name": "model.layers.13.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 9666048 + }, + { + "name": "model.layers.13.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 13245952 + }, + { + "name": "model.layers.13.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 13693440 + }, + { + "name": "model.layers.13.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 13695232 + }, + { + "name": "model.layers.13.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 13697536 + }, + { + "name": "model.layers.13.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 14121472 + }, + { + "name": "model.layers.13.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 14174464 + }, + { + "name": "model.layers.13.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 14504192 + }, + { + "name": "model.layers.14.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 14545408 + }, + { + "name": "model.layers.14.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 14547200 + }, + { + "name": "model.layers.14.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 16296192 + }, + { + "name": "model.layers.14.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 16514816 + }, + { + "name": "model.layers.14.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 20094720 + }, + { + "name": "model.layers.14.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 20542208 + }, + { + "name": "model.layers.14.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 20544000 + }, + { + "name": "model.layers.14.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 20546304 + }, + { + "name": "model.layers.14.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 20970240 + }, + { + "name": "model.layers.14.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 21023232 + }, + { + "name": "model.layers.14.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 21352960 + }, + { + "name": "model.layers.15.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 21394176 + }, + { + "name": "model.layers.15.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 21395968 + }, + { + "name": "model.layers.15.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 23144960 + }, + { + "name": "model.layers.15.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 23363584 + }, + { + "name": "model.layers.15.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 26943488 + }, + { + "name": "model.layers.15.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 27390976 + }, + { + "name": "model.layers.15.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 27392768 + }, + { + "name": "model.layers.15.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 27395072 + }, + { + "name": "model.layers.15.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 27819008 + }, + { + "name": "model.layers.15.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 27872000 + }, + { + "name": "model.layers.15.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 28201728 + }, + { + "name": "model.layers.16.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 28242944 + }, + { + "name": "model.layers.16.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 28244736 + }, + { + "name": "model.layers.16.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 29993728 + } + ], + "md5sum": "ff5ad395f5137cddf89c1e3cb927bee1" + }, + { + "dataPath": "params_shard_3.bin", + "format": "raw-shard", + "nbytes": 32276224, + "records": [ + { + "name": "model.layers.16.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 0 + }, + { + "name": "model.layers.16.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 3579904 + }, + { + "name": "model.layers.16.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 4027392 + }, + { + "name": "model.layers.16.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 4029184 + }, + { + "name": "model.layers.16.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 4031488 + }, + { + "name": "model.layers.16.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 4455424 + }, + { + "name": "model.layers.16.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 4508416 + }, + { + "name": "model.layers.16.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 4838144 + }, + { + "name": "model.layers.17.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 4879360 + }, + { + "name": "model.layers.17.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 4881152 + }, + { + "name": "model.layers.17.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 6630144 + }, + { + "name": "model.layers.17.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 6848768 + }, + { + "name": "model.layers.17.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 10428672 + }, + { + "name": "model.layers.17.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 10876160 + }, + { + "name": "model.layers.17.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 10877952 + }, + { + "name": "model.layers.17.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 10880256 + }, + { + "name": "model.layers.17.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 11304192 + }, + { + "name": "model.layers.17.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 11357184 + }, + { + "name": "model.layers.17.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 11686912 + }, + { + "name": "model.layers.18.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 11728128 + }, + { + "name": "model.layers.18.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 11729920 + }, + { + "name": "model.layers.18.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 13478912 + }, + { + "name": "model.layers.18.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 13697536 + }, + { + "name": "model.layers.18.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 17277440 + }, + { + "name": "model.layers.18.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 17724928 + }, + { + "name": "model.layers.18.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 17726720 + }, + { + "name": "model.layers.18.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 17729024 + }, + { + "name": "model.layers.18.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 18152960 + }, + { + "name": "model.layers.18.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 18205952 + }, + { + "name": "model.layers.18.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 18535680 + }, + { + "name": "model.layers.19.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 18576896 + }, + { + "name": "model.layers.19.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 18578688 + }, + { + "name": "model.layers.19.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 20327680 + }, + { + "name": "model.layers.19.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 20546304 + }, + { + "name": "model.layers.19.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 24126208 + }, + { + "name": "model.layers.19.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 24573696 + }, + { + "name": "model.layers.19.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 24575488 + }, + { + "name": "model.layers.19.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 24577792 + }, + { + "name": "model.layers.19.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 25001728 + }, + { + "name": "model.layers.19.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 25054720 + }, + { + "name": "model.layers.19.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 25384448 + }, + { + "name": "model.layers.2.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 25425664 + }, + { + "name": "model.layers.2.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 25427456 + }, + { + "name": "model.layers.2.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 27176448 + }, + { + "name": "model.layers.2.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 27395072 + }, + { + "name": "model.layers.2.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 30974976 + }, + { + "name": "model.layers.2.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 31422464 + }, + { + "name": "model.layers.2.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 31424256 + }, + { + "name": "model.layers.2.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 31426560 + }, + { + "name": "model.layers.2.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 31850496 + }, + { + "name": "model.layers.2.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 31903488 + }, + { + "name": "model.layers.2.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 32233216 + }, + { + "name": "model.layers.20.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 32274432 + } + ], + "md5sum": "baf3698db4f390e9bb4d4f9455cc2b4f" + }, + { + "dataPath": "params_shard_4.bin", + "format": "raw-shard", + "nbytes": 33394176, + "records": [ + { + "name": "model.layers.20.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 0 + }, + { + "name": "model.layers.20.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 1748992 + }, + { + "name": "model.layers.20.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 1967616 + }, + { + "name": "model.layers.20.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 5547520 + }, + { + "name": "model.layers.20.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 5995008 + }, + { + "name": "model.layers.20.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 5996800 + }, + { + "name": "model.layers.20.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 5999104 + }, + { + "name": "model.layers.20.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 6423040 + }, + { + "name": "model.layers.20.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 6476032 + }, + { + "name": "model.layers.20.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 6805760 + }, + { + "name": "model.layers.21.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 6846976 + }, + { + "name": "model.layers.21.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 6848768 + }, + { + "name": "model.layers.21.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 8597760 + }, + { + "name": "model.layers.21.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 8816384 + }, + { + "name": "model.layers.21.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 12396288 + }, + { + "name": "model.layers.21.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 12843776 + }, + { + "name": "model.layers.21.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 12845568 + }, + { + "name": "model.layers.21.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 12847872 + }, + { + "name": "model.layers.21.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 13271808 + }, + { + "name": "model.layers.21.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 13324800 + }, + { + "name": "model.layers.21.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 13654528 + }, + { + "name": "model.layers.22.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 13695744 + }, + { + "name": "model.layers.22.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 13697536 + }, + { + "name": "model.layers.22.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 15446528 + }, + { + "name": "model.layers.22.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 15665152 + }, + { + "name": "model.layers.22.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 19245056 + }, + { + "name": "model.layers.22.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 19692544 + }, + { + "name": "model.layers.22.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 19694336 + }, + { + "name": "model.layers.22.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 19696640 + }, + { + "name": "model.layers.22.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 20120576 + }, + { + "name": "model.layers.22.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 20173568 + }, + { + "name": "model.layers.22.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 20503296 + }, + { + "name": "model.layers.23.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 20544512 + }, + { + "name": "model.layers.23.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 20546304 + }, + { + "name": "model.layers.23.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 22295296 + }, + { + "name": "model.layers.23.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 22513920 + }, + { + "name": "model.layers.23.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 26093824 + }, + { + "name": "model.layers.23.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 26541312 + }, + { + "name": "model.layers.23.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 26543104 + }, + { + "name": "model.layers.23.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 26545408 + }, + { + "name": "model.layers.23.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 26969344 + }, + { + "name": "model.layers.23.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 27022336 + }, + { + "name": "model.layers.23.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 27352064 + }, + { + "name": "model.layers.3.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 27393280 + }, + { + "name": "model.layers.3.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 27395072 + }, + { + "name": "model.layers.3.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 29144064 + }, + { + "name": "model.layers.3.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 29362688 + }, + { + "name": "model.layers.3.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 32942592 + }, + { + "name": "model.layers.3.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 33390080 + }, + { + "name": "model.layers.3.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 33391872 + } + ], + "md5sum": "84dda69b44a56a9a170355907fb545a9" + }, + { + "dataPath": "params_shard_5.bin", + "format": "raw-shard", + "nbytes": 30212352, + "records": [ + { + "name": "model.layers.3.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 0 + }, + { + "name": "model.layers.3.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 423936 + }, + { + "name": "model.layers.3.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 476928 + }, + { + "name": "model.layers.3.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 806656 + }, + { + "name": "model.layers.4.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 847872 + }, + { + "name": "model.layers.4.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 849664 + }, + { + "name": "model.layers.4.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 2598656 + }, + { + "name": "model.layers.4.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 2817280 + }, + { + "name": "model.layers.4.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 6397184 + }, + { + "name": "model.layers.4.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 6844672 + }, + { + "name": "model.layers.4.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 6846464 + }, + { + "name": "model.layers.4.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 6848768 + }, + { + "name": "model.layers.4.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 7272704 + }, + { + "name": "model.layers.4.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 7325696 + }, + { + "name": "model.layers.4.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 7655424 + }, + { + "name": "model.layers.5.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 7696640 + }, + { + "name": "model.layers.5.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 7698432 + }, + { + "name": "model.layers.5.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 9447424 + }, + { + "name": "model.layers.5.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 9666048 + }, + { + "name": "model.layers.5.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 13245952 + }, + { + "name": "model.layers.5.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 13693440 + }, + { + "name": "model.layers.5.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 13695232 + }, + { + "name": "model.layers.5.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 13697536 + }, + { + "name": "model.layers.5.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 14121472 + }, + { + "name": "model.layers.5.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 14174464 + }, + { + "name": "model.layers.5.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 14504192 + }, + { + "name": "model.layers.6.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 14545408 + }, + { + "name": "model.layers.6.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 14547200 + }, + { + "name": "model.layers.6.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 16296192 + }, + { + "name": "model.layers.6.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 16514816 + }, + { + "name": "model.layers.6.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 20094720 + }, + { + "name": "model.layers.6.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 20542208 + }, + { + "name": "model.layers.6.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 20544000 + }, + { + "name": "model.layers.6.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 20546304 + }, + { + "name": "model.layers.6.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 20970240 + }, + { + "name": "model.layers.6.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 21023232 + }, + { + "name": "model.layers.6.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 21352960 + }, + { + "name": "model.layers.7.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 21394176 + }, + { + "name": "model.layers.7.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 21395968 + }, + { + "name": "model.layers.7.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 23144960 + }, + { + "name": "model.layers.7.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 23363584 + }, + { + "name": "model.layers.7.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 26943488 + }, + { + "name": "model.layers.7.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 27390976 + }, + { + "name": "model.layers.7.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 27392768 + }, + { + "name": "model.layers.7.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 27395072 + }, + { + "name": "model.layers.7.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 27819008 + }, + { + "name": "model.layers.7.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 27872000 + }, + { + "name": "model.layers.7.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 28201728 + }, + { + "name": "model.layers.8.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 28242944 + }, + { + "name": "model.layers.8.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 28244736 + }, + { + "name": "model.layers.8.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 29993728 + } + ], + "md5sum": "80a6ba70f115fb8e5c5373abfbbdf3d4" + }, + { + "dataPath": "params_shard_6.bin", + "format": "raw-shard", + "nbytes": 11729920, + "records": [ + { + "name": "model.layers.8.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 0 + }, + { + "name": "model.layers.8.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 3579904 + }, + { + "name": "model.layers.8.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 4027392 + }, + { + "name": "model.layers.8.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 4029184 + }, + { + "name": "model.layers.8.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 4031488 + }, + { + "name": "model.layers.8.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 4455424 + }, + { + "name": "model.layers.8.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 4508416 + }, + { + "name": "model.layers.8.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 4838144 + }, + { + "name": "model.layers.9.input_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 4879360 + }, + { + "name": "model.layers.9.mlp.down_proj.q_weight", + "shape": [ + 896, + 488 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1748992, + "byteOffset": 4881152 + }, + { + "name": "model.layers.9.mlp.down_proj.q_scale", + "shape": [ + 896, + 122 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 218624, + "byteOffset": 6630144 + }, + { + "name": "model.layers.9.mlp.gate_up_proj.q_weight", + "shape": [ + 9728, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 3579904, + "byteOffset": 6848768 + }, + { + "name": "model.layers.9.mlp.gate_up_proj.q_scale", + "shape": [ + 9728, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 447488, + "byteOffset": 10428672 + }, + { + "name": "model.layers.9.post_attention_layernorm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 10876160 + }, + { + "name": "model.layers.9.self_attn.c_attn.bias", + "shape": [ + 1152 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2304, + "byteOffset": 10877952 + }, + { + "name": "model.layers.9.self_attn.c_attn.q_weight", + "shape": [ + 1152, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 423936, + "byteOffset": 10880256 + }, + { + "name": "model.layers.9.self_attn.c_attn.q_scale", + "shape": [ + 1152, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 52992, + "byteOffset": 11304192 + }, + { + "name": "model.layers.9.self_attn.o_proj.q_weight", + "shape": [ + 896, + 92 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 329728, + "byteOffset": 11357184 + }, + { + "name": "model.layers.9.self_attn.o_proj.q_scale", + "shape": [ + 896, + 23 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 41216, + "byteOffset": 11686912 + }, + { + "name": "model.norm.weight", + "shape": [ + 896 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1792, + "byteOffset": 11728128 + } + ], + "md5sum": "445184376a16e4800bed89b1b7358316" + } + ] +} \ No newline at end of file