diff --git "a/ndarray-cache-b16.json" "b/ndarray-cache-b16.json" new file mode 100644--- /dev/null +++ "b/ndarray-cache-b16.json" @@ -0,0 +1,3137 @@ +{ + "metadata": { + "ParamSize": 195, + "ParamBytes": 32121044992.0, + "BitsPerParam": 32.0 + }, + "records": [ + { + "dataPath": "params_shard_0.bin", + "format": "raw-shard", + "nbytes": 1050673152, + "records": [ + { + "name": "lm_head.weight", + "shape": [ + 128256, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1050673152, + "byteOffset": 0 + } + ], + "md5sum": "e09bca90c15e15f95c1a5e7dc8847aab" + }, + { + "dataPath": "params_shard_1.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.31.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "4eb6d39716b0c749773fdc682e19be03" + }, + { + "dataPath": "params_shard_2.bin", + "format": "raw-shard", + "nbytes": 1050673152, + "records": [ + { + "name": "model.embed_tokens.weight", + "shape": [ + 128256, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1050673152, + "byteOffset": 0 + } + ], + "md5sum": "32555e18d0c234239977469d0a80cc3c" + }, + { + "dataPath": "params_shard_3.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.0.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "8261454b9658c96d391c69f56a10471a" + }, + { + "dataPath": "params_shard_4.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.0.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "93cf707a761f40306f02506094357eb2" + }, + { + "dataPath": "params_shard_5.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.0.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "a048209d3976f367a80c4fe973b784d5" + }, + { + "dataPath": "params_shard_6.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.0.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "f3c2a1ab5b5d57ca8e907b6c35cfef80" + }, + { + "dataPath": "params_shard_7.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.1.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "31db6e32f134586dac84645fa5a6c5ea" + }, + { + "dataPath": "params_shard_8.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.1.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "9469a0ef32c48adc9bf3507745d0ab04" + }, + { + "dataPath": "params_shard_9.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.1.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "557435f60c79552142db3e4cb1808a1c" + }, + { + "dataPath": "params_shard_10.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.1.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "2571c16bc711b2a6f6ab5d3109a40e20" + }, + { + "dataPath": "params_shard_11.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.2.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "e6832047f5d4e07aca87f0169c6d7729" + }, + { + "dataPath": "params_shard_12.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.2.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "f02a031330ee0b3c851d7da9ac690ce3" + }, + { + "dataPath": "params_shard_13.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.2.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "deda1e374abfa1f67a2a639973de9507" + }, + { + "dataPath": "params_shard_14.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.2.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "49cf494d51fcacb729551cf2f93fe6c5" + }, + { + "dataPath": "params_shard_15.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.3.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "76acb288da598ecf5c0e2ea18d964553" + }, + { + "dataPath": "params_shard_16.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.3.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "b622c8d032ed563922e563fdc172ab0f" + }, + { + "dataPath": "params_shard_17.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.3.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "9bc2194c8cb03b44d42eb92f01dbc33d" + }, + { + "dataPath": "params_shard_18.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.3.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "5f395c6fd6153ab6dfcc743f958a05cc" + }, + { + "dataPath": "params_shard_19.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.4.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "524528c61ed75fcd65b459c708c84ae5" + }, + { + "dataPath": "params_shard_20.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.4.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "7efd30f5dbf9f997dc633400e304b337" + }, + { + "dataPath": "params_shard_21.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.4.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "cae5539fa36bde7b662f0a53c3a9b69b" + }, + { + "dataPath": "params_shard_22.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.4.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "b387fa395750878b124daee5dbfbf20a" + }, + { + "dataPath": "params_shard_23.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.5.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "c954eaa0ac38fbc9ade36cb0646ca4cf" + }, + { + "dataPath": "params_shard_24.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.5.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "adf3bd4844fde5779349e005642fa24a" + }, + { + "dataPath": "params_shard_25.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.5.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "fd40d7e2e501019bd6f9ed18071dcdfc" + }, + { + "dataPath": "params_shard_26.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.5.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "c4b746f2d5f369c288a3afd717f94e30" + }, + { + "dataPath": "params_shard_27.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.6.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "479b5d6d3222d2ed388b70758c1e81f2" + }, + { + "dataPath": "params_shard_28.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.6.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "1482ac1db2dd4b0bc0cde5573f83d106" + }, + { + "dataPath": "params_shard_29.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.6.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "1de7f2b6070fc4f40ff6a566dda2d579" + }, + { + "dataPath": "params_shard_30.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.6.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "73afce0bd5c1932586b4cbbafdb8cb0a" + }, + { + "dataPath": "params_shard_31.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.7.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "45960e46a442055c3789b703ba81faa3" + }, + { + "dataPath": "params_shard_32.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.7.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "eefdaba21cf5577f81c3057bdbd60117" + }, + { + "dataPath": "params_shard_33.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.7.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "a52fce13f644b9ba33cad4e2b3bbc11b" + }, + { + "dataPath": "params_shard_34.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.7.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "535877888f6eb5efed8f09c5dfa1c72a" + }, + { + "dataPath": "params_shard_35.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.8.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "9de98d2237c6fbd4c05701618a22c791" + }, + { + "dataPath": "params_shard_36.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.8.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "f45e9be89a1a5aa837b91289163c31bb" + }, + { + "dataPath": "params_shard_37.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.8.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "05e873c0f96a7edb2e2678c509c89f0f" + }, + { + "dataPath": "params_shard_38.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.8.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "a8a3bff46d8106cbab51c3249f02ff92" + }, + { + "dataPath": "params_shard_39.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.10.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "cb39bafba0e6bbbf8039441dd11aea39" + }, + { + "dataPath": "params_shard_40.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.10.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "0916181cf1cd2b59f6cc6915b63c3bfa" + }, + { + "dataPath": "params_shard_41.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.10.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "021b45b77738135afba1eabca76ce7bc" + }, + { + "dataPath": "params_shard_42.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.10.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "7d6027881de96305fcbfc0437a87ac1d" + }, + { + "dataPath": "params_shard_43.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.11.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "7af3d10d6b01b70b1aa523925657bfd6" + }, + { + "dataPath": "params_shard_44.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.11.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "e4bbd708d43eec244db8ac618f849e93" + }, + { + "dataPath": "params_shard_45.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.11.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "1c50401dc80033cb88741ccebe419492" + }, + { + "dataPath": "params_shard_46.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.11.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "dc17c204055dd46ed89ce96dcb073bd1" + }, + { + "dataPath": "params_shard_47.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.12.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "2eff8f4be3e42621f53336713d42d7a4" + }, + { + "dataPath": "params_shard_48.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.12.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "89b1e81c0e6fb4d64399cf304726ca17" + }, + { + "dataPath": "params_shard_49.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.12.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "8ed541396556e9f945a2a403ed30dfff" + }, + { + "dataPath": "params_shard_50.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.12.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "a8008630b457c4fe57b06e7a3bb5fe5d" + }, + { + "dataPath": "params_shard_51.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.13.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "91951886d53f5609abe12bd5abbf9ce6" + }, + { + "dataPath": "params_shard_52.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.13.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "e049a072dff7844fa73a7f4f3dead0cc" + }, + { + "dataPath": "params_shard_53.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.13.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "66274945c12b00b65a83b4b4ba5a62ab" + }, + { + "dataPath": "params_shard_54.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.13.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "03f85675d28a6a22e69b552907064842" + }, + { + "dataPath": "params_shard_55.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.14.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "30690352feaf3c6a250cae87b9bfe1c1" + }, + { + "dataPath": "params_shard_56.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.14.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "0ef0c51f0b782a9c373fc2d0c3ec3761" + }, + { + "dataPath": "params_shard_57.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.14.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "236bc385004d7da0e0212bf67d0b5972" + }, + { + "dataPath": "params_shard_58.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.14.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "3ec6833b9928af2234fc8b2664b955f4" + }, + { + "dataPath": "params_shard_59.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.15.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "0435e1b28aee8211e632b74a66a7ce99" + }, + { + "dataPath": "params_shard_60.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.15.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "9f21ee64f47884f78ef75fcb6c62c90f" + }, + { + "dataPath": "params_shard_61.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.15.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "8af4e6bf20fcd0c8f76df53ad59141df" + }, + { + "dataPath": "params_shard_62.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.15.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "6d891b4ebf9554cc8dc3d1d3f4532ce3" + }, + { + "dataPath": "params_shard_63.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.16.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "ba942be489a1ff89d60dde8d65567626" + }, + { + "dataPath": "params_shard_64.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.16.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "9f08d00c3a37c72ea930fa28be6fde11" + }, + { + "dataPath": "params_shard_65.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.16.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "db9c07561d982e7953426a2d4317bc58" + }, + { + "dataPath": "params_shard_66.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.16.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "577a3b4a7f1f64ba7bc276aef6c7c263" + }, + { + "dataPath": "params_shard_67.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.17.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "e65ad2e6e3f40d68be370d4d84b8078f" + }, + { + "dataPath": "params_shard_68.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.17.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "ca57100db441109435c148c5e548b199" + }, + { + "dataPath": "params_shard_69.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.17.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "2dd83853b6feb07581b3860738ebf29c" + }, + { + "dataPath": "params_shard_70.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.17.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "e3610693398c91b2fdc38c95650fc374" + }, + { + "dataPath": "params_shard_71.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.18.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "45e146eb956216ea2d863e03d1138dbb" + }, + { + "dataPath": "params_shard_72.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.18.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "e88d25d1089d78b5adad200487e33e5d" + }, + { + "dataPath": "params_shard_73.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.18.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "71930eb27abc47f5c7797ad0ddda0ff8" + }, + { + "dataPath": "params_shard_74.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.18.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "310bec8e4483f5423d3c64c0ea9096d8" + }, + { + "dataPath": "params_shard_75.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.19.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "6ecb6ce8a2bf8447a3f1722d9858929b" + }, + { + "dataPath": "params_shard_76.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.19.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "1db0944f57024d4b46cfed243524224e" + }, + { + "dataPath": "params_shard_77.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.19.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "8486a7d99c1faf0d9420def8070c1b76" + }, + { + "dataPath": "params_shard_78.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.19.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "d1a8b0b82fcdcb80d879cfe33397cd8d" + }, + { + "dataPath": "params_shard_79.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.20.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "d43d8bd4ef18d71f3102173917f26a22" + }, + { + "dataPath": "params_shard_80.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.20.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "56d510f0a14488069408566da1fa835b" + }, + { + "dataPath": "params_shard_81.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.20.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "db6930aa2c05246b866170833287b2aa" + }, + { + "dataPath": "params_shard_82.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.9.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "df977b3b99c65a80cf48df50e7118633" + }, + { + "dataPath": "params_shard_83.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.9.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "aa126ba57ade1f0ddd292eeecaceea3a" + }, + { + "dataPath": "params_shard_84.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.9.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "6271d778c31d6dc062546d239250c5dc" + }, + { + "dataPath": "params_shard_85.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.9.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "a0202303d8cef47505bd8794b1b4f609" + }, + { + "dataPath": "params_shard_86.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.20.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "285c33e830ccfeab745abf2d4c8a33bb" + }, + { + "dataPath": "params_shard_87.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.21.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "fd78c7f5634bb76a114a52ac207a5395" + }, + { + "dataPath": "params_shard_88.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.21.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "a05c1dafff0be1a1152c74e2527202da" + }, + { + "dataPath": "params_shard_89.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.21.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "70e01b32ab7f26301fede22ae43832f3" + }, + { + "dataPath": "params_shard_90.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.21.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "ff7acf378f7ff8ae242a0a1d5aa1d2f2" + }, + { + "dataPath": "params_shard_91.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.22.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "4a398af30cc79405e2eae6fca27b504b" + }, + { + "dataPath": "params_shard_92.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.22.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "3fc9c50885d831ad4539086f029885d6" + }, + { + "dataPath": "params_shard_93.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.22.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "fffc98e1235f65188db19c9038b5a679" + }, + { + "dataPath": "params_shard_94.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.22.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "f0b6e99bb826eb652695d0d3ec9caf58" + }, + { + "dataPath": "params_shard_95.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.23.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "38cb73ff9e2467507a68f14fcd101ef3" + }, + { + "dataPath": "params_shard_96.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.23.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "a87ba1e35d87da965695e59ecbf38a18" + }, + { + "dataPath": "params_shard_97.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.23.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "1bfe742c91c1186705a4a44066ec67b6" + }, + { + "dataPath": "params_shard_98.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.23.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "28faeb2f02b12d5fc7a778a480251c7c" + }, + { + "dataPath": "params_shard_99.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.24.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "f12d7a24a6d896d91e647575c0556b31" + }, + { + "dataPath": "params_shard_100.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.24.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "58ed664c8ce3993a309066affd94a4f8" + }, + { + "dataPath": "params_shard_101.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.24.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "662ac15f2e943c704c7cb479d2c73661" + }, + { + "dataPath": "params_shard_102.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.24.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "d4203f17b475d433f60858323c27f897" + }, + { + "dataPath": "params_shard_103.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.25.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "4ea0c6332741cbd6cce18158ab8ce913" + }, + { + "dataPath": "params_shard_104.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.25.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "4928d44b6e55d66c73bc7f43d18cad67" + }, + { + "dataPath": "params_shard_105.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.25.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "6f48ee7398dffea452c9ea85c3b10428" + }, + { + "dataPath": "params_shard_106.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.25.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "b10bdea1a511ba77c3baa6a92660ca80" + }, + { + "dataPath": "params_shard_107.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.26.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "cc1ad960e7f9bc2e00403dad85e0152d" + }, + { + "dataPath": "params_shard_108.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.26.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "a924b4895973a298789ac3df40afa57b" + }, + { + "dataPath": "params_shard_109.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.26.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "7feee4993f1c4a8a9a35fa07601f8e66" + }, + { + "dataPath": "params_shard_110.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.26.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "bc3e8babe81a21273aacae78e2d762ef" + }, + { + "dataPath": "params_shard_111.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.27.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "e7b52f4cdda068f86e6f3b459db0ee91" + }, + { + "dataPath": "params_shard_112.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.27.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "6632e8017758b116b916410883561fad" + }, + { + "dataPath": "params_shard_113.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.27.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "f283251a45345d4754b730283221f985" + }, + { + "dataPath": "params_shard_114.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.27.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "424872f6804017f1c1842eadebfaceeb" + }, + { + "dataPath": "params_shard_115.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.28.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "4e6572593981a76914f992c263e174d8" + }, + { + "dataPath": "params_shard_116.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.28.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "f07310f9b9808ca557b842d538b795c3" + }, + { + "dataPath": "params_shard_117.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.28.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "d234d3038461b1c40763706f137f0544" + }, + { + "dataPath": "params_shard_118.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.28.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "6b816444a00ed86f3a722f1c308d4bfd" + }, + { + "dataPath": "params_shard_119.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.29.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "6f301ea1e9f1cea94df6fa074c72d43a" + }, + { + "dataPath": "params_shard_120.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.29.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "b669ba9da0160824f625437c5be02455" + }, + { + "dataPath": "params_shard_121.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.29.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "84989770351dfb08a55f44c5b09fee95" + }, + { + "dataPath": "params_shard_122.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.29.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "590f0a170264eda5fab4cf83827b038b" + }, + { + "dataPath": "params_shard_123.bin", + "format": "raw-shard", + "nbytes": 117440512, + "records": [ + { + "name": "model.layers.30.mlp.down_proj.weight", + "shape": [ + 4096, + 14336 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 117440512, + "byteOffset": 0 + } + ], + "md5sum": "c6e326c8d5ea7473f25cfd3f5ba814ce" + }, + { + "dataPath": "params_shard_124.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.30.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "71da48dad760327388e0ed8d4624932c" + }, + { + "dataPath": "params_shard_125.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.30.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "3a98668165157fd8e6bacac7dcd76e5a" + }, + { + "dataPath": "params_shard_126.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.30.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "5d6d0aa8ede4077920f7bfd67d59a08b" + }, + { + "dataPath": "params_shard_127.bin", + "format": "raw-shard", + "nbytes": 234881024, + "records": [ + { + "name": "model.layers.31.mlp.gate_up_proj.weight", + "shape": [ + 28672, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 234881024, + "byteOffset": 0 + } + ], + "md5sum": "6b6c18bd4de89746503dad4d2ab84cd4" + }, + { + "dataPath": "params_shard_128.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.31.self_attn.qkv_proj.weight", + "shape": [ + 6144, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "dd84f9b13bdc127fa6c0dcb04a4ae657" + }, + { + "dataPath": "params_shard_129.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "model.layers.31.self_attn.o_proj.weight", + "shape": [ + 4096, + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "b21e0412cb46ffcd8e3aaaa11a08326b" + }, + { + "dataPath": "params_shard_130.bin", + "format": "raw-shard", + "nbytes": 532480, + "records": [ + { + "name": "model.layers.31.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 0 + }, + { + "name": "model.layers.31.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 8192 + }, + { + "name": "model.norm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 16384 + }, + { + "name": "model.layers.0.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 24576 + }, + { + "name": "model.layers.0.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 32768 + }, + { + "name": "model.layers.1.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 40960 + }, + { + "name": "model.layers.1.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 49152 + }, + { + "name": "model.layers.2.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 57344 + }, + { + "name": "model.layers.2.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 65536 + }, + { + "name": "model.layers.3.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 73728 + }, + { + "name": "model.layers.3.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 81920 + }, + { + "name": "model.layers.4.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 90112 + }, + { + "name": "model.layers.4.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 98304 + }, + { + "name": "model.layers.5.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 106496 + }, + { + "name": "model.layers.5.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 114688 + }, + { + "name": "model.layers.6.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 122880 + }, + { + "name": "model.layers.6.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 131072 + }, + { + "name": "model.layers.7.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 139264 + }, + { + "name": "model.layers.7.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 147456 + }, + { + "name": "model.layers.8.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 155648 + }, + { + "name": "model.layers.8.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 163840 + }, + { + "name": "model.layers.10.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 172032 + }, + { + "name": "model.layers.10.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 180224 + }, + { + "name": "model.layers.11.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 188416 + }, + { + "name": "model.layers.11.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 196608 + }, + { + "name": "model.layers.12.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 204800 + }, + { + "name": "model.layers.12.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 212992 + }, + { + "name": "model.layers.13.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 221184 + }, + { + "name": "model.layers.13.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 229376 + }, + { + "name": "model.layers.14.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 237568 + }, + { + "name": "model.layers.14.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 245760 + }, + { + "name": "model.layers.15.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 253952 + }, + { + "name": "model.layers.15.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 262144 + }, + { + "name": "model.layers.16.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 270336 + }, + { + "name": "model.layers.16.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 278528 + }, + { + "name": "model.layers.17.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 286720 + }, + { + "name": "model.layers.17.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 294912 + }, + { + "name": "model.layers.18.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 303104 + }, + { + "name": "model.layers.18.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 311296 + }, + { + "name": "model.layers.19.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 319488 + }, + { + "name": "model.layers.19.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 327680 + }, + { + "name": "model.layers.9.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 335872 + }, + { + "name": "model.layers.9.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 344064 + }, + { + "name": "model.layers.20.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 352256 + }, + { + "name": "model.layers.20.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 360448 + }, + { + "name": "model.layers.21.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 368640 + }, + { + "name": "model.layers.21.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 376832 + }, + { + "name": "model.layers.22.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 385024 + }, + { + "name": "model.layers.22.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 393216 + }, + { + "name": "model.layers.23.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 401408 + }, + { + "name": "model.layers.23.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 409600 + }, + { + "name": "model.layers.24.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 417792 + }, + { + "name": "model.layers.24.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 425984 + }, + { + "name": "model.layers.25.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 434176 + }, + { + "name": "model.layers.25.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 442368 + }, + { + "name": "model.layers.26.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 450560 + }, + { + "name": "model.layers.26.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 458752 + }, + { + "name": "model.layers.27.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 466944 + }, + { + "name": "model.layers.27.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 475136 + }, + { + "name": "model.layers.28.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 483328 + }, + { + "name": "model.layers.28.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 491520 + }, + { + "name": "model.layers.29.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 499712 + }, + { + "name": "model.layers.29.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 507904 + }, + { + "name": "model.layers.30.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 516096 + }, + { + "name": "model.layers.30.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 8192, + "byteOffset": 524288 + } + ], + "md5sum": "feda70cb84f9eabd336ef10effaad2dc" + } + ] +} \ No newline at end of file