Spaces:
Running
Running
| Model,Quantization,Params (B),Model Size (GB),Prefill (tokens/s),Decode (tokens/s),Backend | |
| gemma-2-9b,Q8_0,10.159,10.796,2.169,0.012,llama_cpp | |
| DeepSeek-V2-Lite,Q4_K_M,15.706,10.36,4.304,1.764,llama_cpp | |
| aya-expanse-8b,Q8_0,9.077,9.644,3.1,0.027,llama_cpp | |
| Yi-1.5-9B,Q8_0,8.829,9.382,2.585,0.019,llama_cpp | |
| Qwen2.5-14B,Q4_K_M,14.77,8.982,1.916,0.018,llama_cpp | |
| DeepSeek-V2-Lite,Q4_0_4_4,15.706,8.901,7.788,3.867,llama_cpp | |
| Phi-3-medium-128k-instruct,Q4_K_M,13.96,8.566,1.819,0.02,llama_cpp | |
| Hermes-3-Llama-3.1-8B,Q8_0,8.03,8.533,3.286,0.922,llama_cpp | |
| Qwen2.5-14B,Q4_0_4_4,14.77,8.512,4.698,0.028,llama_cpp | |
| internlm2_5-7b-chat,Q8_0,7.738,8.222,3.258,1.238,llama_cpp | |
| dolphin-2.9.2-qwen2-7b,Q8_0,7.616,8.093,4.241,1.301,llama_cpp | |
| Qwen2.5-7B,Q8_0,7.616,8.093,4.253,1.302,llama_cpp | |
| Phi-3-medium-128k-instruct,Q4_0_4_4,13.96,7.896,4.715,0.038,llama_cpp | |
| NexusRaven-V2-13B,Q4_K_M,13.016,7.865,2.066,0.035,llama_cpp | |
| Mistral-7B-Instruct-v0.3,Q8_0,7.248,7.702,4.104,1.29,llama_cpp | |
| dolphin-2.9.3-mistral-7B-32k,Q8_0,7.248,7.702,4.135,1.294,llama_cpp | |
| Yarn-Mistral-7b-128k,Q8_0,7.242,7.695,4.082,1.292,llama_cpp | |
| Starling-LM-7B-beta,Q8_0,7.242,7.695,4.132,1.296,llama_cpp | |
| Mistral-Nemo-Base-2407,Q4_K_M,12.248,7.469,2.453,1.358,llama_cpp | |
| NexusRaven-V2-13B,Q4_0_4_4,13.016,7.365,4.979,1.348,llama_cpp | |
| OLMoE-1B-7B-0924,Q8_0,6.919,7.358,26.942,7.489,llama_cpp | |
| OLMo-7B-0724-hf,Q8_0,6.888,7.319,4.515,1.371,llama_cpp | |
| mpt-7b-instruct,Q8_0,6.856,7.285,4.287,1.367,llama_cpp | |
| Amber,Q8_0,6.738,7.16,4.442,1.373,llama_cpp | |
| Mistral-Nemo-Base-2407,Q4_0_4_4,12.248,7.064,9.103,1.48,llama_cpp | |
| gemma-2-9b,Q4_K_M,10.159,6.508,3.531,1.629,llama_cpp | |
| Yarn-Solar-10b-64k,Q4_K_M,10.732,6.461,2.905,1.503,llama_cpp | |
| SOLAR-10.7B-v1.0,Q4_K_M,10.732,6.461,2.925,1.505,llama_cpp | |
| SOLAR-10.7B-Instruct-v1.0,Q4_K_M,10.732,6.461,2.916,1.506,llama_cpp | |
| Yi-1.5-6B,Q8_0,6.061,6.441,5.269,1.584,llama_cpp | |
| gemma-2-9b,Q4_0_4_4,10.159,6.19,10.553,1.757,llama_cpp | |
| SOLAR-10.7B-v1.0,Q4_0_4_4,10.732,6.072,9.315,1.635,llama_cpp | |
| SOLAR-10.7B-Instruct-v1.0,Q4_0_4_4,10.732,6.072,9.332,1.635,llama_cpp | |
| Yarn-Solar-10b-64k,Q4_0_4_4,10.732,6.072,9.352,1.638,llama_cpp | |
| aya-expanse-8b,Q4_K_M,9.077,5.906,4.406,1.911,llama_cpp | |
| aya-23-8B,Q4_K_M,9.077,5.906,4.428,1.914,llama_cpp | |
| aya-expanse-8b,Q4_0_4_4,9.077,5.647,14.074,2.05,llama_cpp | |
| aya-23-8B,Q4_0_4_4,9.077,5.647,14.113,2.051,llama_cpp | |
| Yi-1.5-9B,Q4_K_M,8.829,5.327,3.681,1.85,llama_cpp | |
| Yi-1.5-9B,Q4_0_4_4,8.829,5.035,11.33,2.0,llama_cpp | |
| Hermes-3-Llama-3.1-8B,Q4_K_M,8.03,4.913,4.375,2.078,llama_cpp | |
| Llama-3.1-8B,Q4_K_M,8.03,4.913,4.403,2.086,llama_cpp | |
| internlm2_5-7b-chat,Q4_K_M,7.738,4.711,4.4,2.133,llama_cpp | |
| Qwen2.5-7B,Q4_K_M,7.616,4.677,4.769,2.201,llama_cpp | |
| dolphin-2.9.2-qwen2-7b,Q4_K_M,7.616,4.677,4.759,2.204,llama_cpp | |
| Llama-3.1-8B,Q4_0_4_4,8.03,4.653,13.99,2.245,llama_cpp | |
| Hermes-3-Llama-3.1-8B,Q4_0_4_4,8.03,4.653,14.006,2.245,llama_cpp | |
| internlm2_5-7b-chat,Q4_0_4_4,7.738,4.451,14.036,2.31,llama_cpp | |
| mpt-7b-instruct,Q4_K_M,6.856,4.442,4.162,2.213,llama_cpp | |
| Qwen2.5-7B,Q4_0_4_4,7.616,4.425,15.563,2.386,llama_cpp | |
| dolphin-2.9.2-qwen2-7b,Q4_0_4_4,7.616,4.425,15.58,2.387,llama_cpp | |
| dolphin-2.9.3-mistral-7B-32k,Q4_K_M,7.248,4.372,4.387,2.227,llama_cpp | |
| Mistral-7B-Instruct-v0.3,Q4_K_M,7.248,4.372,4.462,2.241,llama_cpp | |
| Starling-LM-7B-beta,Q4_K_M,7.242,4.368,4.406,2.234,llama_cpp | |
| Yarn-Mistral-7b-128k,Q4_K_M,7.242,4.368,4.434,2.245,llama_cpp | |
| OLMoE-1B-7B-0924,Q4_K_M,6.919,4.212,26.902,12.119,llama_cpp | |
| OLMo-7B-0724-hf,Q4_K_M,6.888,4.183,4.706,2.339,llama_cpp | |
| dolphin-2.9.3-mistral-7B-32k,Q4_0_4_4,7.248,4.113,14.053,2.427,llama_cpp | |
| Mistral-7B-Instruct-v0.3,Q4_0_4_4,7.248,4.113,14.177,2.43,llama_cpp | |
| Starling-LM-7B-beta,Q4_0_4_4,7.242,4.108,14.068,2.427,llama_cpp | |
| Yarn-Mistral-7b-128k,Q4_0_4_4,7.242,4.108,14.139,2.436,llama_cpp | |
| Amber,Q4_K_M,6.738,4.08,4.594,2.351,llama_cpp | |
| Phi-3.5-mini-instruct,Q8_0,3.821,4.06,7.951,2.423,llama_cpp | |
| Phi-3-mini-128k-instruct,Q8_0,3.821,4.06,7.947,2.426,llama_cpp | |
| mpt-7b-instruct,Q4_0_4_4,6.856,3.964,14.569,2.533,llama_cpp | |
| OLMoE-1B-7B-0924,Q4_0_4_4,6.919,3.926,50.413,12.989,llama_cpp | |
| Amber,Q4_0_4_4,6.738,3.825,14.442,2.57,llama_cpp | |
| Yi-1.5-6B,Q4_K_M,6.061,3.672,5.58,2.72,llama_cpp | |
| Qwen2.5-3B,Q8_0,3.397,3.61,10.473,2.939,llama_cpp | |
| Yi-1.5-6B,Q4_0_4_4,6.061,3.478,17.017,2.945,llama_cpp | |
| dolphin-2.9.4-gemma2-2b,Q8_0,3.204,3.405,13.966,3.381,llama_cpp | |
| gemma-2-2b,Q8_0,3.204,3.405,13.996,3.385,llama_cpp | |
| stable-code-instruct-3b,Q8_0,2.795,2.971,10.668,3.316,llama_cpp | |
| Phi-3.5-mini-instruct,Q4_K_M,3.821,2.393,7.502,3.936,llama_cpp | |
| Phi-3-mini-128k-instruct,Q4_K_M,3.821,2.393,7.519,3.938,llama_cpp | |
| Llama-3.2-3B,Q4_K_M,3.607,2.335,10.691,4.674,llama_cpp | |
| Llama-3.2-3B,Q4_0_4_4,3.607,2.233,31.72,5.025,llama_cpp | |
| gemma-2-2b,Q4_K_M,3.204,2.186,14.202,5.253,llama_cpp | |
| dolphin-2.9.4-gemma2-2b,Q4_K_M,3.204,2.186,14.218,5.253,llama_cpp | |
| Qwen2.5-3B,Q4_K_M,3.397,2.179,10.638,4.808,llama_cpp | |
| Phi-3.5-mini-instruct,Q4_0_4_4,3.821,2.175,23.369,4.428,llama_cpp | |
| Phi-3-mini-128k-instruct,Q4_0_4_4,3.821,2.175,23.461,4.436,llama_cpp | |
| gemma-2-2b,Q4_0_4_4,3.204,2.107,40.616,5.552,llama_cpp | |
| dolphin-2.9.4-gemma2-2b,Q4_0_4_4,3.204,2.107,40.977,5.58,llama_cpp | |
| Qwen2.5-3B,Q4_0_4_4,3.397,2.072,32.434,5.239,llama_cpp | |
| internlm2_5-1_8b-chat,Q8_0,1.889,2.007,19.329,5.279,llama_cpp | |
| SmolLM2-1.7B-Instruct,Q8_0,1.812,1.926,17.524,5.177,llama_cpp | |
| Qwen2.5-1.5B,Q8_0,1.777,1.889,21.927,5.793,llama_cpp | |
| stable-code-instruct-3b,Q4_K_M,2.795,1.707,10.803,5.564,llama_cpp | |
| stable-code-instruct-3b,Q4_0_4_4,2.795,1.607,28.926,5.957,llama_cpp | |
| Yi-Coder-1.5B,Q8_0,1.476,1.569,23.894,6.596,llama_cpp | |
| OLMo-1B-0724-hf,Q8_0,1.28,1.36,27.787,7.591,llama_cpp | |
| Qwen2.5-1.5B,Q4_K_M,1.777,1.172,22.326,9.56,llama_cpp | |
| internlm2_5-1_8b-chat,Q4_K_M,1.889,1.17,19.453,8.56,llama_cpp | |
| TinyLlama-1.1B-Chat-v1.0,Q8_0,1.1,1.169,28.472,8.637,llama_cpp | |
| TinyLlama_v1.1,Q8_0,1.1,1.169,28.538,8.652,llama_cpp | |
| SmolLM2-1.7B-Instruct,Q4_K_M,1.812,1.136,17.72,8.497,llama_cpp | |
| Qwen2.5-1.5B,Q4_0_4_4,1.777,1.12,65.915,10.128,llama_cpp | |
| internlm2_5-1_8b-chat,Q4_0_4_4,1.889,1.112,57.736,9.243,llama_cpp | |
| SmolLM2-1.7B-Instruct,Q4_0_4_4,1.812,1.072,50.27,9.239,llama_cpp | |
| Llama-3.2-1B,Q4_K_M,1.498,1.015,30.451,11.51,llama_cpp | |
| Llama-3.2-1B,Q4_0_4_4,1.498,0.979,86.772,12.364,llama_cpp | |
| Yi-Coder-1.5B,Q4_K_M,1.476,0.962,23.267,10.03,llama_cpp | |
| Yi-Coder-1.5B,Q4_0_4_4,1.476,0.865,67.713,11.422,llama_cpp | |
| OLMo-1B-0724-hf,Q4_K_M,1.28,0.79,28.276,12.321,llama_cpp | |
| OLMo-1B-0724-hf,Q4_0_4_4,1.28,0.746,84.882,13.339,llama_cpp | |
| Qwen2.5-0.5B,Q8_0,0.63,0.67,75.456,18.06,llama_cpp | |
| TinyLlama-1.1B-Chat-v1.0,Q4_K_M,1.1,0.667,29.44,14.305,llama_cpp | |
| TinyLlama_v1.1,Q4_K_M,1.1,0.667,29.397,14.346,llama_cpp | |
| TinyLlama-1.1B-Chat-v1.0,Q4_0_4_4,1.1,0.636,77.823,15.509,llama_cpp | |
| TinyLlama_v1.1,Q4_0_4_4,1.1,0.636,77.943,15.543,llama_cpp | |
| Qwen2.5-0.5B,Q4_K_M,0.63,0.537,52.916,22.324,llama_cpp | |
| Qwen2.5-0.5B,Q4_0_4_4,0.63,0.491,189.874,26.738,llama_cpp | |
| gpt2-medium,Q8_0,0.406,0.436,83.423,23.016,llama_cpp | |
| SmolLM2-360M-Instruct,Q8_0,0.409,0.435,79.518,22.857,llama_cpp | |
| SmolLM2-360M-Instruct,Q4_K_M,0.409,0.319,55.774,30.718,llama_cpp | |
| SmolLM2-360M-Instruct,Q4_0_4_4,0.409,0.277,173.275,37.176,llama_cpp | |
| gpt2-medium,Q4_K_M,0.406,0.269,73.615,33.913,llama_cpp | |
| gpt2-medium,Q4_0_4_4,0.406,0.247,178.73,37.89,llama_cpp | |
| gpt2,Q8_0,0.163,0.176,302.932,68.191,llama_cpp | |
| SmolLM2-135M-Instruct,Q8_0,0.163,0.173,212.146,57.992,llama_cpp | |
| SmolLM2-135M-Instruct,Q4_K_M,0.163,0.134,153.439,73.272,llama_cpp | |
| SmolLM2-135M-Instruct,Q4_0_4_4,0.163,0.12,381.667,86.735,llama_cpp | |
| gpt2,Q4_K_M,0.163,0.111,269.906,92.707,llama_cpp | |
| gpt2,Q4_0_4_4,0.163,0.105,582.32,101.509,llama_cpp | |