File size: 668 Bytes
26ccd30 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 |
{
"architectures": [
"GPTNeoCodebookModel"
],
"codebook_at": [
"preproj_attention"
],
"codebook_kwargs": {},
"codebook_type": [
"compositional"
],
"k_codebook": [
8
],
"kmeans_init": false,
"kmeans_init_examples": 1000,
"kmeans_kwargs": {
"batch_size": 24576,
"n_init": "auto"
},
"kmeans_path": "/.cache/cb_volume/huggingface/kmeans_embeddings.pt",
"layers_to_snap": [
0
],
"loss": "aeloss",
"model_type": "codebook",
"num_codebooks": [
16
],
"num_codes": 25000,
"replace_codes": false,
"similarity_metric": "inner_product",
"torch_dtype": "float32",
"transformers_version": "4.29.2"
}
|