Training in progress, step 500

config.json

@@ -0,0 +1,42 @@
+{
+  "architectures": [
+    "GPTNeoCodebookModel"
+  ],
+  "codebook_at": [
+    "attn",
+    "mlp"
+  ],
+  "codebook_kwargs": {},
+  "codebook_type": [
+    "vanilla",
+    "vanilla"
+  ],
+  "k_codebook": [
+    100,
+    100
+  ],
+  "kmeans_init": false,
+  "kmeans_init_examples": 1000,
+  "kmeans_kwargs": {
+    "batch_size": 24576,
+    "n_init": "auto"
+  },
+  "kmeans_path": "/.cache/cb_volume/huggingface/kmeans_embeddings.pt",
+  "layers_to_snap": [
+    0
+  ],
+  "loss": "aeloss",
+  "model_type": "codebook",
+  "num_codebooks": [
+    1,
+    1
+  ],
+  "num_codes": [
+    10000,
+    10000
+  ],
+  "replace_codes": false,
+  "similarity_metric": "inner_product",
+  "torch_dtype": "float32",
+  "transformers_version": "4.35.2"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20b3abd3c7ffdf3e6e9d2809b8b7ced8c5df3a75698ab8c4124521377ca1c937
+size 346561224

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,22 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 2048,
+  "pad_token": null,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5a36ce1d44a77d30ab73815fb300281c3d0a4713a6b9a7668b7bb24ea039c53
+size 4856