Training in progress, step 500

config.json

@@ -6,19 +6,19 @@
   "classifier_dropout": null,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
-  "hidden_size": 384,
+  "hidden_size": 768,
   "initializer_range": 0.02,
   "intermediate_size": 1024,
   "layer_norm_eps": 1e-12,
-  "max_position_embeddings": 256,
+  "max_position_embeddings": 512,
   "model_type": "bert",
-  "num_attention_heads": 6,
-  "num_hidden_layers": 6,
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
   "pad_token_id": 0,
   "position_embedding_type": "absolute",
   "torch_dtype": "float32",
   "transformers_version": "4.36.2",
   "type_vocab_size": 2,
   "use_cache": true,
-  "vocab_size": 1000
+  "vocab_size": 50000
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:483a2310bbc51a927e276432e2094dfb9d5300383f4a32df0e955e3b7cb5fc93
-size 35683576
+oid sha256:b921b762510088fccaf745e3cd87e812de89f3ddb263a51360e4fab464e69816
+size 346901984

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d468e43181cfd22853de5cc18c8984e84e9bcbe025b5c0c766db8af116bb0d12
+oid sha256:5f1a3e1473c6b22ad211316435299cd36b19815f3050f35b4cf991733a75124d
 size 4728