Training in progress, step 78

config.json

@@ -1 +1 @@
-{"d_model": 1024, "n_layer": 48, "vocab_size": 50277, "ssm_cfg": {}, "rms_norm": true, "residual_in_fp32": true, "fused_add_norm": true, "pad_vocab_size_multiple": 8, "tie_embeddings": false}
+{"d_model": 768, "n_layer": 24, "vocab_size": 50277, "ssm_cfg": {}, "rms_norm": true, "residual_in_fp32": true, "fused_add_norm": true, "pad_vocab_size_multiple": 8, "tie_embeddings": false}

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:546709915123a8267be7406a9e837a04fb80fd2c7b4f11e23e271d71bfc4bd1d
-size 1486294682
+oid sha256:b8050f241576a0bd8651af7e760ad0d844b0978f484c180ee9615cf25311f334
+size 516640282

special_tokens_map.json

@@ -1,25 +1,7 @@
 {
- "bos_token": {
- "content": "<s>",
- "lstrip": false,
- "normalized": true,
- "rstrip": false,
- "single_word": false
- },
- "cls_token": {
- "content": "<s>",
- "lstrip": false,
- "normalized": true,
- "rstrip": false,
- "single_word": false
- },
- "eos_token": {
- "content": "</s>",
- "lstrip": false,
- "normalized": true,
- "rstrip": false,
- "single_word": false
- },
+ "bos_token": "<s>",
+ "cls_token": "<s>",
+ "eos_token": "</s>",
  "mask_token": {
  "content": "<mask>",
  "lstrip": true,
@@ -28,18 +10,6 @@
  "single_word": false
  },
  "pad_token": "</s>",
- "sep_token": {
- "content": "</s>",
- "lstrip": false,
- "normalized": true,
- "rstrip": false,
- "single_word": false
- },
- "unk_token": {
- "content": "<unk>",
- "lstrip": false,
- "normalized": true,
- "rstrip": false,
- "single_word": false
- }
+ "sep_token": "</s>",
+ "unk_token": "<unk>"
 }

tokenizer.json

@@ -1,6 +1,11 @@
 {
  "version": "1.0",
- "truncation": null,
+ "truncation": {
+ "direction": "Right",
+ "max_length": 512,
+ "strategy": "LongestFirst",
+ "stride": 0
+ },
  "padding": null,
  "added_tokens": [
  {

tokenizer_config.json

@@ -48,7 +48,7 @@
  "eos_token": "</s>",
  "errors": "replace",
  "mask_token": "<mask>",
- "model_max_length": 1000000000000000019884624838656,
+ "model_max_length": 512,
  "pad_token": "</s>",
  "sep_token": "</s>",
  "tokenizer_class": "RobertaTokenizer",

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dedd469097ed7f3be3cae57b6755942647bb27d8d1fd07f924c46c60b6c74431
+oid sha256:dff7dc93a1f6d7a3d32122340a3044405f24749da3781fcf65a4e4deb4c3d14d
 size 4920