Add model

README.md

@@ -1,3 +1,46 @@
 ---
+language: en
+tags:
+- roberta
 license: mit
 ---
+
+# RoBERTa base model fine-tuned on pronoun fill masking
+
+This is RoBERTa base fine-tuned for fill masking of just pronouns.
+The model's purpose is to post process machine translated text where sentence
+level translation may not have enough context to correctly deduce the correct
+pronoun to use.
+
+This model was trained on 10B tokens of literature (private light novel and book dataset as well as books1 and 20\% of books3 from The Pile).
+
+This model achieves an 88\% top1 accuracy, evaluated with a sliding window of 512 tokens (84\% without a sliding window).
+
+### How to use
+
+Mask *all* pronoun tokens. The use the fill mask pipeline to get the
+model's predictions.
+
+```python
+PRONOUN_TOKENS = {
+    'I',                           'ĠI',
+    'you',    'You',    'Ġyou',    'ĠYou',
+    'he',     'He',     'Ġhe',     'ĠHe',
+    'she',    'She',    'Ġshe',    'ĠShe',
+    'it',     'It',     'Ġit',     'ĠIt',
+    'we',     'We',     'Ġwe',     'ĠWe',
+    'they',   'They',   'Ġthey',   'ĠThey',
+    'my',     'My',     'Ġmy',     'ĠMy',
+    'your',   'Your',   'Ġyour',   'ĠYour',
+    'his',    'His',    'Ġhis',    'ĠHis',
+    'her',    'Her',    'Ġher',    'ĠHer',
+    'its',    'Its',    'Ġits',    'ĠIts',
+    'our',    'Our',    'Ġour',    'ĠOur',
+    'their',  'Their',  'Ġtheir',  'ĠTheir',
+    'mine',   'Mine',   'Ġmine',   'ĠMine',
+    'yours',  'Yours',  'Ġyours',  'ĠYours',
+    'hers',   'Hers',   'Ġhers',   'ĠHers',
+    'ours',   'Ours',   'Ġours',   'ĠOurs',
+    'theirs', 'Theirs', 'Ġtheirs', 'ĠTheirs',
+}
+```

config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "roberta-base",
+  "architectures": [
+    "RobertaForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.22.0.dev0",
+  "type_vocab_size": 1,
+  "use_cache": false,
+  "vocab_size": 50265
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:735e1a4d31dcc652dcf1619c1a589aead87e6b3cd73be4ed1df2b15b9ee06a13
+size 498863417

special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "errors": "replace",
+  "mask_token": "<mask>",
+  "model_max_length": 512,
+  "name_or_path": "roberta-base",
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "special_tokens_map_file": null,
+  "tokenizer_class": "RobertaTokenizer",
+  "trim_offsets": true,
+  "unk_token": "<unk>"
+}