Added files

README.md

@@ -0,0 +1,34 @@
+---
+language: he
+tags:
+- roberta
+- language model
+datasets:
+- oscar
+---
+# TavBERT base model
+A Hebrew BERT-style masked language model operating over characters, pre-trained by masking spans of characters, similarly to SpanBERT (Joshi et al., 2020).
+
+### How to use
+
+```python
+import numpy as np
+import torch
+from transformers import AutoModelForMaskedLM, AutoTokenizer
+
+model = AutoModelForMaskedLM.from_pretrained("tau/tavbert-he")
+tokenizer = AutoTokenizer.from_pretrained("tau/tavbert-he")
+
+def mask_sentence(sent, span_len=5):
+    start_pos = np.random.randint(0, len(sent) - span_len)
+    masked_sent = sent[:start_pos] + '[MASK]' * span_len + sent[start_pos + span_len:]
+    print("Masked sentence:", masked_sent)
+    output = model(**tokenizer.encode_plus(masked_sent, 
+                                           return_tensors='pt'))['logits'][0][1:-1]
+    preds = [int(x) for x in torch.argmax(torch.softmax(output, axis=1), axis=1)[start_pos:start_pos + span_len]]
+    pred_sent = sent[:start_pos] + ''.join(tokenizer.convert_ids_to_tokens(preds)) + sent[start_pos + span_len:]
+    print("Model's prediction:", pred_sent)
+```
+## Training data
+OSCAR (Ortiz, 2019) Hebrew section (10 GB text, 20 million sentences).
+

config.json

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "RobertaForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 2050,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.6.0.dev0",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 345
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bc4df9b73af0a1806b249f634eb389582f064ae20eba1dadf69f8e2bb2fdc20
+size 350059547

special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

tokenizer.json

@@ -0,0 +1 @@
+{"version":"1.0","truncation":null,"padding":null,"added_tokens":[],"normalizer":null,"pre_tokenizer":{"type":"Split","pattern":{"String":""},"behavior":"Isolated","invert":false},"post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}}],"pair":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}},{"Sequence":{"id":"B","type_id":1}},{"SpecialToken":{"id":"[SEP]","type_id":1}}],"special_tokens":{"[CLS]":{"id":"[CLS]","ids":[0],"tokens":["[CLS]"]},"[SEP]":{"id":"[SEP]","ids":[2],"tokens":["[SEP]"]}}},"decoder":null,"model":{"type":"WordLevel","vocab":{"[CLS]":0,"[PAD]":1,"[SEP]":2,"[UNK]":3,"unused0":4,"unused1":5,"unused2":6,"unused3":7,"unused4":8,"unused5":9,"unused6":10,"unused7":11,"unused8":12,"unused9":13,"unused10":14,"unused11":15,"unused12":16,"unused13":17,"unused14":18,"unused15":19,"unused16":20,"unused17":21,"unused18":22,"unused19":23,"unused20":24,"unused21":25,"unused22":26,"unused23":27,"unused24":28,"unused25":29,"unused26":30,"unused27":31,"unused28":32,"unused29":33,"unused30":34,"unused31":35,"unused32":36,"unused33":37,"unused34":38,"unused35":39,"unused36":40,"unused37":41,"unused38":42,"unused39":43,"unused40":44,"unused41":45,"unused42":46,"unused43":47,"unused44":48,"unused45":49,"unused46":50,"unused47":51,"unused48":52,"unused49":53,"unused50":54,"unused51":55,"unused52":56,"unused53":57,"unused54":58,"unused55":59,"unused56":60,"unused57":61,"unused58":62,"unused59":63,"unused60":64,"unused61":65,"unused62":66,"unused63":67,"unused64":68,"unused65":69,"unused66":70,"unused67":71,"unused68":72,"unused69":73,"unused70":74,"unused71":75,"unused72":76,"unused73":77,"unused74":78,"unused75":79,"unused76":80,"unused77":81,"unused78":82,"unused79":83,"unused80":84,"unused81":85,"unused82":86,"unused83":87,"unused84":88,"unused85":89,"unused86":90,"unused87":91,"unused88":92,"unused89":93,"unused90":94,"unused91":95,"unused92":96,"unused93":97,"unused94":98,"unused95":99,"unused96":100,"unused97":101,"unused98":102,"unused99":103," ":104,"!":105,"\"":106,"#":107,"$":108,"%":109,"&":110,"'":111,"(":112,")":113,"*":114,"+":115,",":116,"-":117,".":118,"/":119,"0":120,"1":121,"2":122,"3":123,"4":124,"5":125,"6":126,"7":127,"8":128,"9":129,":":130,";":131,"<":132,"=":133,">":134,"?":135,"@":136,"A":137,"B":138,"C":139,"D":140,"E":141,"F":142,"G":143,"H":144,"I":145,"J":146,"K":147,"L":148,"M":149,"N":150,"O":151,"P":152,"Q":153,"R":154,"S":155,"T":156,"U":157,"V":158,"W":159,"X":160,"Y":161,"Z":162,"[":163,"\\":164,"]":165,"^":166,"_":167,"`":168,"a":169,"b":170,"c":171,"d":172,"e":173,"f":174,"g":175,"h":176,"i":177,"j":178,"k":179,"l":180,"m":181,"n":182,"o":183,"p":184,"q":185,"r":186,"s":187,"t":188,"u":189,"v":190,"w":191,"x":192,"y":193,"z":194,"{":195,"|":196,"}":197,"¢":198,"¨":199,"©":200,"°":201,"³":202,"´":203,"·":204,"»":205,"×":206,"é":207,"А":208,"Б":209,"В":210,"Г":211,"Д":212,"И":213,"К":214,"М":215,"Н":216,"О":217,"П":218,"Р":219,"С":220,"Т":221,"Х":222,"Ш":223,"Э":224,"Я":225,"а":226,"б":227,"в":228,"г":229,"д":230,"е":231,"ж":232,"з":233,"и":234,"й":235,"к":236,"л":237,"м":238,"н":239,"о":240,"п":241,"р":242,"с":243,"т":244,"у":245,"ф":246,"х":247,"ц":248,"ч":249,"ш":250,"щ":251,"ы":252,"ь":253,"э":254,"ю":255,"я":256,"֑":257,"֔":258,"֖":259,"֗":260,"֙":261,"֛":262,"֣":263,"֤":264,"֥":265,"֨":266,"ְ":267,"ֱ":268,"ֲ":269,"ֳ":270,"ִ":271,"ֵ":272,"ֶ":273,"ַ":274,"ָ":275,"ֹ":276,"ֻ":277,"ּ":278,"ֽ":279,"־":280,"ׁ":281,"ׂ":282,"׃":283,"א":284,"ב":285,"ג":286,"ד":287,"ה":288,"ו":289,"ז":290,"ח":291,"ט":292,"י":293,"ך":294,"כ":295,"ל":296,"ם":297,"מ":298,"ן":299,"נ":300,"ס":301,"ע":302,"ף":303,"פ":304,"ץ":305,"צ":306,"ק":307,"ר":308,"ש":309,"ת":310,"׳":311,"״":312,"ا":313,"ب":314,"ة":315,"ت":316,"ح":317,"د":318,"ر":319,"س":320,"ع":321,"ف":322,"ق":323,"ك":324,"ل":325,"م":326,"ن":327,"و":328,"ي":329,"–":330,"—":331,"‘":332,"’":333,"“":334,"”":335,"•":336,"…":337,"₪":338,"€":339,"™":340,"←":341,"■":342,"�":343,"[MASK]":344},"unk_token":"[UNK]"}}

tokenizer_config.json

@@ -0,0 +1 @@
+{"special_tokens_map_file": "./special_tokens_map.json", "name_or_path": ".", "tokenizer_class": "PreTrainedTokenizerFast"}

vocab.txt

@@ -0,0 +1,345 @@
+[CLS]
+[PAD]
+[SEP]
+[UNK]
+unused0
+unused1
+unused2
+unused3
+unused4
+unused5
+unused6
+unused7
+unused8
+unused9
+unused10
+unused11
+unused12
+unused13
+unused14
+unused15
+unused16
+unused17
+unused18
+unused19
+unused20
+unused21
+unused22
+unused23
+unused24
+unused25
+unused26
+unused27
+unused28
+unused29
+unused30
+unused31
+unused32
+unused33
+unused34
+unused35
+unused36
+unused37
+unused38
+unused39
+unused40
+unused41
+unused42
+unused43
+unused44
+unused45
+unused46
+unused47
+unused48
+unused49
+unused50
+unused51
+unused52
+unused53
+unused54
+unused55
+unused56
+unused57
+unused58
+unused59
+unused60
+unused61
+unused62
+unused63
+unused64
+unused65
+unused66
+unused67
+unused68
+unused69
+unused70
+unused71
+unused72
+unused73
+unused74
+unused75
+unused76
+unused77
+unused78
+unused79
+unused80
+unused81
+unused82
+unused83
+unused84
+unused85
+unused86
+unused87
+unused88
+unused89
+unused90
+unused91
+unused92
+unused93
+unused94
+unused95
+unused96
+unused97
+unused98
+unused99
+ 
+!
+"
+#
+$
+%
+&
+'
+(
+)
+*
++
+,
+-
+.
+/
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+:
+;
+<
+=
+>
+?
+@
+A
+B
+C
+D
+E
+F
+G
+H
+I
+J
+K
+L
+M
+N
+O
+P
+Q
+R
+S
+T
+U
+V
+W
+X
+Y
+Z
+[
+\
+]
+^
+_
+`
+a
+b
+c
+d
+e
+f
+g
+h
+i
+j
+k
+l
+m
+n
+o
+p
+q
+r
+s
+t
+u
+v
+w
+x
+y
+z
+{
+|
+}
+¢
+¨
+©
+°
+³
+´
+·
+»
+×
+é
+А
+Б
+В
+Г
+Д
+И
+К
+М
+Н
+О
+П
+Р
+С
+Т
+Х
+Ш
+Э
+Я
+а
+б
+в
+г
+д
+е
+ж
+з
+и
+й
+к
+л
+м
+н
+о
+п
+р
+с
+т
+у
+ф
+х
+ц
+ч
+ш
+щ
+ы
+ь
+э
+ю
+я
+֑
+֔
+֖
+֗
+֙
+֛
+֣
+֤
+֥
+֨
+ְ
+ֱ
+ֲ
+ֳ
+ִ
+ֵ
+ֶ
+ַ
+ָ
+ֹ
+ֻ
+ּ
+ֽ
+־
+ׁ
+ׂ
+׃
+א
+ב
+ג
+ד
+ה
+ו
+ז
+ח
+ט
+י
+ך
+כ
+ל
+ם
+מ
+ן
+נ
+ס
+ע
+ף
+פ
+ץ
+צ
+ק
+ר
+ש
+ת
+׳
+״
+ا
+ب
+ة
+ت
+ح
+د
+ر
+س
+ع
+ف
+ق
+ك
+ل
+م
+ن
+و
+ي
+–
+—
+‘
+’
+“
+”
+•
+…
+₪
+€
+™
+←
+■
+�
+[MASK]