KoichiYasuoka commited on
Commit
152b813
1 Parent(s): d8e3170

initial release

Browse files
README.md ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language:
3
+ - "ja"
4
+ tags:
5
+ - "japanese"
6
+ - "token-classification"
7
+ - "pos"
8
+ base_model: HPLT/hplt_bert_base_ja
9
+ datasets:
10
+ - "universal_dependencies"
11
+ license: "apache-2.0"
12
+ pipeline_tag: "token-classification"
13
+ widget:
14
+ - text: "国境の長いトンネルを抜けると雪国であった。"
15
+ ---
16
+
17
+ # ltgbert-base-japanese-upos
18
+
19
+ ## Model Description
20
+
21
+ This is a LTG-BERT model for POS-tagging, derived from [hplt_bert_base_ja](https://huggingface.co/HPLT/hplt_bert_base_ja). Every short-unit-word is tagged by [UPOS](https://universaldependencies.org/u/pos/) (Universal Part-Of-Speech) and [FEATS](https://universaldependencies.org/u/feat/).
22
+
23
+ ## How to Use
24
+
25
+ ```py
26
+ from transformers import pipeline
27
+ nlp=pipeline("upos","KoichiYasuoka/ltgbert-base-japanese-upos",trust_remote_code=True,aggregation_strategy="simple")
28
+ print(nlp("国境の長いトンネルを抜けると雪国であった。"))
29
+ ```
30
+
config.json ADDED
@@ -0,0 +1,159 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "LtgbertForTokenClassification"
4
+ ],
5
+ "attention_probs_dropout_prob": 0.1,
6
+ "auto_map": {
7
+ "AutoConfig": "configuration_ltgbert.LtgbertConfig",
8
+ "AutoModel": "modeling_ltgbert.LtgbertModel",
9
+ "AutoModelForMaskedLM": "modeling_ltgbert.LtgbertForMaskedLM",
10
+ "AutoModelForMultipleChoice": "modeling_ltgbert.LtgbertForMultipleChoice",
11
+ "AutoModelForQuestionAnswering": "modeling_ltgbert.LtgbertForQuestionAnswering",
12
+ "AutoModelForSequenceClassification": "modeling_ltgbert.LtgbertForSequenceClassification",
13
+ "AutoModelForTokenClassification": "modeling_ltgbert.LtgbertForTokenClassification"
14
+ },
15
+ "classifier_dropout": 0.1,
16
+ "custom_pipelines": {
17
+ "upos": {
18
+ "impl": "upos.BellmanFordTokenClassificationPipeline",
19
+ "pt": "AutoModelForTokenClassification"
20
+ }
21
+ },
22
+ "hidden_dropout_prob": 0.1,
23
+ "hidden_size": 768,
24
+ "id2label": {
25
+ "0": "ADJ",
26
+ "1": "B-ADJ",
27
+ "2": "I-ADJ",
28
+ "3": "ADJ|Polarity=Neg",
29
+ "4": "B-ADJ|Polarity=Neg",
30
+ "5": "I-ADJ|Polarity=Neg",
31
+ "6": "ADP",
32
+ "7": "B-ADP",
33
+ "8": "I-ADP",
34
+ "9": "ADV",
35
+ "10": "B-ADV",
36
+ "11": "I-ADV",
37
+ "12": "AUX",
38
+ "13": "B-AUX",
39
+ "14": "I-AUX",
40
+ "15": "AUX|Polarity=Neg",
41
+ "16": "B-AUX|Polarity=Neg",
42
+ "17": "I-AUX|Polarity=Neg",
43
+ "18": "CCONJ",
44
+ "19": "B-CCONJ",
45
+ "20": "I-CCONJ",
46
+ "21": "DET",
47
+ "22": "B-DET",
48
+ "23": "I-DET",
49
+ "24": "INTJ",
50
+ "25": "B-INTJ",
51
+ "26": "I-INTJ",
52
+ "27": "NOUN",
53
+ "28": "B-NOUN",
54
+ "29": "I-NOUN",
55
+ "30": "NOUN|Polarity=Neg",
56
+ "31": "B-NOUN|Polarity=Neg",
57
+ "32": "I-NOUN|Polarity=Neg",
58
+ "33": "NUM",
59
+ "34": "B-NUM",
60
+ "35": "I-NUM",
61
+ "36": "PART",
62
+ "37": "B-PART",
63
+ "38": "I-PART",
64
+ "39": "PRON",
65
+ "40": "B-PRON",
66
+ "41": "I-PRON",
67
+ "42": "PROPN",
68
+ "43": "B-PROPN",
69
+ "44": "I-PROPN",
70
+ "45": "PUNCT",
71
+ "46": "B-PUNCT",
72
+ "47": "I-PUNCT",
73
+ "48": "SCONJ",
74
+ "49": "B-SCONJ",
75
+ "50": "I-SCONJ",
76
+ "51": "SYM",
77
+ "52": "B-SYM",
78
+ "53": "I-SYM",
79
+ "54": "VERB",
80
+ "55": "B-VERB",
81
+ "56": "I-VERB",
82
+ "57": "X",
83
+ "58": "B-X",
84
+ "59": "I-X"
85
+ },
86
+ "intermediate_size": 2560,
87
+ "label2id": {
88
+ "ADJ": 0,
89
+ "ADJ|Polarity=Neg": 3,
90
+ "ADP": 6,
91
+ "ADV": 9,
92
+ "AUX": 12,
93
+ "AUX|Polarity=Neg": 15,
94
+ "B-ADJ": 1,
95
+ "B-ADJ|Polarity=Neg": 4,
96
+ "B-ADP": 7,
97
+ "B-ADV": 10,
98
+ "B-AUX": 13,
99
+ "B-AUX|Polarity=Neg": 16,
100
+ "B-CCONJ": 19,
101
+ "B-DET": 22,
102
+ "B-INTJ": 25,
103
+ "B-NOUN": 28,
104
+ "B-NOUN|Polarity=Neg": 31,
105
+ "B-NUM": 34,
106
+ "B-PART": 37,
107
+ "B-PRON": 40,
108
+ "B-PROPN": 43,
109
+ "B-PUNCT": 46,
110
+ "B-SCONJ": 49,
111
+ "B-SYM": 52,
112
+ "B-VERB": 55,
113
+ "B-X": 58,
114
+ "CCONJ": 18,
115
+ "DET": 21,
116
+ "I-ADJ": 2,
117
+ "I-ADJ|Polarity=Neg": 5,
118
+ "I-ADP": 8,
119
+ "I-ADV": 11,
120
+ "I-AUX": 14,
121
+ "I-AUX|Polarity=Neg": 17,
122
+ "I-CCONJ": 20,
123
+ "I-DET": 23,
124
+ "I-INTJ": 26,
125
+ "I-NOUN": 29,
126
+ "I-NOUN|Polarity=Neg": 32,
127
+ "I-NUM": 35,
128
+ "I-PART": 38,
129
+ "I-PRON": 41,
130
+ "I-PROPN": 44,
131
+ "I-PUNCT": 47,
132
+ "I-SCONJ": 50,
133
+ "I-SYM": 53,
134
+ "I-VERB": 56,
135
+ "I-X": 59,
136
+ "INTJ": 24,
137
+ "NOUN": 27,
138
+ "NOUN|Polarity=Neg": 30,
139
+ "NUM": 33,
140
+ "PART": 36,
141
+ "PRON": 39,
142
+ "PROPN": 42,
143
+ "PUNCT": 45,
144
+ "SCONJ": 48,
145
+ "SYM": 51,
146
+ "VERB": 54,
147
+ "X": 57
148
+ },
149
+ "layer_norm_eps": 1e-07,
150
+ "max_position_embeddings": 512,
151
+ "num_attention_heads": 12,
152
+ "num_hidden_layers": 12,
153
+ "output_all_encoded_layers": true,
154
+ "position_bucket_size": 32,
155
+ "tokenizer_class": "PreTrainedTokenizerFast",
156
+ "torch_dtype": "float32",
157
+ "transformers_version": "4.44.2",
158
+ "vocab_size": 32768
159
+ }
configuration_ltgbert.py ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers.configuration_utils import PretrainedConfig
2
+
3
+
4
+ class LtgbertConfig(PretrainedConfig):
5
+ """Configuration class to store the configuration of a `LtgbertModel`.
6
+ """
7
+ def __init__(
8
+ self,
9
+ vocab_size=32768,
10
+ attention_probs_dropout_prob=0.1,
11
+ hidden_dropout_prob=0.1,
12
+ hidden_size=768,
13
+ intermediate_size=2048,
14
+ max_position_embeddings=512,
15
+ position_bucket_size=32,
16
+ num_attention_heads=12,
17
+ num_hidden_layers=12,
18
+ layer_norm_eps=1.0e-7,
19
+ output_all_encoded_layers=True,
20
+ **kwargs,
21
+ ):
22
+ super().__init__(**kwargs)
23
+
24
+ self.vocab_size = vocab_size
25
+ self.hidden_size = hidden_size
26
+ self.num_hidden_layers = num_hidden_layers
27
+ self.num_attention_heads = num_attention_heads
28
+ self.intermediate_size = intermediate_size
29
+ self.hidden_dropout_prob = hidden_dropout_prob
30
+ self.attention_probs_dropout_prob = attention_probs_dropout_prob
31
+ self.max_position_embeddings = max_position_embeddings
32
+ self.output_all_encoded_layers = output_all_encoded_layers
33
+ self.position_bucket_size = position_bucket_size
34
+ self.layer_norm_eps = layer_norm_eps
maker.py ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #! /usr/bin/python3
2
+ src="hplt_bert_base_ja"
3
+ tgt="KoichiYasuoka/ltgbert-base-japanese-upos"
4
+
5
+ import os
6
+ from transformers import AutoTokenizer,AutoConfig,AutoModelForTokenClassification,DataCollatorForTokenClassification,TrainingArguments,Trainer
7
+ os.system(f"test -d {src} || ( curl -L https://data.hplt-project.org/one/models/encoder/{src}.tar.gz | tar xvzf - )")
8
+ os.system("test -f ja_gsd_modern.conllu || curl -LO https://github.com/KoichiYasuoka/SuPar-UniDic/raw/main/suparunidic/suparmodels/ja_gsd_modern.conllu")
9
+
10
+ class UPOSFileDataset(object):
11
+ def __init__(self,conllu,tokenizer):
12
+ self.conllu=open(conllu,"r",encoding="utf-8")
13
+ self.tokenizer=tokenizer
14
+ self.seeks=[0]
15
+ label=set(["SYM"])
16
+ s=self.conllu.readline()
17
+ while s!="":
18
+ if s=="\n":
19
+ self.seeks.append(self.conllu.tell())
20
+ else:
21
+ w=s.split("\t")
22
+ if len(w)==10:
23
+ if w[0].isdecimal():
24
+ label.add(w[3] if w[5]=="_" else w[3]+"|"+w[5])
25
+ s=self.conllu.readline()
26
+ lid={}
27
+ for i,l in enumerate(sorted(label)):
28
+ lid[l],lid["B-"+l],lid["I-"+l]=i*3,i*3+1,i*3+2
29
+ self.label2id=lid
30
+ def __call__(*args):
31
+ lid={l:i for i,l in enumerate(sorted(set(sum([list(t.label2id) for t in args],[]))))}
32
+ for t in args:
33
+ t.label2id=lid
34
+ return lid
35
+ def __del__(self):
36
+ self.conllu.close()
37
+ __len__=lambda self:len(self.seeks)-1
38
+ def __getitem__(self,i):
39
+ self.conllu.seek(self.seeks[i])
40
+ form,upos=[],[]
41
+ while self.conllu.tell()<self.seeks[i+1]:
42
+ w=self.conllu.readline().split("\t")
43
+ if len(w)==10:
44
+ form.append(w[1])
45
+ if w[0].isdecimal():
46
+ upos.append(w[3] if w[5]=="_" else w[3]+"|"+w[5])
47
+ v=self.tokenizer(form,add_special_tokens=False)
48
+ i,u=[],[]
49
+ for j,(x,y) in enumerate(zip(v["input_ids"],upos)):
50
+ if x!=[]:
51
+ i+=x
52
+ u+=[y] if len(x)==1 else ["B-"+y]+["I-"+y]*(len(x)-1)
53
+ if len(i)<self.tokenizer.model_max_length-3:
54
+ ids=[self.tokenizer.cls_token_id]+i+[self.tokenizer.sep_token_id]
55
+ upos=["SYM"]+u+["SYM"]
56
+ else:
57
+ ids=i[0:self.tokenizer.model_max_length-2]
58
+ upos=u[0:self.tokenizer.model_max_length-2]
59
+ return {"input_ids":ids,"labels":[self.label2id[t] for t in upos]}
60
+
61
+ tkz=AutoTokenizer.from_pretrained(src,model_max_length=512)
62
+ trainDS=UPOSFileDataset("ja_gsd_modern.conllu",tkz)
63
+ lid=trainDS.label2id
64
+ cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True,trust_remote_code=True)
65
+ arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=8,output_dir=tgt,overwrite_output_dir=True,save_total_limit=2,learning_rate=5e-05,warmup_ratio=0.1,save_safetensors=False)
66
+ trn=Trainer(args=arg,data_collator=DataCollatorForTokenClassification(tkz),model=AutoModelForTokenClassification.from_pretrained(src,config=cfg,ignore_mismatched_sizes=True,trust_remote_code=True),train_dataset=trainDS)
67
+ trn.train()
68
+ trn.save_model(tgt)
69
+ tkz.save_pretrained(tgt)
modeling_ltgbert.py ADDED
@@ -0,0 +1,639 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ from typing import List, Optional, Tuple, Union
3
+
4
+ import torch
5
+ import torch.nn as nn
6
+ import torch.nn.functional as F
7
+ from torch.utils import checkpoint
8
+
9
+ from .configuration_ltgbert import LtgbertConfig
10
+ from transformers.modeling_utils import PreTrainedModel
11
+ from transformers.activations import gelu_new
12
+ from transformers.modeling_outputs import (
13
+ MaskedLMOutput,
14
+ MultipleChoiceModelOutput,
15
+ QuestionAnsweringModelOutput,
16
+ SequenceClassifierOutput,
17
+ TokenClassifierOutput,
18
+ BaseModelOutput
19
+ )
20
+ from transformers.pytorch_utils import softmax_backward_data
21
+
22
+
23
+ class Encoder(nn.Module):
24
+ def __init__(self, config, activation_checkpointing=False):
25
+ super().__init__()
26
+ self.layers = nn.ModuleList([EncoderLayer(config) for _ in range(config.num_hidden_layers)])
27
+
28
+ for i, layer in enumerate(self.layers):
29
+ layer.mlp.mlp[1].weight.data *= math.sqrt(1.0 / (2.0 * (1 + i)))
30
+ layer.mlp.mlp[-2].weight.data *= math.sqrt(1.0 / (2.0 * (1 + i)))
31
+
32
+ self.activation_checkpointing = activation_checkpointing
33
+
34
+ def forward(self, hidden_states, attention_mask, relative_embedding):
35
+ hidden_states, attention_probs = [hidden_states], []
36
+
37
+ for layer in self.layers:
38
+ if self.activation_checkpointing:
39
+ hidden_state, attention_p = checkpoint.checkpoint(layer, hidden_states[-1], attention_mask, relative_embedding)
40
+ else:
41
+ hidden_state, attention_p = layer(hidden_states[-1], attention_mask, relative_embedding)
42
+
43
+ hidden_states.append(hidden_state)
44
+ attention_probs.append(attention_p)
45
+
46
+ return hidden_states, attention_probs
47
+
48
+
49
+ class MaskClassifier(nn.Module):
50
+ def __init__(self, config, subword_embedding):
51
+ super().__init__()
52
+ self.nonlinearity = nn.Sequential(
53
+ nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=False),
54
+ nn.Linear(config.hidden_size, config.hidden_size),
55
+ nn.GELU(),
56
+ nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=False),
57
+ nn.Dropout(config.hidden_dropout_prob),
58
+ nn.Linear(subword_embedding.size(1), subword_embedding.size(0))
59
+ )
60
+
61
+ def forward(self, x, masked_lm_labels=None):
62
+ if masked_lm_labels is not None:
63
+ x = torch.index_select(x.flatten(0, 1), 0, torch.nonzero(masked_lm_labels.flatten() != -100).squeeze())
64
+ x = self.nonlinearity(x)
65
+ return x
66
+
67
+
68
+ class EncoderLayer(nn.Module):
69
+ def __init__(self, config):
70
+ super().__init__()
71
+ self.attention = Attention(config)
72
+ self.mlp = FeedForward(config)
73
+
74
+ def forward(self, x, padding_mask, relative_embedding):
75
+ attention_output, attention_probs = self.attention(x, padding_mask, relative_embedding)
76
+ x = x + attention_output
77
+ x = x + self.mlp(x)
78
+ return x, attention_probs
79
+
80
+
81
+ class GeGLU(nn.Module):
82
+ def forward(self, x):
83
+ x, gate = x.chunk(2, dim=-1)
84
+ x = x * gelu_new(gate)
85
+ return x
86
+
87
+
88
+ class FeedForward(nn.Module):
89
+ def __init__(self, config):
90
+ super().__init__()
91
+ self.mlp = nn.Sequential(
92
+ nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps, elementwise_affine=False),
93
+ nn.Linear(config.hidden_size, 2*config.intermediate_size, bias=False),
94
+ GeGLU(),
95
+ nn.LayerNorm(config.intermediate_size, eps=config.layer_norm_eps, elementwise_affine=False),
96
+ nn.Linear(config.intermediate_size, config.hidden_size, bias=False),
97
+ nn.Dropout(config.hidden_dropout_prob)
98
+ )
99
+
100
+ def forward(self, x):
101
+ return self.mlp(x)
102
+
103
+
104
+ class MaskedSoftmax(torch.autograd.Function):
105
+ @staticmethod
106
+ def forward(self, x, mask, dim):
107
+ self.dim = dim
108
+ x.masked_fill_(mask, float('-inf'))
109
+ x = torch.softmax(x, self.dim)
110
+ x.masked_fill_(mask, 0.0)
111
+ self.save_for_backward(x)
112
+ return x
113
+
114
+ @staticmethod
115
+ def backward(self, grad_output):
116
+ output, = self.saved_tensors
117
+ input_grad = softmax_backward_data(self, grad_output, output, self.dim, output)
118
+ return input_grad, None, None
119
+
120
+
121
+ class Attention(nn.Module):
122
+ def __init__(self, config):
123
+ super().__init__()
124
+
125
+ self.config = config
126
+
127
+ if config.hidden_size % config.num_attention_heads != 0:
128
+ raise ValueError(f"The hidden size {config.hidden_size} is not a multiple of the number of attention heads {config.num_attention_heads}")
129
+
130
+ self.hidden_size = config.hidden_size
131
+ self.num_heads = config.num_attention_heads
132
+ self.head_size = config.hidden_size // config.num_attention_heads
133
+
134
+ self.in_proj_qk = nn.Linear(config.hidden_size, 2*config.hidden_size, bias=True)
135
+ self.in_proj_v = nn.Linear(config.hidden_size, config.hidden_size, bias=True)
136
+ self.out_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=True)
137
+
138
+ self.pre_layer_norm = nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=False)
139
+ self.post_layer_norm = nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=True)
140
+
141
+ position_indices = torch.arange(config.max_position_embeddings, dtype=torch.long).unsqueeze(1) \
142
+ - torch.arange(config.max_position_embeddings, dtype=torch.long).unsqueeze(0)
143
+ position_indices = self.make_log_bucket_position(position_indices, config.position_bucket_size, config.max_position_embeddings)
144
+ position_indices = config.position_bucket_size - 1 + position_indices
145
+ self.register_buffer("position_indices", position_indices, persistent=True)
146
+
147
+ self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
148
+ self.scale = 1.0 / math.sqrt(3 * self.head_size)
149
+
150
+ def make_log_bucket_position(self, relative_pos, bucket_size, max_position):
151
+ sign = torch.sign(relative_pos)
152
+ mid = bucket_size // 2
153
+ abs_pos = torch.where((relative_pos < mid) & (relative_pos > -mid), mid - 1, torch.abs(relative_pos).clamp(max=max_position - 1))
154
+ log_pos = torch.ceil(torch.log(abs_pos / mid) / math.log((max_position-1) / mid) * (mid - 1)).int() + mid
155
+ bucket_pos = torch.where(abs_pos <= mid, relative_pos, log_pos * sign).long()
156
+ return bucket_pos
157
+
158
+ def compute_attention_scores(self, hidden_states, relative_embedding):
159
+ key_len, batch_size, _ = hidden_states.size()
160
+ query_len = key_len
161
+
162
+ if self.position_indices.size(0) < query_len:
163
+ position_indices = torch.arange(query_len, dtype=torch.long).unsqueeze(1) \
164
+ - torch.arange(query_len, dtype=torch.long).unsqueeze(0)
165
+ position_indices = self.make_log_bucket_position(position_indices, self.config.position_bucket_size, 512)
166
+ position_indices = self.config.position_bucket_size - 1 + position_indices
167
+ self.position_indices = position_indices.to(hidden_states.device)
168
+
169
+ hidden_states = self.pre_layer_norm(hidden_states)
170
+
171
+ query, key = self.in_proj_qk(hidden_states).chunk(2, dim=2) # shape: [T, B, D]
172
+ value = self.in_proj_v(hidden_states) # shape: [T, B, D]
173
+
174
+ query = query.reshape(query_len, batch_size * self.num_heads, self.head_size).transpose(0, 1)
175
+ key = key.reshape(key_len, batch_size * self.num_heads, self.head_size).transpose(0, 1)
176
+ value = value.view(key_len, batch_size * self.num_heads, self.head_size).transpose(0, 1)
177
+
178
+ attention_scores = torch.bmm(query, key.transpose(1, 2) * self.scale)
179
+
180
+ query_pos, key_pos = self.in_proj_qk(self.dropout(relative_embedding)).chunk(2, dim=-1) # shape: [2T-1, D]
181
+ query_pos = query_pos.view(-1, self.num_heads, self.head_size) # shape: [2T-1, H, D]
182
+ key_pos = key_pos.view(-1, self.num_heads, self.head_size) # shape: [2T-1, H, D]
183
+
184
+ query = query.view(batch_size, self.num_heads, query_len, self.head_size)
185
+ key = key.view(batch_size, self.num_heads, query_len, self.head_size)
186
+
187
+ attention_c_p = torch.einsum("bhqd,khd->bhqk", query, key_pos.squeeze(1) * self.scale)
188
+ attention_p_c = torch.einsum("bhkd,qhd->bhqk", key * self.scale, query_pos.squeeze(1))
189
+
190
+ position_indices = self.position_indices[:query_len, :key_len].expand(batch_size, self.num_heads, -1, -1)
191
+ attention_c_p = attention_c_p.gather(3, position_indices)
192
+ attention_p_c = attention_p_c.gather(2, position_indices)
193
+
194
+ attention_scores = attention_scores.view(batch_size, self.num_heads, query_len, key_len)
195
+ attention_scores.add_(attention_c_p)
196
+ attention_scores.add_(attention_p_c)
197
+
198
+ return attention_scores, value
199
+
200
+ def compute_output(self, attention_probs, value):
201
+ attention_probs = self.dropout(attention_probs)
202
+ context = torch.bmm(attention_probs.flatten(0, 1), value) # shape: [B*H, Q, D]
203
+ context = context.transpose(0, 1).reshape(context.size(1), -1, self.hidden_size) # shape: [Q, B, H*D]
204
+ context = self.out_proj(context)
205
+ context = self.post_layer_norm(context)
206
+ context = self.dropout(context)
207
+ return context
208
+
209
+ def forward(self, hidden_states, attention_mask, relative_embedding):
210
+ attention_scores, value = self.compute_attention_scores(hidden_states, relative_embedding)
211
+ attention_probs = MaskedSoftmax.apply(attention_scores, attention_mask, -1)
212
+ return self.compute_output(attention_probs, value), attention_probs.detach()
213
+
214
+
215
+ class Embedding(nn.Module):
216
+ def __init__(self, config):
217
+ super().__init__()
218
+ self.hidden_size = config.hidden_size
219
+
220
+ self.word_embedding = nn.Embedding(config.vocab_size, config.hidden_size)
221
+ self.word_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps, elementwise_affine=False)
222
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
223
+
224
+ self.relative_embedding = nn.Parameter(torch.empty(2 * config.position_bucket_size - 1, config.hidden_size))
225
+ self.relative_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
226
+
227
+ def forward(self, input_ids):
228
+ word_embedding = self.dropout(self.word_layer_norm(self.word_embedding(input_ids)))
229
+ relative_embeddings = self.relative_layer_norm(self.relative_embedding)
230
+ return word_embedding, relative_embeddings
231
+
232
+
233
+ #
234
+ # HuggingFace wrappers
235
+ #
236
+
237
+ class LtgbertPreTrainedModel(PreTrainedModel):
238
+ config_class = LtgbertConfig
239
+ supports_gradient_checkpointing = True
240
+
241
+ def _set_gradient_checkpointing(self, module, value=False):
242
+ if isinstance(module, Encoder):
243
+ module.activation_checkpointing = value
244
+
245
+ def _init_weights(self, module):
246
+ std = math.sqrt(2.0 / (5.0 * self.hidden_size))
247
+
248
+ if isinstance(module, nn.Linear):
249
+ nn.init.trunc_normal_(module.weight.data, mean=0.0, std=std, a=-2*std, b=2*std)
250
+ if module.bias is not None:
251
+ module.bias.data.zero_()
252
+ elif isinstance(module, nn.Embedding):
253
+ nn.init.trunc_normal_(module.weight.data, mean=0.0, std=std, a=-2*std, b=2*std)
254
+ elif isinstance(module, nn.LayerNorm):
255
+ if module.bias is not None:
256
+ module.bias.data.zero_()
257
+ if module.weight is not None:
258
+ module.weight.data.fill_(1.0)
259
+
260
+
261
+ class LtgbertModel(LtgbertPreTrainedModel):
262
+ def __init__(self, config, add_mlm_layer=False, gradient_checkpointing=False, **kwargs):
263
+ super().__init__(config, **kwargs)
264
+ self.config = config
265
+ self.hidden_size = config.hidden_size
266
+
267
+ self.embedding = Embedding(config)
268
+ self.transformer = Encoder(config, activation_checkpointing=gradient_checkpointing)
269
+ self.classifier = MaskClassifier(config, self.embedding.word_embedding.weight) if add_mlm_layer else None
270
+
271
+
272
+ def get_input_embeddings(self):
273
+ return self.embedding.word_embedding
274
+
275
+ def set_input_embeddings(self, value):
276
+ self.embedding.word_embedding = value
277
+
278
+ def get_contextualized_embeddings(
279
+ self,
280
+ input_ids: Optional[torch.Tensor] = None,
281
+ attention_mask: Optional[torch.Tensor] = None
282
+ ) -> List[torch.Tensor]:
283
+ if input_ids is not None:
284
+ input_shape = input_ids.size()
285
+ else:
286
+ raise ValueError("You have to specify input_ids")
287
+
288
+ batch_size, seq_length = input_shape
289
+ device = input_ids.device
290
+
291
+ if attention_mask is None:
292
+ attention_mask = torch.zeros(batch_size, seq_length, dtype=torch.bool, device=device)
293
+ else:
294
+ attention_mask = ~attention_mask.bool()
295
+ attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)
296
+
297
+ static_embeddings, relative_embedding = self.embedding(input_ids.t())
298
+ contextualized_embeddings, attention_probs = self.transformer(static_embeddings, attention_mask, relative_embedding)
299
+ contextualized_embeddings = [e.transpose(0, 1) for e in contextualized_embeddings]
300
+ last_layer = contextualized_embeddings[-1]
301
+ contextualized_embeddings = [contextualized_embeddings[0]] + [
302
+ contextualized_embeddings[i] - contextualized_embeddings[i - 1]
303
+ for i in range(1, len(contextualized_embeddings))
304
+ ]
305
+ return last_layer, contextualized_embeddings, attention_probs
306
+
307
+ def forward(
308
+ self,
309
+ input_ids: Optional[torch.Tensor] = None,
310
+ attention_mask: Optional[torch.Tensor] = None,
311
+ token_type_ids: Optional[torch.Tensor] = None,
312
+ position_ids: Optional[torch.Tensor] = None,
313
+ output_hidden_states: Optional[bool] = None,
314
+ output_attentions: Optional[bool] = None,
315
+ return_dict: Optional[bool] = None,
316
+ **kwargs
317
+ ) -> Union[Tuple[torch.Tensor], BaseModelOutput]:
318
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
319
+
320
+ sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
321
+
322
+ if not return_dict:
323
+ return (
324
+ sequence_output,
325
+ *([contextualized_embeddings] if output_hidden_states else []),
326
+ *([attention_probs] if output_attentions else [])
327
+ )
328
+
329
+ return BaseModelOutput(
330
+ last_hidden_state=sequence_output,
331
+ hidden_states=contextualized_embeddings if output_hidden_states else None,
332
+ attentions=attention_probs if output_attentions else None
333
+ )
334
+
335
+
336
+ class LtgbertForMaskedLM(LtgbertModel):
337
+ _keys_to_ignore_on_load_unexpected = ["head"]
338
+
339
+ def __init__(self, config, **kwargs):
340
+ super().__init__(config, add_mlm_layer=True, **kwargs)
341
+
342
+ def get_output_embeddings(self):
343
+ return self.classifier.nonlinearity[-1].weight
344
+
345
+ def set_output_embeddings(self, new_embeddings):
346
+ self.classifier.nonlinearity[-1].weight = new_embeddings
347
+
348
+ def forward(
349
+ self,
350
+ input_ids: Optional[torch.Tensor] = None,
351
+ attention_mask: Optional[torch.Tensor] = None,
352
+ token_type_ids: Optional[torch.Tensor] = None,
353
+ position_ids: Optional[torch.Tensor] = None,
354
+ output_hidden_states: Optional[bool] = None,
355
+ output_attentions: Optional[bool] = None,
356
+ return_dict: Optional[bool] = None,
357
+ labels: Optional[torch.LongTensor] = None,
358
+ **kwargs
359
+ ) -> Union[Tuple[torch.Tensor], MaskedLMOutput]:
360
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
361
+
362
+ sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
363
+ subword_prediction = self.classifier(sequence_output)
364
+ subword_prediction[:, :, :106+1] = float("-inf")
365
+
366
+ masked_lm_loss = None
367
+ if labels is not None:
368
+ masked_lm_loss = F.cross_entropy(subword_prediction.flatten(0, 1), labels.flatten())
369
+
370
+ if not return_dict:
371
+ output = (
372
+ subword_prediction,
373
+ *([contextualized_embeddings] if output_hidden_states else []),
374
+ *([attention_probs] if output_attentions else [])
375
+ )
376
+ return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output
377
+
378
+ return MaskedLMOutput(
379
+ loss=masked_lm_loss,
380
+ logits=subword_prediction,
381
+ hidden_states=contextualized_embeddings if output_hidden_states else None,
382
+ attentions=attention_probs if output_attentions else None
383
+ )
384
+
385
+
386
+ class Classifier(nn.Module):
387
+ def __init__(self, config, num_labels: int):
388
+ super().__init__()
389
+
390
+ drop_out = getattr(config, "cls_dropout", None)
391
+ drop_out = config.hidden_dropout_prob if drop_out is None else drop_out
392
+
393
+ self.nonlinearity = nn.Sequential(
394
+ nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=False),
395
+ nn.Linear(config.hidden_size, config.hidden_size),
396
+ nn.GELU(),
397
+ nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=False),
398
+ nn.Dropout(drop_out),
399
+ nn.Linear(config.hidden_size, num_labels)
400
+ )
401
+
402
+ def forward(self, x):
403
+ x = self.nonlinearity(x)
404
+ return x
405
+
406
+
407
+ class LtgbertForSequenceClassification(LtgbertModel):
408
+ _keys_to_ignore_on_load_unexpected = ["classifier"]
409
+ _keys_to_ignore_on_load_missing = ["head"]
410
+
411
+ def __init__(self, config, **kwargs):
412
+ super().__init__(config, add_mlm_layer=False, **kwargs)
413
+
414
+ self.num_labels = config.num_labels
415
+ self.head = Classifier(config, self.num_labels)
416
+
417
+ def forward(
418
+ self,
419
+ input_ids: Optional[torch.Tensor] = None,
420
+ attention_mask: Optional[torch.Tensor] = None,
421
+ token_type_ids: Optional[torch.Tensor] = None,
422
+ position_ids: Optional[torch.Tensor] = None,
423
+ output_attentions: Optional[bool] = None,
424
+ output_hidden_states: Optional[bool] = None,
425
+ return_dict: Optional[bool] = None,
426
+ labels: Optional[torch.LongTensor] = None,
427
+ **kwargs
428
+ ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]:
429
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
430
+
431
+ sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
432
+ logits = self.head(sequence_output[:, 0, :])
433
+
434
+ loss = None
435
+ if labels is not None:
436
+ if self.config.problem_type is None:
437
+ if self.num_labels == 1:
438
+ self.config.problem_type = "regression"
439
+ elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
440
+ self.config.problem_type = "single_label_classification"
441
+ else:
442
+ self.config.problem_type = "multi_label_classification"
443
+
444
+ if self.config.problem_type == "regression":
445
+ loss_fct = nn.MSELoss()
446
+ if self.num_labels == 1:
447
+ loss = loss_fct(logits.squeeze(), labels.squeeze())
448
+ else:
449
+ loss = loss_fct(logits, labels)
450
+ elif self.config.problem_type == "single_label_classification":
451
+ loss_fct = nn.CrossEntropyLoss()
452
+ loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
453
+ elif self.config.problem_type == "multi_label_classification":
454
+ loss_fct = nn.BCEWithLogitsLoss()
455
+ loss = loss_fct(logits, labels)
456
+
457
+ if not return_dict:
458
+ output = (
459
+ logits,
460
+ *([contextualized_embeddings] if output_hidden_states else []),
461
+ *([attention_probs] if output_attentions else [])
462
+ )
463
+ return ((loss,) + output) if loss is not None else output
464
+
465
+ return SequenceClassifierOutput(
466
+ loss=loss,
467
+ logits=logits,
468
+ hidden_states=contextualized_embeddings if output_hidden_states else None,
469
+ attentions=attention_probs if output_attentions else None
470
+ )
471
+
472
+
473
+ class LtgbertForTokenClassification(LtgbertModel):
474
+ _keys_to_ignore_on_load_unexpected = ["classifier"]
475
+ _keys_to_ignore_on_load_missing = ["head"]
476
+
477
+ def __init__(self, config, **kwargs):
478
+ super().__init__(config, add_mlm_layer=False, **kwargs)
479
+
480
+ self.num_labels = config.num_labels
481
+ self.head = Classifier(config, self.num_labels)
482
+
483
+ def forward(
484
+ self,
485
+ input_ids: Optional[torch.Tensor] = None,
486
+ attention_mask: Optional[torch.Tensor] = None,
487
+ token_type_ids: Optional[torch.Tensor] = None,
488
+ position_ids: Optional[torch.Tensor] = None,
489
+ output_attentions: Optional[bool] = None,
490
+ output_hidden_states: Optional[bool] = None,
491
+ return_dict: Optional[bool] = None,
492
+ labels: Optional[torch.LongTensor] = None,
493
+ **kwargs
494
+ ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]:
495
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
496
+
497
+ sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
498
+ logits = self.head(sequence_output)
499
+
500
+ loss = None
501
+ if labels is not None:
502
+ loss_fct = nn.CrossEntropyLoss()
503
+ loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
504
+
505
+ if not return_dict:
506
+ output = (
507
+ logits,
508
+ *([contextualized_embeddings] if output_hidden_states else []),
509
+ *([attention_probs] if output_attentions else [])
510
+ )
511
+ return ((loss,) + output) if loss is not None else output
512
+
513
+ return TokenClassifierOutput(
514
+ loss=loss,
515
+ logits=logits,
516
+ hidden_states=contextualized_embeddings if output_hidden_states else None,
517
+ attentions=attention_probs if output_attentions else None
518
+ )
519
+
520
+
521
+ class LtgbertForQuestionAnswering(LtgbertModel):
522
+ _keys_to_ignore_on_load_unexpected = ["classifier"]
523
+ _keys_to_ignore_on_load_missing = ["head"]
524
+
525
+ def __init__(self, config, **kwargs):
526
+ super().__init__(config, add_mlm_layer=False, **kwargs)
527
+
528
+ self.num_labels = config.num_labels
529
+ self.head = Classifier(config, self.num_labels)
530
+
531
+ def forward(
532
+ self,
533
+ input_ids: Optional[torch.Tensor] = None,
534
+ attention_mask: Optional[torch.Tensor] = None,
535
+ token_type_ids: Optional[torch.Tensor] = None,
536
+ position_ids: Optional[torch.Tensor] = None,
537
+ output_attentions: Optional[bool] = None,
538
+ output_hidden_states: Optional[bool] = None,
539
+ return_dict: Optional[bool] = None,
540
+ start_positions: Optional[torch.Tensor] = None,
541
+ end_positions: Optional[torch.Tensor] = None,
542
+ **kwargs
543
+ ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]:
544
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
545
+
546
+ sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
547
+ logits = self.head(sequence_output)
548
+
549
+ start_logits, end_logits = logits.split(1, dim=-1)
550
+ start_logits = start_logits.squeeze(-1).contiguous()
551
+ end_logits = end_logits.squeeze(-1).contiguous()
552
+
553
+ total_loss = None
554
+ if start_positions is not None and end_positions is not None:
555
+ # If we are on multi-GPU, split add a dimension
556
+ if len(start_positions.size()) > 1:
557
+ start_positions = start_positions.squeeze(-1)
558
+ if len(end_positions.size()) > 1:
559
+ end_positions = end_positions.squeeze(-1)
560
+
561
+ # sometimes the start/end positions are outside our model inputs, we ignore these terms
562
+ ignored_index = start_logits.size(1)
563
+ start_positions = start_positions.clamp(0, ignored_index)
564
+ end_positions = end_positions.clamp(0, ignored_index)
565
+
566
+ loss_fct = nn.CrossEntropyLoss(ignore_index=ignored_index)
567
+ start_loss = loss_fct(start_logits, start_positions)
568
+ end_loss = loss_fct(end_logits, end_positions)
569
+ total_loss = (start_loss + end_loss) / 2
570
+
571
+ if not return_dict:
572
+ output = (
573
+ start_logits,
574
+ end_logits,
575
+ *([contextualized_embeddings] if output_hidden_states else []),
576
+ *([attention_probs] if output_attentions else [])
577
+ )
578
+ return ((total_loss,) + output) if total_loss is not None else output
579
+
580
+ return QuestionAnsweringModelOutput(
581
+ loss=total_loss,
582
+ start_logits=start_logits,
583
+ end_logits=end_logits,
584
+ hidden_states=contextualized_embeddings if output_hidden_states else None,
585
+ attentions=attention_probs if output_attentions else None
586
+ )
587
+
588
+
589
+ class LtgbertForMultipleChoice(LtgbertModel):
590
+ _keys_to_ignore_on_load_unexpected = ["classifier"]
591
+ _keys_to_ignore_on_load_missing = ["head"]
592
+
593
+ def __init__(self, config, **kwargs):
594
+ super().__init__(config, add_mlm_layer=False, **kwargs)
595
+
596
+ self.num_labels = getattr(config, "num_labels", 2)
597
+ self.head = Classifier(config, self.num_labels)
598
+
599
+ def forward(
600
+ self,
601
+ input_ids: Optional[torch.Tensor] = None,
602
+ attention_mask: Optional[torch.Tensor] = None,
603
+ token_type_ids: Optional[torch.Tensor] = None,
604
+ position_ids: Optional[torch.Tensor] = None,
605
+ labels: Optional[torch.Tensor] = None,
606
+ output_attentions: Optional[bool] = None,
607
+ output_hidden_states: Optional[bool] = None,
608
+ return_dict: Optional[bool] = None,
609
+ **kwargs
610
+ ) -> Union[Tuple[torch.Tensor], MultipleChoiceModelOutput]:
611
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
612
+ num_choices = input_ids.shape[1]
613
+
614
+ flat_input_ids = input_ids.view(-1, input_ids.size(-1))
615
+ flat_attention_mask = attention_mask.view(-1, attention_mask.size(-1)) if attention_mask is not None else None
616
+
617
+ sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(flat_input_ids, flat_attention_mask)
618
+ logits = self.head(sequence_output)
619
+ reshaped_logits = logits.view(-1, num_choices)
620
+
621
+ loss = None
622
+ if labels is not None:
623
+ loss_fct = nn.CrossEntropyLoss()
624
+ loss = loss_fct(reshaped_logits, labels)
625
+
626
+ if not return_dict:
627
+ output = (
628
+ reshaped_logits,
629
+ *([contextualized_embeddings] if output_hidden_states else []),
630
+ *([attention_probs] if output_attentions else [])
631
+ )
632
+ return ((loss,) + output) if loss is not None else output
633
+
634
+ return MultipleChoiceModelOutput(
635
+ loss=loss,
636
+ logits=reshaped_logits,
637
+ hidden_states=contextualized_embeddings if output_hidden_states else None,
638
+ attentions=attention_probs if output_attentions else None
639
+ )
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4ba06cd5f37e676787d5d83261aa3beca1c4cb3f919efe730d13499b5bd016d7
3
+ size 525206650
special_tokens_map.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "[BOS]",
3
+ "cls_token": {
4
+ "content": "[CLS]",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false
9
+ },
10
+ "eos_token": "[EOS]",
11
+ "mask_token": {
12
+ "content": "[MASK]",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false
17
+ },
18
+ "pad_token": {
19
+ "content": "[PAD]",
20
+ "lstrip": false,
21
+ "normalized": false,
22
+ "rstrip": false,
23
+ "single_word": false
24
+ },
25
+ "sep_token": {
26
+ "content": "[SEP]",
27
+ "lstrip": false,
28
+ "normalized": false,
29
+ "rstrip": false,
30
+ "single_word": false
31
+ },
32
+ "unk_token": {
33
+ "content": "[UNK]",
34
+ "lstrip": false,
35
+ "normalized": false,
36
+ "rstrip": false,
37
+ "single_word": false
38
+ }
39
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,870 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "[PAD]",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "[UNK]",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "[CLS]",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "[SEP]",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "4": {
36
+ "content": "[MASK]",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ },
43
+ "5": {
44
+ "content": "[MASK_1]",
45
+ "lstrip": false,
46
+ "normalized": false,
47
+ "rstrip": false,
48
+ "single_word": false,
49
+ "special": true
50
+ },
51
+ "6": {
52
+ "content": "[MASK_2]",
53
+ "lstrip": false,
54
+ "normalized": false,
55
+ "rstrip": false,
56
+ "single_word": false,
57
+ "special": true
58
+ },
59
+ "7": {
60
+ "content": "[MASK_3]",
61
+ "lstrip": false,
62
+ "normalized": false,
63
+ "rstrip": false,
64
+ "single_word": false,
65
+ "special": true
66
+ },
67
+ "8": {
68
+ "content": "[MASK_4]",
69
+ "lstrip": false,
70
+ "normalized": false,
71
+ "rstrip": false,
72
+ "single_word": false,
73
+ "special": true
74
+ },
75
+ "9": {
76
+ "content": "[MASK_5]",
77
+ "lstrip": false,
78
+ "normalized": false,
79
+ "rstrip": false,
80
+ "single_word": false,
81
+ "special": true
82
+ },
83
+ "10": {
84
+ "content": "[MASK_6]",
85
+ "lstrip": false,
86
+ "normalized": false,
87
+ "rstrip": false,
88
+ "single_word": false,
89
+ "special": true
90
+ },
91
+ "11": {
92
+ "content": "[MASK_7]",
93
+ "lstrip": false,
94
+ "normalized": false,
95
+ "rstrip": false,
96
+ "single_word": false,
97
+ "special": true
98
+ },
99
+ "12": {
100
+ "content": "[MASK_8]",
101
+ "lstrip": false,
102
+ "normalized": false,
103
+ "rstrip": false,
104
+ "single_word": false,
105
+ "special": true
106
+ },
107
+ "13": {
108
+ "content": "[MASK_9]",
109
+ "lstrip": false,
110
+ "normalized": false,
111
+ "rstrip": false,
112
+ "single_word": false,
113
+ "special": true
114
+ },
115
+ "14": {
116
+ "content": "[MASK_10]",
117
+ "lstrip": false,
118
+ "normalized": false,
119
+ "rstrip": false,
120
+ "single_word": false,
121
+ "special": true
122
+ },
123
+ "15": {
124
+ "content": "[MASK_11]",
125
+ "lstrip": false,
126
+ "normalized": false,
127
+ "rstrip": false,
128
+ "single_word": false,
129
+ "special": true
130
+ },
131
+ "16": {
132
+ "content": "[MASK_12]",
133
+ "lstrip": false,
134
+ "normalized": false,
135
+ "rstrip": false,
136
+ "single_word": false,
137
+ "special": true
138
+ },
139
+ "17": {
140
+ "content": "[MASK_13]",
141
+ "lstrip": false,
142
+ "normalized": false,
143
+ "rstrip": false,
144
+ "single_word": false,
145
+ "special": true
146
+ },
147
+ "18": {
148
+ "content": "[MASK_14]",
149
+ "lstrip": false,
150
+ "normalized": false,
151
+ "rstrip": false,
152
+ "single_word": false,
153
+ "special": true
154
+ },
155
+ "19": {
156
+ "content": "[MASK_15]",
157
+ "lstrip": false,
158
+ "normalized": false,
159
+ "rstrip": false,
160
+ "single_word": false,
161
+ "special": true
162
+ },
163
+ "20": {
164
+ "content": "[MASK_16]",
165
+ "lstrip": false,
166
+ "normalized": false,
167
+ "rstrip": false,
168
+ "single_word": false,
169
+ "special": true
170
+ },
171
+ "21": {
172
+ "content": "[MASK_17]",
173
+ "lstrip": false,
174
+ "normalized": false,
175
+ "rstrip": false,
176
+ "single_word": false,
177
+ "special": true
178
+ },
179
+ "22": {
180
+ "content": "[MASK_18]",
181
+ "lstrip": false,
182
+ "normalized": false,
183
+ "rstrip": false,
184
+ "single_word": false,
185
+ "special": true
186
+ },
187
+ "23": {
188
+ "content": "[MASK_19]",
189
+ "lstrip": false,
190
+ "normalized": false,
191
+ "rstrip": false,
192
+ "single_word": false,
193
+ "special": true
194
+ },
195
+ "24": {
196
+ "content": "[MASK_20]",
197
+ "lstrip": false,
198
+ "normalized": false,
199
+ "rstrip": false,
200
+ "single_word": false,
201
+ "special": true
202
+ },
203
+ "25": {
204
+ "content": "[MASK_21]",
205
+ "lstrip": false,
206
+ "normalized": false,
207
+ "rstrip": false,
208
+ "single_word": false,
209
+ "special": true
210
+ },
211
+ "26": {
212
+ "content": "[MASK_22]",
213
+ "lstrip": false,
214
+ "normalized": false,
215
+ "rstrip": false,
216
+ "single_word": false,
217
+ "special": true
218
+ },
219
+ "27": {
220
+ "content": "[MASK_23]",
221
+ "lstrip": false,
222
+ "normalized": false,
223
+ "rstrip": false,
224
+ "single_word": false,
225
+ "special": true
226
+ },
227
+ "28": {
228
+ "content": "[MASK_24]",
229
+ "lstrip": false,
230
+ "normalized": false,
231
+ "rstrip": false,
232
+ "single_word": false,
233
+ "special": true
234
+ },
235
+ "29": {
236
+ "content": "[MASK_25]",
237
+ "lstrip": false,
238
+ "normalized": false,
239
+ "rstrip": false,
240
+ "single_word": false,
241
+ "special": true
242
+ },
243
+ "30": {
244
+ "content": "[MASK_26]",
245
+ "lstrip": false,
246
+ "normalized": false,
247
+ "rstrip": false,
248
+ "single_word": false,
249
+ "special": true
250
+ },
251
+ "31": {
252
+ "content": "[MASK_27]",
253
+ "lstrip": false,
254
+ "normalized": false,
255
+ "rstrip": false,
256
+ "single_word": false,
257
+ "special": true
258
+ },
259
+ "32": {
260
+ "content": "[MASK_28]",
261
+ "lstrip": false,
262
+ "normalized": false,
263
+ "rstrip": false,
264
+ "single_word": false,
265
+ "special": true
266
+ },
267
+ "33": {
268
+ "content": "[MASK_29]",
269
+ "lstrip": false,
270
+ "normalized": false,
271
+ "rstrip": false,
272
+ "single_word": false,
273
+ "special": true
274
+ },
275
+ "34": {
276
+ "content": "[MASK_30]",
277
+ "lstrip": false,
278
+ "normalized": false,
279
+ "rstrip": false,
280
+ "single_word": false,
281
+ "special": true
282
+ },
283
+ "35": {
284
+ "content": "[MASK_31]",
285
+ "lstrip": false,
286
+ "normalized": false,
287
+ "rstrip": false,
288
+ "single_word": false,
289
+ "special": true
290
+ },
291
+ "36": {
292
+ "content": "[MASK_32]",
293
+ "lstrip": false,
294
+ "normalized": false,
295
+ "rstrip": false,
296
+ "single_word": false,
297
+ "special": true
298
+ },
299
+ "37": {
300
+ "content": "[MASK_33]",
301
+ "lstrip": false,
302
+ "normalized": false,
303
+ "rstrip": false,
304
+ "single_word": false,
305
+ "special": true
306
+ },
307
+ "38": {
308
+ "content": "[MASK_34]",
309
+ "lstrip": false,
310
+ "normalized": false,
311
+ "rstrip": false,
312
+ "single_word": false,
313
+ "special": true
314
+ },
315
+ "39": {
316
+ "content": "[MASK_35]",
317
+ "lstrip": false,
318
+ "normalized": false,
319
+ "rstrip": false,
320
+ "single_word": false,
321
+ "special": true
322
+ },
323
+ "40": {
324
+ "content": "[MASK_36]",
325
+ "lstrip": false,
326
+ "normalized": false,
327
+ "rstrip": false,
328
+ "single_word": false,
329
+ "special": true
330
+ },
331
+ "41": {
332
+ "content": "[MASK_37]",
333
+ "lstrip": false,
334
+ "normalized": false,
335
+ "rstrip": false,
336
+ "single_word": false,
337
+ "special": true
338
+ },
339
+ "42": {
340
+ "content": "[MASK_38]",
341
+ "lstrip": false,
342
+ "normalized": false,
343
+ "rstrip": false,
344
+ "single_word": false,
345
+ "special": true
346
+ },
347
+ "43": {
348
+ "content": "[MASK_39]",
349
+ "lstrip": false,
350
+ "normalized": false,
351
+ "rstrip": false,
352
+ "single_word": false,
353
+ "special": true
354
+ },
355
+ "44": {
356
+ "content": "[MASK_40]",
357
+ "lstrip": false,
358
+ "normalized": false,
359
+ "rstrip": false,
360
+ "single_word": false,
361
+ "special": true
362
+ },
363
+ "45": {
364
+ "content": "[MASK_41]",
365
+ "lstrip": false,
366
+ "normalized": false,
367
+ "rstrip": false,
368
+ "single_word": false,
369
+ "special": true
370
+ },
371
+ "46": {
372
+ "content": "[MASK_42]",
373
+ "lstrip": false,
374
+ "normalized": false,
375
+ "rstrip": false,
376
+ "single_word": false,
377
+ "special": true
378
+ },
379
+ "47": {
380
+ "content": "[MASK_43]",
381
+ "lstrip": false,
382
+ "normalized": false,
383
+ "rstrip": false,
384
+ "single_word": false,
385
+ "special": true
386
+ },
387
+ "48": {
388
+ "content": "[MASK_44]",
389
+ "lstrip": false,
390
+ "normalized": false,
391
+ "rstrip": false,
392
+ "single_word": false,
393
+ "special": true
394
+ },
395
+ "49": {
396
+ "content": "[MASK_45]",
397
+ "lstrip": false,
398
+ "normalized": false,
399
+ "rstrip": false,
400
+ "single_word": false,
401
+ "special": true
402
+ },
403
+ "50": {
404
+ "content": "[MASK_46]",
405
+ "lstrip": false,
406
+ "normalized": false,
407
+ "rstrip": false,
408
+ "single_word": false,
409
+ "special": true
410
+ },
411
+ "51": {
412
+ "content": "[MASK_47]",
413
+ "lstrip": false,
414
+ "normalized": false,
415
+ "rstrip": false,
416
+ "single_word": false,
417
+ "special": true
418
+ },
419
+ "52": {
420
+ "content": "[MASK_48]",
421
+ "lstrip": false,
422
+ "normalized": false,
423
+ "rstrip": false,
424
+ "single_word": false,
425
+ "special": true
426
+ },
427
+ "53": {
428
+ "content": "[MASK_49]",
429
+ "lstrip": false,
430
+ "normalized": false,
431
+ "rstrip": false,
432
+ "single_word": false,
433
+ "special": true
434
+ },
435
+ "54": {
436
+ "content": "[MASK_50]",
437
+ "lstrip": false,
438
+ "normalized": false,
439
+ "rstrip": false,
440
+ "single_word": false,
441
+ "special": true
442
+ },
443
+ "55": {
444
+ "content": "[MASK_51]",
445
+ "lstrip": false,
446
+ "normalized": false,
447
+ "rstrip": false,
448
+ "single_word": false,
449
+ "special": true
450
+ },
451
+ "56": {
452
+ "content": "[MASK_52]",
453
+ "lstrip": false,
454
+ "normalized": false,
455
+ "rstrip": false,
456
+ "single_word": false,
457
+ "special": true
458
+ },
459
+ "57": {
460
+ "content": "[MASK_53]",
461
+ "lstrip": false,
462
+ "normalized": false,
463
+ "rstrip": false,
464
+ "single_word": false,
465
+ "special": true
466
+ },
467
+ "58": {
468
+ "content": "[MASK_54]",
469
+ "lstrip": false,
470
+ "normalized": false,
471
+ "rstrip": false,
472
+ "single_word": false,
473
+ "special": true
474
+ },
475
+ "59": {
476
+ "content": "[MASK_55]",
477
+ "lstrip": false,
478
+ "normalized": false,
479
+ "rstrip": false,
480
+ "single_word": false,
481
+ "special": true
482
+ },
483
+ "60": {
484
+ "content": "[MASK_56]",
485
+ "lstrip": false,
486
+ "normalized": false,
487
+ "rstrip": false,
488
+ "single_word": false,
489
+ "special": true
490
+ },
491
+ "61": {
492
+ "content": "[MASK_57]",
493
+ "lstrip": false,
494
+ "normalized": false,
495
+ "rstrip": false,
496
+ "single_word": false,
497
+ "special": true
498
+ },
499
+ "62": {
500
+ "content": "[MASK_58]",
501
+ "lstrip": false,
502
+ "normalized": false,
503
+ "rstrip": false,
504
+ "single_word": false,
505
+ "special": true
506
+ },
507
+ "63": {
508
+ "content": "[MASK_59]",
509
+ "lstrip": false,
510
+ "normalized": false,
511
+ "rstrip": false,
512
+ "single_word": false,
513
+ "special": true
514
+ },
515
+ "64": {
516
+ "content": "[MASK_60]",
517
+ "lstrip": false,
518
+ "normalized": false,
519
+ "rstrip": false,
520
+ "single_word": false,
521
+ "special": true
522
+ },
523
+ "65": {
524
+ "content": "[MASK_61]",
525
+ "lstrip": false,
526
+ "normalized": false,
527
+ "rstrip": false,
528
+ "single_word": false,
529
+ "special": true
530
+ },
531
+ "66": {
532
+ "content": "[MASK_62]",
533
+ "lstrip": false,
534
+ "normalized": false,
535
+ "rstrip": false,
536
+ "single_word": false,
537
+ "special": true
538
+ },
539
+ "67": {
540
+ "content": "[MASK_63]",
541
+ "lstrip": false,
542
+ "normalized": false,
543
+ "rstrip": false,
544
+ "single_word": false,
545
+ "special": true
546
+ },
547
+ "68": {
548
+ "content": "[MASK_64]",
549
+ "lstrip": false,
550
+ "normalized": false,
551
+ "rstrip": false,
552
+ "single_word": false,
553
+ "special": true
554
+ },
555
+ "69": {
556
+ "content": "[MASK_65]",
557
+ "lstrip": false,
558
+ "normalized": false,
559
+ "rstrip": false,
560
+ "single_word": false,
561
+ "special": true
562
+ },
563
+ "70": {
564
+ "content": "[MASK_66]",
565
+ "lstrip": false,
566
+ "normalized": false,
567
+ "rstrip": false,
568
+ "single_word": false,
569
+ "special": true
570
+ },
571
+ "71": {
572
+ "content": "[MASK_67]",
573
+ "lstrip": false,
574
+ "normalized": false,
575
+ "rstrip": false,
576
+ "single_word": false,
577
+ "special": true
578
+ },
579
+ "72": {
580
+ "content": "[MASK_68]",
581
+ "lstrip": false,
582
+ "normalized": false,
583
+ "rstrip": false,
584
+ "single_word": false,
585
+ "special": true
586
+ },
587
+ "73": {
588
+ "content": "[MASK_69]",
589
+ "lstrip": false,
590
+ "normalized": false,
591
+ "rstrip": false,
592
+ "single_word": false,
593
+ "special": true
594
+ },
595
+ "74": {
596
+ "content": "[MASK_70]",
597
+ "lstrip": false,
598
+ "normalized": false,
599
+ "rstrip": false,
600
+ "single_word": false,
601
+ "special": true
602
+ },
603
+ "75": {
604
+ "content": "[MASK_71]",
605
+ "lstrip": false,
606
+ "normalized": false,
607
+ "rstrip": false,
608
+ "single_word": false,
609
+ "special": true
610
+ },
611
+ "76": {
612
+ "content": "[MASK_72]",
613
+ "lstrip": false,
614
+ "normalized": false,
615
+ "rstrip": false,
616
+ "single_word": false,
617
+ "special": true
618
+ },
619
+ "77": {
620
+ "content": "[MASK_73]",
621
+ "lstrip": false,
622
+ "normalized": false,
623
+ "rstrip": false,
624
+ "single_word": false,
625
+ "special": true
626
+ },
627
+ "78": {
628
+ "content": "[MASK_74]",
629
+ "lstrip": false,
630
+ "normalized": false,
631
+ "rstrip": false,
632
+ "single_word": false,
633
+ "special": true
634
+ },
635
+ "79": {
636
+ "content": "[MASK_75]",
637
+ "lstrip": false,
638
+ "normalized": false,
639
+ "rstrip": false,
640
+ "single_word": false,
641
+ "special": true
642
+ },
643
+ "80": {
644
+ "content": "[MASK_76]",
645
+ "lstrip": false,
646
+ "normalized": false,
647
+ "rstrip": false,
648
+ "single_word": false,
649
+ "special": true
650
+ },
651
+ "81": {
652
+ "content": "[MASK_77]",
653
+ "lstrip": false,
654
+ "normalized": false,
655
+ "rstrip": false,
656
+ "single_word": false,
657
+ "special": true
658
+ },
659
+ "82": {
660
+ "content": "[MASK_78]",
661
+ "lstrip": false,
662
+ "normalized": false,
663
+ "rstrip": false,
664
+ "single_word": false,
665
+ "special": true
666
+ },
667
+ "83": {
668
+ "content": "[MASK_79]",
669
+ "lstrip": false,
670
+ "normalized": false,
671
+ "rstrip": false,
672
+ "single_word": false,
673
+ "special": true
674
+ },
675
+ "84": {
676
+ "content": "[MASK_80]",
677
+ "lstrip": false,
678
+ "normalized": false,
679
+ "rstrip": false,
680
+ "single_word": false,
681
+ "special": true
682
+ },
683
+ "85": {
684
+ "content": "[MASK_81]",
685
+ "lstrip": false,
686
+ "normalized": false,
687
+ "rstrip": false,
688
+ "single_word": false,
689
+ "special": true
690
+ },
691
+ "86": {
692
+ "content": "[MASK_82]",
693
+ "lstrip": false,
694
+ "normalized": false,
695
+ "rstrip": false,
696
+ "single_word": false,
697
+ "special": true
698
+ },
699
+ "87": {
700
+ "content": "[MASK_83]",
701
+ "lstrip": false,
702
+ "normalized": false,
703
+ "rstrip": false,
704
+ "single_word": false,
705
+ "special": true
706
+ },
707
+ "88": {
708
+ "content": "[MASK_84]",
709
+ "lstrip": false,
710
+ "normalized": false,
711
+ "rstrip": false,
712
+ "single_word": false,
713
+ "special": true
714
+ },
715
+ "89": {
716
+ "content": "[MASK_85]",
717
+ "lstrip": false,
718
+ "normalized": false,
719
+ "rstrip": false,
720
+ "single_word": false,
721
+ "special": true
722
+ },
723
+ "90": {
724
+ "content": "[MASK_86]",
725
+ "lstrip": false,
726
+ "normalized": false,
727
+ "rstrip": false,
728
+ "single_word": false,
729
+ "special": true
730
+ },
731
+ "91": {
732
+ "content": "[MASK_87]",
733
+ "lstrip": false,
734
+ "normalized": false,
735
+ "rstrip": false,
736
+ "single_word": false,
737
+ "special": true
738
+ },
739
+ "92": {
740
+ "content": "[MASK_88]",
741
+ "lstrip": false,
742
+ "normalized": false,
743
+ "rstrip": false,
744
+ "single_word": false,
745
+ "special": true
746
+ },
747
+ "93": {
748
+ "content": "[MASK_89]",
749
+ "lstrip": false,
750
+ "normalized": false,
751
+ "rstrip": false,
752
+ "single_word": false,
753
+ "special": true
754
+ },
755
+ "94": {
756
+ "content": "[MASK_90]",
757
+ "lstrip": false,
758
+ "normalized": false,
759
+ "rstrip": false,
760
+ "single_word": false,
761
+ "special": true
762
+ },
763
+ "95": {
764
+ "content": "[MASK_91]",
765
+ "lstrip": false,
766
+ "normalized": false,
767
+ "rstrip": false,
768
+ "single_word": false,
769
+ "special": true
770
+ },
771
+ "96": {
772
+ "content": "[MASK_92]",
773
+ "lstrip": false,
774
+ "normalized": false,
775
+ "rstrip": false,
776
+ "single_word": false,
777
+ "special": true
778
+ },
779
+ "97": {
780
+ "content": "[MASK_93]",
781
+ "lstrip": false,
782
+ "normalized": false,
783
+ "rstrip": false,
784
+ "single_word": false,
785
+ "special": true
786
+ },
787
+ "98": {
788
+ "content": "[MASK_94]",
789
+ "lstrip": false,
790
+ "normalized": false,
791
+ "rstrip": false,
792
+ "single_word": false,
793
+ "special": true
794
+ },
795
+ "99": {
796
+ "content": "[MASK_95]",
797
+ "lstrip": false,
798
+ "normalized": false,
799
+ "rstrip": false,
800
+ "single_word": false,
801
+ "special": true
802
+ },
803
+ "100": {
804
+ "content": "[MASK_96]",
805
+ "lstrip": false,
806
+ "normalized": false,
807
+ "rstrip": false,
808
+ "single_word": false,
809
+ "special": true
810
+ },
811
+ "101": {
812
+ "content": "[MASK_97]",
813
+ "lstrip": false,
814
+ "normalized": false,
815
+ "rstrip": false,
816
+ "single_word": false,
817
+ "special": true
818
+ },
819
+ "102": {
820
+ "content": "[MASK_98]",
821
+ "lstrip": false,
822
+ "normalized": false,
823
+ "rstrip": false,
824
+ "single_word": false,
825
+ "special": true
826
+ },
827
+ "103": {
828
+ "content": "[MASK_99]",
829
+ "lstrip": false,
830
+ "normalized": false,
831
+ "rstrip": false,
832
+ "single_word": false,
833
+ "special": true
834
+ },
835
+ "104": {
836
+ "content": "█",
837
+ "lstrip": false,
838
+ "normalized": false,
839
+ "rstrip": false,
840
+ "single_word": false,
841
+ "special": true
842
+ },
843
+ "32768": {
844
+ "content": "[BOS]",
845
+ "lstrip": false,
846
+ "normalized": false,
847
+ "rstrip": false,
848
+ "single_word": false,
849
+ "special": true
850
+ },
851
+ "32769": {
852
+ "content": "[EOS]",
853
+ "lstrip": false,
854
+ "normalized": false,
855
+ "rstrip": false,
856
+ "single_word": false,
857
+ "special": true
858
+ }
859
+ },
860
+ "bos_token": "[BOS]",
861
+ "clean_up_tokenization_spaces": true,
862
+ "cls_token": "[CLS]",
863
+ "eos_token": "[EOS]",
864
+ "mask_token": "[MASK]",
865
+ "model_max_length": 512,
866
+ "pad_token": "[PAD]",
867
+ "sep_token": "[SEP]",
868
+ "tokenizer_class": "PreTrainedTokenizerFast",
869
+ "unk_token": "[UNK]"
870
+ }
upos.py ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers import TokenClassificationPipeline
2
+
3
+ class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
4
+ def __init__(self,**kwargs):
5
+ import numpy
6
+ super().__init__(**kwargs)
7
+ x=self.model.config.label2id
8
+ y=[k for k in x if not k.startswith("I-")]
9
+ self.transition=numpy.full((len(x),len(x)),numpy.nan)
10
+ for k,v in x.items():
11
+ for j in ["I-"+k[2:]] if k.startswith("B-") else [k]+y if k.startswith("I-") else y:
12
+ self.transition[v,x[j]]=0
13
+ def check_model_type(self,supported_models):
14
+ pass
15
+ def postprocess(self,model_outputs,**kwargs):
16
+ import numpy
17
+ if "logits" not in model_outputs:
18
+ return self.postprocess(model_outputs[0],**kwargs)
19
+ m=model_outputs["logits"][0].numpy()
20
+ e=numpy.exp(m-numpy.max(m,axis=-1,keepdims=True))
21
+ z=e/e.sum(axis=-1,keepdims=True)
22
+ for i in range(m.shape[0]-1,0,-1):
23
+ m[i-1]+=numpy.nanmax(m[i]+self.transition,axis=1)
24
+ k=[numpy.nanargmax(m[0]+self.transition[0])]
25
+ for i in range(1,m.shape[0]):
26
+ k.append(numpy.nanargmax(m[i]+self.transition[k[-1]]))
27
+ w=[{"entity":self.model.config.id2label[j],"start":s,"end":e,"score":z[i,j]} for i,((s,e),j) in enumerate(zip(model_outputs["offset_mapping"][0].tolist(),k)) if s<e]
28
+ if "aggregation_strategy" in kwargs and kwargs["aggregation_strategy"]!="none":
29
+ for i,t in reversed(list(enumerate(w))):
30
+ p=t.pop("entity")
31
+ if p.startswith("I-"):
32
+ w[i-1]["score"]=min(w[i-1]["score"],t["score"])
33
+ w[i-1]["end"]=w.pop(i)["end"]
34
+ elif p.startswith("B-"):
35
+ t["entity_group"]=p[2:]
36
+ else:
37
+ t["entity_group"]=p
38
+ for t in w:
39
+ t["text"]=model_outputs["sentence"][t["start"]:t["end"]]
40
+ return w
41
+