|
|
|
src="KoichiYasuoka/gpt2-small-japanese-wikipedia-juman-upos" |
|
tgt="KoichiYasuoka/gpt2-small-japanese-wikipedia-juman-ud-causal" |
|
url="https://github.com/UniversalDependencies/UD_Japanese-GSDLUW" |
|
import os |
|
d=os.path.basename(url) |
|
os.system("test -d "+d+" || git clone --depth=1 "+url) |
|
os.system("for F in train dev test ; do cp "+d+"/*-$F.conllu $F.conllu ; done") |
|
class UDCausalDataset(object): |
|
def __init__(self,conllu,tokenizer,embeddings=None): |
|
self.conllu=open(conllu,"r",encoding="utf-8") |
|
self.tokenizer=tokenizer |
|
self.embeddings=embeddings |
|
self.max_tokens=3 |
|
self.seeks=[(0,0)] |
|
label=set(["SYM"]) |
|
dep=set() |
|
s=self.conllu.readline() |
|
while s!="": |
|
if s=="\n": |
|
self.seeks.append((self.conllu.tell(),0)) |
|
else: |
|
w=s.split("\t") |
|
if len(w)==10: |
|
if w[0].isdecimal(): |
|
p=w[3] if w[5]=="_" else w[3]+"|"+w[5] |
|
label.add(p) |
|
dep.add(p+("|" if w[6]=="0" else "|l-" if int(w[0])<int(w[6]) else "|r-")+w[7]) |
|
self.seeks.append((self.seeks[-1][0],int(w[0]))) |
|
self.max_tokens=max(self.max_tokens,int(w[0])*2+1) |
|
s=self.conllu.readline() |
|
lid={} |
|
for i,l in enumerate(sorted(label)): |
|
lid[l],lid["B-"+l],lid["I-"+l]=i*3,i*3+1,i*3+2 |
|
for i,d in enumerate(sorted(dep),len(lid)): |
|
lid[d]=i |
|
self.label2id=lid |
|
def __call__(*args): |
|
lid={l:i for i,l in enumerate(sorted(set(sum([list(t.label2id) for t in args],[]))))} |
|
for t in args: |
|
t.label2id=lid |
|
return lid |
|
def __del__(self): |
|
self.conllu.close() |
|
__len__=lambda self:len(self.seeks)-1 |
|
def __getitem__(self,i): |
|
s,t=self.seeks[i] |
|
self.conllu.seek(s) |
|
form,upos,deps,w=[],[],[],[""] |
|
while w[0]!="\n": |
|
w=self.conllu.readline().split("\t") |
|
if len(w)==10: |
|
form.append(w[1]) |
|
if w[0].isdecimal(): |
|
upos.append(w[3] if w[5]=="_" else w[3]+"|"+w[5]) |
|
deps.append((int(w[6]),w[7])) |
|
v=self.tokenizer(form,add_special_tokens=False) |
|
if t==0: |
|
i,u=[],[] |
|
for j,(x,y) in enumerate(zip(v["input_ids"],upos)): |
|
if x!=[]: |
|
i+=x |
|
u+=[y] if len(x)==1 else ["B-"+y]+["I-"+y]*(len(x)-1) |
|
emb=self.embeddings |
|
pad=self.tokenizer.pad_token_id |
|
else: |
|
import torch |
|
m=[] |
|
for x in v["input_ids"]: |
|
if x==[]: |
|
m.append(self.embeddings[self.tokenizer.unk_token_id,:]) |
|
else: |
|
m.append(self.embeddings[x,:].sum(axis=0)) |
|
m.append(self.embeddings[self.tokenizer.sep_token_id,:]) |
|
m.append(self.embeddings[self.tokenizer.pad_token_id,:]) |
|
emb=torch.stack(m) |
|
i,u=list(range(len(upos)+1)),upos+["SYM"] |
|
i.append(t-1) |
|
k,d=deps[t-1] |
|
u.append(upos[t-1]+"|"+d if k==0 else upos[t-1]) |
|
for j in range(t,len(upos)): |
|
i.append(j) |
|
a,b=deps[j] |
|
u.append(upos[j]+"|r-"+b if a==t else upos[t-1]+"|l-"+d if j+1==k else upos[j]) |
|
pad=-1 |
|
j=self.max_tokens-len(i) |
|
if j>0: |
|
ids=i+[pad]*j |
|
upos=u+["SYM"]*j |
|
else: |
|
ids=i[0:self.max_tokens] |
|
upos=u[0:self.max_tokens] |
|
return {"inputs_embeds":emb[ids,:],"labels":[self.label2id[p] for p in upos]} |
|
from transformers import ReformerTokenizer,AutoConfig,GPT2ForTokenClassification,DefaultDataCollator,TrainingArguments,Trainer |
|
tkz=ReformerTokenizer.from_pretrained(src,use_fast=False) |
|
trainDS=UDCausalDataset("train.conllu",tkz) |
|
devDS=UDCausalDataset("dev.conllu",tkz) |
|
testDS=UDCausalDataset("test.conllu",tkz) |
|
lid=trainDS(devDS,testDS) |
|
cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True) |
|
mdl=GPT2ForTokenClassification.from_pretrained(src,config=cfg,ignore_mismatched_sizes=True) |
|
trainDS.embeddings=mdl.get_input_embeddings().weight |
|
trainDS.max_tokens=min(trainDS.max_tokens,cfg.max_position_embeddings) |
|
arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=32,dataloader_pin_memory=False,output_dir=tgt,overwrite_output_dir=True,save_total_limit=2,learning_rate=5e-05,warmup_ratio=0.1,save_safetensors=False) |
|
trn=Trainer(args=arg,data_collator=DefaultDataCollator(),model=mdl,train_dataset=trainDS) |
|
trn.train() |
|
trn.save_model(tgt) |
|
tkz.save_pretrained(tgt) |
|
|
