model improved

maker.py

@@ -8,11 +8,11 @@ from transformers import AutoTokenizer,AutoConfig,Qwen2ForTokenClassification,De
 d=os.path.basename(url)
 os.system("test -d "+d+" || git clone --depth=1 "+url)
 os.system("for F in train dev test ; do cp "+d+"/*-$F.conllu $F.conllu ; done")
-tkz=AutoTokenizer.from_pretrained(src,unk_token="<|im_start|>",sep_token="<|im_end|>")
-tkz.save_pretrained("tmpdir")
+otk=AutoTokenizer.from_pretrained(src,unk_token="<|im_start|>",sep_token="<|im_end|>")
+otk.save_pretrained("tmpdir")
 os.rename("tmpdir/tokenizer.json","tmpdir/tokenizer.json.old")
 os.rename("tmpdir/merges.txt","tmpdir/oldmerges.txt")
-d=json.loads(tkz.backend_tokenizer.to_str())
+d=json.loads(otk.backend_tokenizer.to_str())
 form=set()
 with open("train.conllu","r",encoding="utf-8") as r:
   for s in r:
@@ -20,21 +20,22 @@ with open("train.conllu","r",encoding="utf-8") as r:
     if len(w)==10 and w[0].isdecimal():
       form.add(w[1])
 m=[t for t in d["model"]["merges"] if len(t)<5 and unicodedata.category(t[0])[0]!="P"]
-for i in range(len(tkz)):
-  w=tkz.decode(i)
+for i in range(len(otk)):
+  w=otk.decode(i)
   if len(w)==2 and w in form and not unicodedata.name(w[0]).startswith("HIRAGANA"):
-    k=tkz([w[0],w[1]],add_special_tokens=False)["input_ids"]
+    k=otk([w[0],w[1]],add_special_tokens=False)["input_ids"]
     if len(k[0])==1 and len(k[1])==1:
-      m.append(" ".join(tkz.convert_ids_to_tokens([k[0][0],k[1][0]])))
+      m.append(" ".join(otk.convert_ids_to_tokens([k[0][0],k[1][0]])))
 with open("tmpdir/merges.txt","w",encoding="utf-8") as w:
   print("#version: 0.2",file=w)
   print("\n".join(m),file=w)
-tkz=AutoTokenizer.from_pretrained("tmpdir")
+ntk=AutoTokenizer.from_pretrained("tmpdir")
 
 class UDCausalDataset(object):
-  def __init__(self,conllu,tokenizer,embeddings=None):
+  def __init__(self,conllu,tokenizer,oldtokenizer=None,embeddings=None):
     self.conllu=open(conllu,"r",encoding="utf-8")
     self.tokenizer=tokenizer
+    self.oldtokenizer=oldtokenizer if oldtokenizer else tokenizer
     self.embeddings=embeddings
     self.max_tokens=3
     self.seeks=[(0,0)]
@@ -79,8 +80,8 @@ class UDCausalDataset(object):
         if w[0].isdecimal():
           upos.append(w[3] if w[5]=="_" else w[3]+"|"+w[5])
           deps.append((int(w[6]),w[7]))
-    v=self.tokenizer(form,add_special_tokens=False)
     if t==0:
+      v=self.tokenizer(form,add_special_tokens=False)
       i,u=[],[]
       for j,(x,y) in enumerate(zip(v["input_ids"],upos)):
         if x!=[]:
@@ -90,6 +91,7 @@ class UDCausalDataset(object):
       pad=self.tokenizer.pad_token_id
     else:
       import torch
+      v=self.oldtokenizer(form,add_special_tokens=False)
       m=[]
       for x in v["input_ids"]:
         if x==[]:
@@ -117,9 +119,9 @@ class UDCausalDataset(object):
       upos=u[0:self.max_tokens]
     return {"inputs_embeds":emb[ids,:],"labels":[self.label2id[p] for p in upos]}
 
-trainDS=UDCausalDataset("train.conllu",tkz)
-devDS=UDCausalDataset("dev.conllu",tkz)
-testDS=UDCausalDataset("test.conllu",tkz)
+trainDS=UDCausalDataset("train.conllu",ntk,otk)
+devDS=UDCausalDataset("dev.conllu",ntk,otk)
+testDS=UDCausalDataset("test.conllu",ntk,otk)
 lid=trainDS(devDS,testDS)
 cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True)
 mdl=Qwen2ForTokenClassification.from_pretrained(src,config=cfg,ignore_mismatched_sizes=True)
@@ -129,4 +131,4 @@ arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=32,dataload
 trn=Trainer(args=arg,data_collator=DefaultDataCollator(),model=mdl,train_dataset=trainDS)
 trn.train()
 trn.save_model(tgt)
-tkz.save_pretrained(tgt)
+ntk.save_pretrained(tgt)

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dab62fe8682839188aaa62be1ef62d06d3e724852661e54cfabe34a6e6693870
+oid sha256:aea606a7a9a7b46f6f045932dc3960fb0deefe348b28aa3fbf419d4b189795db
 size 1856725466

tokenizer_config.json

@@ -32,7 +32,6 @@
     "<|im_end|>"
   ],
   "bos_token": null,
-  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "errors": "replace",

ud.py

@@ -1,5 +1,10 @@
 import numpy
-from transformers import TokenClassificationPipeline
+from transformers import TokenClassificationPipeline,AutoTokenizer
+try:
+  from transformers.utils import cached_file
+except:
+  from transformers.file_utils import cached_path,hf_bucket_url
+  cached_file=lambda x,y:os.path.join(x,y) if os.path.isdir(x) else cached_path(hf_bucket_url(x,y))
 
 class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
   def __init__(self,**kwargs):
@@ -42,6 +47,7 @@ class UniversalDependenciesCausalPipeline(BellmanFordTokenClassificationPipeline
   def __init__(self,**kwargs):
     kwargs["aggregation_strategy"]="simple"
     super().__init__(**kwargs)
+    self.oldtokenizer=AutoTokenizer.from_pretrained(self.tokenizer.name_or_path,merges_file=cached_file(self.tokenizer.name_or_path,"oldmerges.txt"))
     x=self.model.config.label2id
     self.root=numpy.full((len(x)),numpy.nan)
     self.left_arc=numpy.full((len(x)),numpy.nan)
@@ -87,7 +93,7 @@ class UniversalDependenciesCausalPipeline(BellmanFordTokenClassificationPipeline
       if d[i].strip()=="":
         d.pop(i)
         w.pop(i)
-    v=self.tokenizer(d,add_special_tokens=False)
+    v=self.oldtokenizer(d,add_special_tokens=False)
     e=self.model.get_input_embeddings().weight
     m=[]
     for x in v["input_ids"]: