variants augmentatiion

README.md

@@ -18,7 +18,7 @@ widget:
 
 ## Model Description
 
-This is a ModernBERT model pre-trained on 青空文庫 texts. You can fine-tune `modernbert-base-japanese-aozora` for downstream tasks, such as POS-tagging, dependency-parsing, and so on.
+This is a ModernBERT model pre-trained on 青空文庫 texts. NVIDIA A100-SXM4-40GB×8 took 5 hours 55 minutes for training. You can fine-tune `modernbert-base-japanese-aozora` for downstream tasks, such as POS-tagging, dependency-parsing, and so on.
 
 ## How to Use
 

maker.py

@@ -2,12 +2,10 @@
 #pip3 install transformers accelerate deepspeed triton datasets fugashi unidic-lite
 import os,json
 os.system("""
-if test -d transformers 
+if test -d transformers
 then :
 else git clone --depth=1 https://github.com/huggingface/transformers transformers-all
      ln -s transformers-all/src/transformers transformers
-     sed 's/-> \\(.*\\) | \\(.*\\):/-> Union[\\1, \\2]:/' transformers/models/modernbert/modeling_modernbert.py > modeling_modernbert.py
-     cp modeling_modernbert.py transformers/models/modernbert
 fi
 test -d ModernBERT-base || git clone --depth=1 https://huggingface.co/answerdotai/ModernBERT-base
 test -f ModernBERT-base/configuration_modernbert.py || sed 's/^from \\.\\.\\./from transformers./' transformers/models/modernbert/configuration_modernbert.py > ModernBERT-base/configuration_modernbert.py
@@ -27,17 +25,24 @@ if not "auto_map" in d:
     json.dump(d,w,indent=2)
 if not os.path.isfile("train.txt"):
   import datasets
+  aug=lambda x:(x.replace("侠","俠").replace("倶","俱").replace("洗","冼").replace("剥","剝").replace("即","卽").replace("呑","吞").replace("呉","吳").replace("填","塡").replace("巣","巢").replace("徴","徵").replace("徳","德").replace("掲","揭").replace("撃","擊").replace("教","敎").replace("晩","晚").replace("横","橫").replace("歩","步").replace("歴","歷").replace("毎","每").replace("冷","泠").replace("渉","涉").replace("涙","淚").replace("清","淸").replace("渇","渴").replace("温","溫").replace("状","狀").replace("産","產").replace("痩","瘦").replace("禰","祢").replace("箪","簞").replace("緑","綠").replace("緒","緖").replace("縁","緣").replace("繋","繫").replace("莱","萊").replace("薫","薰").replace("虚","虛").replace("蝉","蟬").replace("説","說").replace("躯","軀").replace("郎","郞").replace("醤","醬").replace("録","錄").replace("錬","鍊").replace("間","閒").replace("頬","頰").replace("顛","顚").replace("鴎","鷗").replace("麺","麵").replace("黄","黃").replace("黒","黑").replace("叱","𠮟"))
   with open("train.txt","w",encoding="utf-8") as w:
-    d,i=datasets.load_dataset("globis-university/aozorabunko-clean"),0
+    d,u,v=datasets.load_dataset("globis-university/aozorabunko-clean"),"",""
     for t in d["train"]:
       for s in t["text"].replace("。","。\n").replace("\u3000"," ").split("\n"):
-        if i+len(s)<10000:
-          print(s,end="",file=w)
-          i+=len(s)
+        r=aug(s)
+        if r!=s:
+          if len(r)+len(v)<10000:
+            v+=r
+          else:
+            print(v,file=w)
+            v=r
+        if len(s)+len(u)<10000:
+          u+=s
         else:
-          print("\n"+s,end="",file=w)
-          i=len(s)
-    print("",file=w)
+          print(u,file=w)
+          u=s
+    print(u,v,file=w)
 os.system("test -s token.txt || fugashi -Owakati < train.txt > token.txt")
 
 from transformers import DebertaV2TokenizerFast

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e33012f766bf187fcbc322e69c2b82c44d5610cbbf437b8c6975f8a2117f648c
+oid sha256:71a8a58812cf68dee53784bc77332caf4e40b0211db98eef49fd3827aada2735
 size 643674094