Addding multiple orthographies

no Japanese yet, only latin based characters still

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "models/t5-1.3_base_nouns/",
+  "_name_or_path": "models/t5-1.5_special_chars_full/",
   "architectures": [
     "T5ForConditionalGeneration"
   ],

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f15ee67ce40176628e800736efe8b306d9184767fa5404cf75a0d2cf5a627014
+oid sha256:95f56b3ca947f730118bea445d4fac2ab563df7413b9d37c86af323b4fd54679
 size 242250792

special_tokens_map.json

@@ -140,13 +140,6 @@
       "rstrip": false,
       "single_word": false
     },
-    {
-      "content": "abo",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
     {
       "content": "[GERUND_SUFFIX:CLASS_1]",
       "lstrip": false,
@@ -168,13 +161,6 @@
       "rstrip": false,
       "single_word": false
     },
-    {
-      "content": "[GERUND_SUFFIX:CLASS_1:IYU]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
     {
       "content": "[NEGATION_SUFFIX]",
       "lstrip": false,
@@ -246,28 +232,28 @@
       "single_word": false
     },
     {
-      "content": "[SUBJECT:1ppe]",
+      "content": "[SUBJECT:1ps]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "[SUBJECT:1ps]",
+      "content": "[SUBJECT:1ppe]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "a",
+      "content": "[w1q]ep[w4q]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "[w1q]ep[w4q]",
+      "content": "a",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -288,14 +274,14 @@
       "single_word": false
     },
     {
-      "content": "[OBJECT:2ps]",
+      "content": "pe[w10q]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "pe[w10q]",
+      "content": "[OBJECT:2ps]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -337,14 +323,14 @@
       "single_word": false
     },
     {
-      "content": "[NEGATION_PREFIX]",
+      "content": "ix[w4q]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "ix[w4q]",
+      "content": "[NEGATION_PREFIX]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -393,14 +379,14 @@
       "single_word": false
     },
     {
-      "content": "[OBJECT_MARKER:3p:DEFAULT]",
+      "content": "opo",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "opo",
+      "content": "[OBJECT_MARKER:3p:DEFAULT]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -568,14 +554,14 @@
       "single_word": false
     },
     {
-      "content": "pa",
+      "content": "ramo",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "ramo",
+      "content": "pa",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -595,13 +581,6 @@
       "rstrip": false,
       "single_word": false
     },
-    {
-      "content": "[w5q]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
     {
       "content": "[GERUND_SUFFIX:CLASS_1:NASAL_VOWEL]",
       "lstrip": false,
@@ -638,14 +617,14 @@
       "single_word": false
     },
     {
-      "content": "oro",
+      "content": "[OBJECT:2ps:SUBJECT_1P]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "[OBJECT:2ps:SUBJECT_1P]",
+      "content": "oro",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -666,14 +645,14 @@
       "single_word": false
     },
     {
-      "content": "[PLURIFORM_PREFIX:R]",
+      "content": "[SUBJECT:2ps]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "[SUBJECT:2ps]",
+      "content": "[PLURIFORM_PREFIX:R]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,

tokenizer_config.json

@@ -1975,11 +1975,9 @@
     "e[w15q]ym",
     "[OBJECT_MARKER:3p:PLURIFORM_PREFIX:MONOSYLLABIC]",
     "[OBJECT:1ppi]",
-    "abo",
     "[GERUND_SUFFIX:CLASS_1]",
     "[SUB_VERB]",
     "n[w15q]",
-    "[GERUND_SUFFIX:CLASS_1:IYU]",
     "[NEGATION_SUFFIX]",
     "[GERUND_SUBJECT_PREFIX:1ppe]",
     "i",
@@ -1990,29 +1988,29 @@
     "[w1q]",
     "xe",
     "t",
-    "[SUBJECT:1ppe]",
     "[SUBJECT:1ps]",
-    "a",
+    "[SUBJECT:1ppe]",
     "[w1q]ep[w4q]",
+    "a",
     "[SUBJECT_PREFIX:1ppe]",
     "amo",
-    "[OBJECT:2ps]",
     "pe[w10q]",
+    "[OBJECT:2ps]",
     "[OBJECT:1ppe]",
     "[NEGATION_SUFFIX:CONSONANT_ENDING]",
     "[OBJECT:MUTUAL]",
     "[GERUND_SUFFIX:CLASS_2:ORAL_VOWEL_ENDING]",
     "[w1q]o",
-    "[NEGATION_PREFIX]",
     "ix[w4q]",
+    "[NEGATION_PREFIX]",
     "[CIRCUMSTANTIAL_SUFFIX:NULL_ENDING]",
     "[SUBJECT:1ppi]",
     "[PERMISSIVE_PREFIX:VOWEL]",
     "[w1q]e",
     "pe",
     "nde",
-    "[OBJECT_MARKER:3p:DEFAULT]",
     "opo",
+    "[OBJECT_MARKER:3p:DEFAULT]",
     "or[w4q]",
     "[GERUND_SUBJECT_PREFIX:2ps]",
     "[w1q]a",
@@ -2036,22 +2034,21 @@
     "[NEGATION_SUFFIX:VOWEL_ENDING]",
     "[SUBJECT_PREFIX:1ps]",
     "[GERUND_SUFFIX:CLASS_1:ORAL_VOWEL]",
-    "pa",
     "ramo",
+    "pa",
     "ere",
     "[OBJECT:REFLEXIVE]",
-    "[w5q]",
     "[GERUND_SUFFIX:CLASS_1:NASAL_VOWEL]",
     "na",
     "[GERUND_SUFFIX:CLASS_1:B]",
     "[OBJECT:3p:MONOSYLLABIC]",
     "[GERUND_SUFFIX:CLASS_1:CONSONANT]",
-    "oro",
     "[OBJECT:2ps:SUBJECT_1P]",
+    "oro",
     "[GERUND_SUBJECT_PREFIX:1ps]",
     "[PERMISSIVE_PREFIX:CONSONANT]",
-    "[PLURIFORM_PREFIX:R]",
     "[SUBJECT:2ps]",
+    "[PLURIFORM_PREFIX:R]",
     "o",
     "[SUBJECT:2ps:OBJECT_1P]",
     "end[w4q]",