Upload tokenizer

Browse files

Files changed (2) hide show

tokenizer.json +149 -1
vocab.json +1 -1

tokenizer.json CHANGED Viewed

@@ -105,7 +105,155 @@
       "UNK": 0,
       "PAD": 1,
       "WORD_BOUNDARY": 2,
-      "UTT_BOUNDARY": 3
     },
     "unk_token": "UNK"
   }

       "UNK": 0,
       "PAD": 1,
       "WORD_BOUNDARY": 2,
+      "UTT_BOUNDARY": 3,
+      "aː˧": 4,
+      "t": 5,
+      "ɐ˥": 6,
+      "k": 7,
+      "l": 8,
+      "j": 9,
+      "ʊ˥": 10,
+      "aː˧˩̰": 11,
+      "ɛː˥": 12,
+      "n": 13,
+      "ei˩˧": 14,
+      "w": 15,
+      "aː˨": 16,
+      "ɐi˧˥": 17,
+      "m̩˧˥": 18,
+      "m": 19,
+      "ou˥": 20,
+      "aː˧˥": 21,
+      "ei˥": 22,
+      "iː˧": 23,
+      "ts": 24,
+      "ɔː˧˥": 25,
+      "tʰ": 26,
+      "iː˥": 27,
+      "f": 28,
+      "aːĭ˧": 29,
+      "ɐ˨": 30,
+      "p": 31,
+      "h": 32,
+      "ɵy˧": 33,
+      "aː˥": 34,
+      "ou˨": 35,
+      "ɔː˧": 36,
+      "ɐi˧˩̰": 37,
+      "uː˧": 38,
+      "ŋ": 39,
+      "s": 40,
+      "ɔːĭ˥": 41,
+      "ɐu˨": 42,
+      "iː˨": 43,
+      "ei˧˥": 44,
+      "ɐi˨": 45,
+      "ʊ˧˩̰": 46,
+      "ʊ˨": 47,
+      "aː˩˧": 48,
+      "aːĭ˧˥": 49,
+      "ɔː˨": 50,
+      "ɛː˩˧": 51,
+      "ɪ˨": 52,
+      "iːŭ˧": 53,
+      "ɛː˧˩̰": 54,
+      "ɪ˧˥": 55,
+      "̩˧˩̰": 56,
+      "ɵ˧˥": 57,
+      "ei˧": 58,
+      "ɐu˧˩̰": 59,
+      "m̩˧": 60,
+      "ɐu˧˥": 61,
+      "ɐu˩˧": 62,
+      "ɐi˥": 63,
+      "ɔː˥": 64,
+      "ɔːĭ˧": 65,
+      "ou˧˥": 66,
+      "ou˩˧": 67,
+      "ɐ˧": 68,
+      "tsʰ": 69,
+      "ɛː˧˥": 70,
+      "iː˧˥": 71,
+      "ɔː˩˧": 72,
+      "kʰ": 73,
+      "ɐ˧˩̰": 74,
+      "aːŭ˧˥": 75,
+      "pʰ": 76,
+      "aːĭ˧˩̰": 77,
+      "ɵy˩˧": 78,
+      "ɵ˧": 79,
+      "ɛː˧": 80,
+      "ei˧˩̰": 81,
+      "uː˧˥": 82,
+      "ɔː˧˩̰": 83,
+      "ɛː˨": 84,
+      "uː˥": 85,
+      "ʊ˧": 86,
+      "iː˧˩̰": 87,
+      "yː˨": 88,
+      "aːŭ˧": 89,
+      "œː˩˧": 90,
+      "ɐ˧˥": 91,
+      "iː˩˧": 92,
+      "ɪ˧˩̰": 93,
+      "iːŭ˧˩̰": 94,
+      "œː˧˥": 95,
+      "yː˧": 96,
+      "uːĭ˩˧": 97,
+      "ɵy˧˥": 98,
+      "yː˧˩̰": 99,
+      "ɔːĭ˧˥": 100,
+      "ɛː": 101,
+      "u˨": 102,
+      "ou˧": 103,
+      "ei˨": 104,
+      "ɐu˥": 105,
+      "ɵ˥": 106,
+      "uː˧˩̰": 107,
+      "yː˥": 108,
+      "ɪ˥": 109,
+      "œː˥": 110,
+      "œː˧˩̰": 111,
+      "aːĭ˨": 112,
+      "ɐ˩˧": 113,
+      "œː˧": 114,
+      "uːĭ˧˥": 115,
+      "ɐu˧": 116,
+      "ɐi˧": 117,
+      "ou˧˩̰": 118,
+      "aːĭ˥": 119,
+      "aːŭ˥": 120,
+      "yː˧˥": 121,
+      "iːŭ˥": 122,
+      "ɔːĭ˨": 123,
+      "ʊ˧˥": 124,
+      "m̩˥": 125,
+      "iːŭ˧˥": 126,
+      "ɐi˩˧": 127,
+      "ɵy˥": 128,
+      "uːĭ˧": 129,
+      "ɵy˧˩̰": 130,
+      "uːĭ˥": 131,
+      "aːŭ˧˩̰": 132,
+      "yː˩˧": 133,
+      "ɔːĭ˧˩̰": 134,
+      "aːŭ˩˧": 135,
+      "aːĭ˩˧": 136,
+      "uːĭ˨": 137,
+      "œː˨": 138,
+      "uː˨": 139,
+      "ɵy˨": 140,
+      "aːŭ˨": 141,
+      "m̩˩˧": 142,
+      "ŋ˩˧": 143,
+      "ɪ˧": 144,
+      "m̩˨": 145,
+      "iːŭ˩˧": 146,
+      "iːŭ˨": 147,
+      "ɵ˨": 148,
+      "uːĭ˧˩̰": 149,
+      "uː˩˧": 150,
+      "ɵ˧˩̰": 151
     },
     "unk_token": "UNK"
   }

vocab.json CHANGED Viewed

	@@ -1 +1 @@
1	- {"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3}

+ {"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"aː˧":4,"t":5,"ɐ˥":6,"k":7,"l":8,"j":9,"ʊ˥":10,"aː˧˩̰":11,"ɛː˥":12,"n":13,"ei˩˧":14,"w":15,"aː˨":16,"ɐi˧˥":17,"m̩˧˥":18,"m":19,"ou˥":20,"aː˧˥":21,"ei˥":22,"iː˧":23,"ts":24,"ɔː˧˥":25,"tʰ":26,"iː˥":27,"f":28,"aːĭ˧":29,"ɐ˨":30,"p":31,"h":32,"ɵy˧":33,"aː˥":34,"ou˨":35,"ɔː˧":36,"ɐi˧˩̰":37,"uː˧":38,"ŋ":39,"s":40,"ɔːĭ˥":41,"ɐu˨":42,"iː˨":43,"ei˧˥":44,"ɐi˨":45,"ʊ˧˩̰":46,"ʊ˨":47,"aː˩˧":48,"aːĭ˧˥":49,"ɔː˨":50,"ɛː˩˧":51,"ɪ˨":52,"iːŭ˧":53,"ɛː˧˩̰":54,"ɪ˧˥":55,"̩˧˩̰":56,"ɵ˧˥":57,"ei˧":58,"ɐu˧˩̰":59,"m̩˧":60,"ɐu˧˥":61,"ɐu˩˧":62,"ɐi˥":63,"ɔː˥":64,"ɔːĭ˧":65,"ou˧˥":66,"ou˩˧":67,"ɐ˧":68,"tsʰ":69,"ɛː˧˥":70,"iː˧˥":71,"ɔː˩˧":72,"kʰ":73,"ɐ˧˩̰":74,"aːŭ˧˥":75,"pʰ":76,"aːĭ˧˩̰":77,"ɵy˩˧":78,"ɵ˧":79,"ɛː˧":80,"ei˧˩̰":81,"uː˧˥":82,"ɔː˧˩̰":83,"ɛː˨":84,"uː˥":85,"ʊ˧":86,"iː˧˩̰":87,"yː˨":88,"aːŭ˧":89,"œː˩˧":90,"ɐ˧˥":91,"iː˩˧":92,"ɪ˧˩̰":93,"iːŭ˧˩̰":94,"œː˧˥":95,"yː˧":96,"uːĭ˩˧":97,"ɵy˧˥":98,"yː˧˩̰":99,"ɔːĭ˧˥":100,"ɛː":101,"u˨":102,"ou˧":103,"ei˨":104,"ɐu˥":105,"ɵ˥":106,"uː˧˩̰":107,"yː˥":108,"ɪ˥":109,"œː˥":110,"œː˧˩̰":111,"aːĭ˨":112,"ɐ˩˧":113,"œː˧":114,"uːĭ˧˥":115,"ɐu˧":116,"ɐi˧":117,"ou˧˩̰":118,"aːĭ˥":119,"aːŭ˥":120,"yː˧˥":121,"iːŭ˥":122,"ɔːĭ˨":123,"ʊ˧˥":124,"m̩˥":125,"iːŭ˧˥":126,"ɐi˩˧":127,"ɵy˥":128,"uːĭ˧":129,"ɵy˧˩̰":130,"uːĭ˥":131,"aːŭ˧˩̰":132,"yː˩˧":133,"ɔːĭ˧˩̰":134,"aːŭ˩˧":135,"aːĭ˩˧":136,"uːĭ˨":137,"œː˨":138,"uː˨":139,"ɵy˨":140,"aːŭ˨":141,"m̩˩˧":142,"ŋ˩˧":143,"ɪ˧":144,"m̩˨":145,"iːŭ˩˧":146,"iːŭ˨":147,"ɵ˨":148,"uːĭ˧˩̰":149,"uː˩˧":150,"ɵ˧˩̰":151}