add tokenizer

special_tokens_map.json

@@ -1 +1 @@
-{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}

tokenizer_config.json

@@ -1 +1 @@
-{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": "/root/.cache/huggingface/transformers/02c153bf53bcd0a6d3bc7f4a41f5448df74644b94a7137e6471c3cd75a57452f.a21d51735cf8667bcd610f057e88548d5d6a381401f6b4501a8bc6c1a9dc8498", "tokenizer_file": null, "name_or_path": "manandey/wav2vec2-large-xlsr-punjabi", "tokenizer_class": "Wav2Vec2CTCTokenizer"}
+{"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "<pad>", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": "/root/.cache/huggingface/transformers/784e5c6fc76efa418c10d5fb8e48bd37007867ec328b8a7f29a9104782b452c8.9d6cd81ef646692fb1c169a880161ea1cb95f49694f220aced9b704b457e51dd", "tokenizer_file": null, "name_or_path": "Harveenchadha/vakyansh-wav2vec2-hindi-him-4200", "tokenizer_class": "Wav2Vec2CTCTokenizer"}

vocab.json

@@ -1 +1 @@
-{"ਜ": 0, "ਖ": 1, "ਝ": 2, "ਗ਼": 3, "ੁ": 4, "ਰ": 5, "ਸ": 6, "ਢ": 7, "ਦ": 8, "ੋ": 9, "ਠ": 10, "ਣ": 11, "ਸ਼": 12, "ਊ": 13, "ੌ": 14, "ਹ": 15, "ਓ": 16, "ਆ": 17, "ਇ": 18, "ੰ": 19, "ਾ": 20, "ਘ": 21, "ਕ": 23, "ੀ": 24, "ਤ": 25, "ਐ": 26, "ਜ਼": 27, "ਡ": 28, "ਈ": 29, "ਚ": 30, "ਛ": 31, "ਖ਼": 32, "ਧ": 33, "ੜ": 34, "ੱ": 35, "ਥ": 36, "ਫ਼": 37, "ਬ": 38, "ਂ": 39, "ਟ": 40, "ਭ": 41, "ਨ": 42, "ੂ": 43, "ਮ": 44, "੍": 45, "ਪ": 46, "ਵ": 47, "ਉ": 48, "ਫ": 49, "ਏ": 50, "ਔ": 51, "ੇ": 52, "ਲ਼": 53, "਼": 54, "ਯ": 55, "ਗ": 56, "ਅ": 57, "ਿ": 58, "ੈ": 59, "ਲ": 60, "|": 22, "[UNK]": 61, "[PAD]": 62}
+{"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3, "|": 4, "ँ": 5, "ं": 6, "ः": 7, "अ": 8, "आ": 9, "इ": 10, "ई": 11, "उ": 12, "ऊ": 13, "ऋ": 14, "ए": 15, "ऐ": 16, "ऑ": 17, "ओ": 18, "औ": 19, "क": 20, "ख": 21, "ग": 22, "घ": 23, "ङ": 24, "च": 25, "छ": 26, "ज": 27, "झ": 28, "ञ": 29, "ट": 30, "ठ": 31, "ड": 32, "ढ": 33, "ण": 34, "त": 35, "थ": 36, "द": 37, "ध": 38, "न": 39, "प": 40, "फ": 41, "ब": 42, "भ": 43, "म": 44, "य": 45, "र": 46, "ल": 47, "व": 48, "श": 49, "ष": 50, "स": 51, "ह": 52, "़": 53, "ा": 54, "ि": 55, "ी": 56, "ु": 57, "ू": 58, "ृ": 59, "ॅ": 60, "े": 61, "ै": 62, "ॉ": 63, "ो": 64, "ौ": 65, "्": 66}