Upload 86 files

.dockerignore

@@ -0,0 +1,7 @@
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.env
+.venv/
+.git/

.gitattributes

@@ -33,3 +33,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+libs/IndicTransToolkit/build/lib.linux-x86_64-cpython-310/IndicTransToolkit/fast_processor.cpython-310-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+libs/IndicTransToolkit/build/lib.linux-x86_64-cpython-310/IndicTransToolkit/processor.cpython-310-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+libs/IndicTransToolkit/build/temp.linux-x86_64-cpython-310/IndicTransToolkit/fast_processor.o filter=lfs diff=lfs merge=lfs -text
+libs/IndicTransToolkit/build/temp.linux-x86_64-cpython-310/IndicTransToolkit/processor.o filter=lfs diff=lfs merge=lfs -text
+libs/IndicTransToolkit/IndicTransToolkit/processor.cp313-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
+libs/IndicTransToolkit/IndicTransToolkit/processor.cpython-310-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,40 @@
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+import torch
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+import os
+import sys
+
+sys.path.append(os.path.abspath("libs/IndicTransToolkit"))
+from IndicTransToolkit.processor import IndicProcessor
+
+app = FastAPI(title="IndicTrans Translator API")
+
+ip = IndicProcessor(inference=True)
+tokenizer = AutoTokenizer.from_pretrained("ai4bharat/indictrans2-en-indic-dist-200M", trust_remote_code=True)
+model = AutoModelForSeq2SeqLM.from_pretrained("ai4bharat/indictrans2-en-indic-dist-200M", trust_remote_code=True)
+
+class TranslationRequest(BaseModel):
+    text: str
+    target_lang: str  
+
+@app.post("/translate")
+def translate_text(req: TranslationRequest):
+    if not req.text.strip():
+        raise HTTPException(status_code=400, detail="Input text is empty.")
+
+    try:
+        batch = ip.preprocess_batch([req.text], src_lang="eng_Latn", tgt_lang=req.target_lang)
+        batch = tokenizer(batch, padding="longest", truncation=True, max_length=256, return_tensors="pt")
+
+        with torch.inference_mode():
+            outputs = model.generate(**batch, num_beams=5, max_length=256)
+
+        with tokenizer.as_target_tokenizer():
+            decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True, clean_up_tokenization_spaces=True)
+
+        translated = ip.postprocess_batch(decoded, lang=req.target_lang)[0]
+        return {"translation": translated}
+
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))

libs/IndicTransToolkit/.gitignore

@@ -0,0 +1,4 @@
+dist/
+build/
+*.egg-info/
+*/*/__pycache__/

libs/IndicTransToolkit/CHANGELOG.md

@@ -0,0 +1,16 @@
+# Changelog
+
+# 📢 Release v1.0.3
+- 🚨 The `IndicProcessor` class has been re-written in [Cython](https://github.com/cython/cython) for faster implementation. This gives us atleast `+10 lines/s`.
+- A new `visualize` argument as been added to `preprocess_batch` to track the processing with a `tqdm` bar.
+
+# 📢 Release v1.0.2
+- The repository has been renamed to `IndicTransToolkit`.
+- 🚨 The custom tokenizer is now **removed** from the repository. Please revert to a previous commit ([v1.0.1](https://github.com/VarunGumma/IndicTransToolkit/tree/0e68fb5872f4d821578a5252f90ad43c9649370f)) to use it **(strongly discouraged)**. The official _(and only tokenizer)_ is available on HF along with the models.
+
+# 📢 Release v1.0.0
+- The [PreTrainedTokenizer](https://huggingface.co/docs/transformers/main_classes/tokenizer) for IndicTrans2 is now available on HF 🎉🎉 Note that, you still need the `IndicProcessor` to pre-process the sentences before tokenization.
+- 🚨 **In favor of the standard PreTrainedTokenizer, we deprecated the custom tokenizer. However, this custom tokenizer will still be available here for backward compatibility, but no further updates/bug-fixes will be provided.**
+- The `indic_evaluate` function is now consolidated into a concrete `IndicEvaluator` class.
+- The data collation function for training is consolidated into a concrete `IndicDataCollator` class.
+- A simple batching method is now available in the `IndicProcessor`.

libs/IndicTransToolkit/IndicTransToolkit.egg-info/PKG-INFO

@@ -0,0 +1,131 @@
+Metadata-Version: 2.4
+Name: IndicTransToolkit
+Version: 1.0.3
+Summary: A simple, consistent, and extendable module for IndicTrans2 tokenizer compatible with HuggingFace models
+Home-page: https://github.com/VarunGumma/IndicTransToolkit
+Author: Varun Gumma
+Author-email: [email protected]
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: setuptools>=68.2.2
+Requires-Dist: torch
+Requires-Dist: cython
+Requires-Dist: sacremoses
+Requires-Dist: sentencepiece
+Requires-Dist: transformers
+Requires-Dist: sacrebleu
+Requires-Dist: indic-nlp-library-IT2@ git+https://github.com/VarunGumma/indic_nlp_library.git
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# IndicTransToolkit
+
+## About
+The goal of this repository is to provide a simple, modular, and extendable toolkit for [IndicTrans2](https://github.com/AI4Bharat/IndicTrans2) and be compatible with the HuggingFace models released. Please refer to the `CHANGELOG.md` for latest developments.
+
+## Pre-requisites
+ - `Python 3.8+`
+ - [Indic NLP Library](https://github.com/VarunGumma/indic_nlp_library)
+ - Other requirements as listed in `requirements.txt`
+
+## Configuration
+ - Editable installation (Note, this may take a while):
+```bash 
+git clone https://github.com/VarunGumma/IndicTransToolkit
+cd IndicTransToolkit
+
+pip install --editable . --use-pep517 # required for pip >= 25.0
+
+# in case it fails, try:
+# pip install --editable . --use-pep517 --config-settings editable_mode=compat
+```
+
+## Examples
+For the training usecase, please refer [here](https://github.com/AI4Bharat/IndicTrans2/tree/main/huggingface_interface).
+
+### PreTainedTokenizer 
+```python
+import torch
+from IndicTransToolkit.processor import IndicProcessor # NOW IMPLEMENTED IN CYTHON !!
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+
+ip = IndicProcessor(inference=True)
+tokenizer = AutoTokenizer.from_pretrained("ai4bharat/indictrans2-en-indic-dist-200M", trust_remote_code=True)
+model = AutoModelForSeq2SeqLM.from_pretrained("ai4bharat/indictrans2-en-indic-dist-200M", trust_remote_code=True)
+
+sentences = [
+    "This is a test sentence.",
+    "This is another longer different test sentence.",
+    "Please send an SMS to 9876543210 and an email on [email protected] by 15th October, 2023.",
+]
+
+batch = ip.preprocess_batch(sentences, src_lang="eng_Latn", tgt_lang="hin_Deva", visualize=False) # set it to visualize=True to print a progress bar
+batch = tokenizer(batch, padding="longest", truncation=True, max_length=256, return_tensors="pt")
+
+with torch.inference_mode():
+    outputs = model.generate(**batch, num_beams=5, num_return_sequences=1, max_length=256)
+
+with tokenizer.as_target_tokenizer():
+    # This scoping is absolutely necessary, as it will instruct the tokenizer to tokenize using the target vocabulary.
+    # Failure to use this scoping will result in gibberish/unexpected predictions as the output will be de-tokenized with the source vocabulary instead.
+    outputs = tokenizer.batch_decode(outputs, skip_special_tokens=True, clean_up_tokenization_spaces=True)
+
+outputs = ip.postprocess_batch(outputs, lang="hin_Deva")
+print(outputs)
+
+>>> ['यह एक परीक्षण वाक्य है।', 'यह एक और लंबा अलग परीक्षण वाक्य है।', 'कृपया 9876543210 पर एक एस. एम. एस. भेजें और 15 अक्टूबर, 2023 तक [email protected] पर एक ईमेल भेजें।']
+```
+
+### Evaluation
+- `IndicEvaluator` is a python implementation of [compute_metrics.sh](https://github.com/AI4Bharat/IndicTrans2/blob/main/compute_metrics.sh). 
+- We have found that this python implementation gives slightly lower scores than the original `compute_metrics.sh`. So, please use this function cautiously, and feel free to raise a PR if you have found the bug/fix. 
+```python
+from IndicTransToolkit import IndicEvaluator
+
+# this method returns a dictionary with BLEU and ChrF2++ scores with appropriate signatures
+evaluator = IndicEvaluator()
+scores = evaluator.evaluate(tgt_lang=tgt_lang, preds=pred_file, refs=ref_file) 
+
+# alternatively, you can pass the list of predictions and references instead of files 
+# scores = evaluator.evaluate(tgt_lang=tgt_lang, preds=preds, refs=refs)
+```
+
+## Authors
+ - Varun Gumma ([email protected])
+ - Jay Gala ([email protected])
+ - Pranjal Agadh Chitale ([email protected])
+ - Raj Dabre ([email protected])
+
+
+## Bugs and Contribution
+Since this a bleeding-edge module, you may encounter broken stuff and import issues once in a while. In case you encounter any bugs or want additional functionalities, please feel free to raise `Issues`/`Pull Requests` or contact the authors. 
+
+
+## Citation
+If you use our codebase, or models, please do cite the following paper:
+```bibtex
+@article{
+    gala2023indictrans,
+    title={IndicTrans2: Towards High-Quality and Accessible Machine Translation Models for all 22 Scheduled Indian Languages},
+    author={Jay Gala and Pranjal A Chitale and A K Raghavan and Varun Gumma and Sumanth Doddapaneni and Aswanth Kumar M and Janki Atul Nawale and Anupama Sujatha and Ratish Puduppully and Vivek Raghavan and Pratyush Kumar and Mitesh M Khapra and Raj Dabre and Anoop Kunchukuttan},
+    journal={Transactions on Machine Learning Research},
+    issn={2835-8856},
+    year={2023},
+    url={https://openreview.net/forum?id=vfT4YuzAYA},
+    note={}
+}
+```

libs/IndicTransToolkit/IndicTransToolkit.egg-info/SOURCES.txt

@@ -0,0 +1,15 @@
+LICENSE
+README.md
+pyproject.toml
+setup.py
+IndicTransToolkit/__init__.py
+IndicTransToolkit/collator.py
+IndicTransToolkit/evaluator.py
+IndicTransToolkit/processor.c
+IndicTransToolkit/version.py
+IndicTransToolkit.egg-info/PKG-INFO
+IndicTransToolkit.egg-info/SOURCES.txt
+IndicTransToolkit.egg-info/dependency_links.txt
+IndicTransToolkit.egg-info/not-zip-safe
+IndicTransToolkit.egg-info/requires.txt
+IndicTransToolkit.egg-info/top_level.txt

libs/IndicTransToolkit/IndicTransToolkit.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

libs/IndicTransToolkit/IndicTransToolkit.egg-info/not-zip-safe

@@ -0,0 +1 @@
+

libs/IndicTransToolkit/IndicTransToolkit.egg-info/requires.txt

@@ -0,0 +1,8 @@
+setuptools>=68.2.2
+torch
+cython
+sacremoses
+sentencepiece
+transformers
+sacrebleu
+indic-nlp-library-IT2@ git+https://github.com/VarunGumma/indic_nlp_library.git

libs/IndicTransToolkit/IndicTransToolkit.egg-info/top_level.txt

@@ -0,0 +1 @@
+IndicTransToolkit

libs/IndicTransToolkit/IndicTransToolkit/__init__.py

@@ -0,0 +1,9 @@
+from .evaluator import IndicEvaluator
+from .collator import IndicDataCollator
+from .processor import IndicProcessor
+
+__all__ = [
+    "IndicEvaluator",
+    "IndicDataCollator",
+    "IndicProcessor",
+]

libs/IndicTransToolkit/IndicTransToolkit/collator.py

@@ -0,0 +1,74 @@
+import numpy as np
+from dataclasses import dataclass
+from typing import Any, Optional, Union
+
+from transformers.utils import PaddingStrategy
+from transformers.tokenization_utils import PreTrainedTokenizerBase
+from transformers.data.data_collator import pad_without_fast_tokenizer_warning
+
+
+@dataclass
+class IndicDataCollator:
+    tokenizer: PreTrainedTokenizerBase
+    model: Optional[Any] = None
+    padding: Union[bool, str, PaddingStrategy] = True
+    max_length: Optional[int] = None
+    pad_to_multiple_of: Optional[int] = None
+    label_pad_token_id: int = -100
+    return_tensors: str = "pt"
+
+    def __call__(self, features, return_tensors=None):
+
+        if return_tensors is None:
+            return_tensors = self.return_tensors
+
+        labels = (
+            [feature["labels"] for feature in features]
+            if "labels" in features[0].keys()
+            else None
+        )
+        # We have to pad the labels before calling `tokenizer.pad` as
+        # this method won't pad them and needs them of the same length to return tensors.
+        if labels is not None:
+            max_label_length = max(len(l) for l in labels)
+            if self.pad_to_multiple_of is not None:
+                max_label_length = (
+                    (max_label_length + self.pad_to_multiple_of - 1)
+                    // self.pad_to_multiple_of
+                    * self.pad_to_multiple_of
+                )
+
+            # fairseq by defaults right pad the labels for seq2seq tasks
+            for feature in features:
+                remainder = [self.label_pad_token_id] * (
+                    max_label_length - len(feature["labels"])
+                )
+                if isinstance(feature["labels"], list):
+                    feature["labels"] = feature["labels"] + remainder
+                else:
+                    feature["labels"] = np.concatenate(
+                        [feature["labels"], remainder]
+                    ).astype(np.int64)
+
+        self.tokenizer.padding_side = "left"
+        features = pad_without_fast_tokenizer_warning(
+            self.tokenizer,
+            features,
+            padding=self.padding,
+            max_length=self.max_length,
+            return_tensors=return_tensors,
+            pad_to_multiple_of=self.pad_to_multiple_of,
+        )
+
+        # prepare decoder_input_ids
+        if (
+            labels is not None
+            and self.model is not None
+            and hasattr(self.model, "prepare_decoder_input_ids_from_labels")
+        ):
+            decoder_input_ids = self.model.prepare_decoder_input_ids_from_labels(
+                labels=features["labels"]
+            )
+            features["decoder_input_ids"] = decoder_input_ids
+
+        return features

libs/IndicTransToolkit/IndicTransToolkit/evaluator.py

@@ -0,0 +1,151 @@
+from typing import List, Union
+from sacrebleu.metrics import CHRF, BLEU
+
+from indicnlp.tokenize import indic_tokenize
+from indicnlp.normalize.indic_normalize import IndicNormalizerFactory
+
+
+class IndicEvaluator:
+    def __init__(self):
+        # === Metrics ===
+        self._chrf2_metric = CHRF(word_order=2)
+        self._bleu_metric_13a = BLEU(tokenize="13a")
+        self._bleu_metric_none = BLEU(tokenize="none")
+
+        # === Normalizer factory and cache ===
+        self._indic_norm_factory = IndicNormalizerFactory()
+        self._normalizer_cache = {}  # Cache normalizers by iso_lang
+
+        # === FLORES -> ISO codes ===
+        self._flores_codes = {
+            "asm_Beng": "as",
+            "awa_Deva": "hi",
+            "ben_Beng": "bn",
+            "bho_Deva": "hi",
+            "brx_Deva": "hi",
+            "doi_Deva": "hi",
+            "eng_Latn": "en",
+            "gom_Deva": "kK",
+            "gon_Deva": "hi",
+            "guj_Gujr": "gu",
+            "hin_Deva": "hi",
+            "hne_Deva": "hi",
+            "kan_Knda": "kn",
+            "kas_Arab": "ur",
+            "kas_Deva": "hi",
+            "kha_Latn": "en",
+            "lus_Latn": "en",
+            "mag_Deva": "hi",
+            "mai_Deva": "hi",
+            "mal_Mlym": "ml",
+            "mar_Deva": "mr",
+            "mni_Beng": "bn",
+            "mni_Mtei": "hi",
+            "npi_Deva": "ne",
+            "ory_Orya": "or",
+            "pan_Guru": "pa",
+            "san_Deva": "hi",
+            "sat_Olck": "or",
+            "snd_Arab": "ur",
+            "snd_Deva": "hi",
+            "tam_Taml": "ta",
+            "tel_Telu": "te",
+            "urd_Arab": "ur",
+            "unr_Deva": "hi",
+        }
+
+    def _get_normalizer(self, iso_lang: str):
+        """
+        Return a cached normalizer for a given iso_lang.
+        """
+        if iso_lang not in self._normalizer_cache:
+            self._normalizer_cache[iso_lang] = self._indic_norm_factory.get_normalizer(iso_lang)
+        return self._normalizer_cache[iso_lang]
+
+    def _preprocess(self, sentences: List[str], lang: str) -> List[str]:
+        """
+        Preprocess the sentences using IndicNLP: 
+        1) Normalization (using a cached normalizer), 
+        2) Trivial tokenization.
+        """
+        iso_lang = self._flores_codes.get(lang, "hi")
+        # Fetch from cache to avoid reconstructing the normalizer
+        normalizer = self._get_normalizer(iso_lang)
+
+        # Local references for speed
+        trivial_tokenize = indic_tokenize.trivial_tokenize
+        normalize_fn = normalizer.normalize
+
+        processed_sentences = []
+        for line in sentences:
+            # single .strip() before normalizing
+            line = line.strip()
+            norm_line = normalize_fn(line)
+            tokens = trivial_tokenize(norm_line, iso_lang)
+            processed_sentences.append(" ".join(tokens))
+
+        return processed_sentences
+
+    def evaluate(
+        self,
+        tgt_lang: str,
+        preds: Union[List[str], str],
+        refs: Union[List[str], str],
+    ):
+        """
+        Evaluate BLEU and chrF2++ scores for the given predictions and references.
+        - If preds/refs are strings (filenames), read them from disk.
+        - If they are lists, evaluate them directly.
+        - For non-English languages, applies Indic NLP preprocessing before scoring.
+        """
+        assert preds is not None and refs is not None, "Predictions and References cannot be None"
+
+        # Convert file paths to lists if needed
+        if isinstance(preds, str):
+            with open(preds, "r", encoding="utf-8") as fp:
+                preds = [line.strip() for line in fp]
+        if isinstance(refs, str):
+            with open(refs, "r", encoding="utf-8") as fr:
+                refs = [line.strip() for line in fr]
+
+        assert len(preds) == len(refs), "Number of predictions and references do not match"
+
+        # Local references to metrics for speed
+        bleu_none = self._bleu_metric_none
+        bleu_13a = self._bleu_metric_13a
+        chrf2 = self._chrf2_metric
+
+        scores = {}
+
+        # For English (eng_Latn), skip Indic NLP normalization
+        if tgt_lang != "eng_Latn":
+            preds_ = self._preprocess(preds, tgt_lang)
+            refs_ = self._preprocess(refs, tgt_lang)
+
+            bleu_score = bleu_none.corpus_score(preds_, [refs_])
+            chrf_score = chrf2.corpus_score(preds_, [refs_])
+
+            scores["bleu"] = {
+                "score": round(bleu_score.score, 1),
+                "signature": bleu_none.get_signature().format(),
+            }
+            scores["chrF2++"] = {
+                "score": round(chrf_score.score, 1),
+                "signature": chrf2.get_signature().format(),
+            }
+
+        else:
+            # For English, 13a tokenization is standard
+            bleu_score = bleu_13a.corpus_score(preds, [refs])
+            chrf_score = chrf2.corpus_score(preds, [refs])
+
+            scores["bleu"] = {
+                "score": round(bleu_score.score, 1),
+                "signature": bleu_13a.get_signature().format(),
+            }
+            scores["chrF2++"] = {
+                "score": round(chrf_score.score, 1),
+                "signature": chrf2.get_signature().format(),
+            }
+
+        return scores

libs/IndicTransToolkit/IndicTransToolkit/processor.cp313-win_amd64.pyd

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6171c90f22a4602e67c36a791a618ccbf0d3703f17ea0c214186cb5fe3030487
+size 139776

libs/IndicTransToolkit/IndicTransToolkit/processor.cpython-310-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1a101ecb27adaf367f00c90b3f8e96e7fbda3bf0560d48c368fec3750a040a4
+size 229200

libs/IndicTransToolkit/IndicTransToolkit/processor.pyx

@@ -0,0 +1,503 @@
+# cython: language_level=3, boundscheck=False, cdivision=True, wraparound=False
+"""
+Cython version of the IndicProcessor class with optimizations for performance.
+Only preprocess_batch and postprocess_batch are exposed as cpdef methods.
+All other methods are internal (cdef) for optimized Cython usage.
+"""
+
+import regex as re
+from tqdm import tqdm
+from queue import Queue
+from typing import List, Dict, Union
+
+# Importing Python objects since these libraries don't offer C-extensions
+from indicnlp.tokenize import indic_tokenize, indic_detokenize
+from indicnlp.normalize.indic_normalize import IndicNormalizerFactory
+from sacremoses import MosesPunctNormalizer, MosesTokenizer, MosesDetokenizer
+from indicnlp.transliterate.unicode_transliterate import UnicodeIndicTransliterator
+
+
+cdef class IndicProcessor:
+    cdef public bint inference
+
+    # Precompiled regex patterns and placeholders
+    cdef object _MULTISPACE_REGEX
+    cdef object _DIGIT_SPACE_PERCENT
+    cdef object _DOUBLE_QUOT_PUNC
+    cdef object _DIGIT_NBSP_DIGIT
+    cdef object _END_BRACKET_SPACE_PUNC_REGEX
+
+    cdef object _URL_PATTERN
+    cdef object _NUMERAL_PATTERN
+    cdef object _EMAIL_PATTERN
+    cdef object _OTHER_PATTERN
+
+    cdef list _PUNC_REPLACEMENTS
+    cdef list _INDIC_FAILURE_CASES
+
+    cdef dict _flores_codes
+    cdef dict _digits_translation_table
+
+    # Placeholder maps stored in a Python Queue (treated as `object` for Cython)
+    cdef object _placeholder_entity_maps
+
+    # Tools (also Python objects)
+    cdef object _en_tok
+    cdef object _en_normalizer
+    cdef object _en_detok
+    cdef object _xliterator
+
+    def __cinit__(self, bint inference=True):
+        """
+        Constructor for IndicProcessor. Initializes all necessary components.
+        """
+        self.inference = inference
+
+        ##############################
+        # FLORES -> ISO CODES
+        ##############################
+        self._flores_codes = {
+            "asm_Beng": "as",
+            "awa_Deva": "hi",
+            "ben_Beng": "bn",
+            "bho_Deva": "hi",
+            "brx_Deva": "hi",
+            "doi_Deva": "hi",
+            "eng_Latn": "en",
+            "gom_Deva": "kK",
+            "gon_Deva": "hi",
+            "guj_Gujr": "gu",
+            "hin_Deva": "hi",
+            "hne_Deva": "hi",
+            "kan_Knda": "kn",
+            "kas_Arab": "ur",
+            "kas_Deva": "hi",
+            "kha_Latn": "en",
+            "lus_Latn": "en",
+            "mag_Deva": "hi",
+            "mai_Deva": "hi",
+            "mal_Mlym": "ml",
+            "mar_Deva": "mr",
+            "mni_Beng": "bn",
+            "mni_Mtei": "hi",
+            "npi_Deva": "ne",
+            "ory_Orya": "or",
+            "pan_Guru": "pa",
+            "san_Deva": "hi",
+            "sat_Olck": "or",
+            "snd_Arab": "ur",
+            "snd_Deva": "hi",
+            "tam_Taml": "ta",
+            "tel_Telu": "te",
+            "urd_Arab": "ur",
+            "unr_Deva": "hi",
+        }
+
+        ##############################
+        # INDIC DIGIT TRANSLATION (str.translate)
+        ##############################
+        self._digits_translation_table = {}
+        cdef dict digits_dict = {
+            "\u09e6": "0", "\u0ae6": "0", "\u0ce6": "0", "\u0966": "0",
+            "\u0660": "0", "\uabf0": "0", "\u0b66": "0", "\u0a66": "0",
+            "\u1c50": "0", "\u06f0": "0",
+
+            "\u09e7": "1", "\u0ae7": "1", "\u0967": "1", "\u0ce7": "1",
+            "\u06f1": "1", "\uabf1": "1", "\u0b67": "1", "\u0a67": "1",
+            "\u1c51": "1", "\u0c67": "1",
+
+            "\u09e8": "2", "\u0ae8": "2", "\u0968": "2", "\u0ce8": "2",
+            "\u06f2": "2", "\uabf2": "2", "\u0b68": "2", "\u0a68": "2",
+            "\u1c52": "2", "\u0c68": "2",
+
+            "\u09e9": "3", "\u0ae9": "3", "\u0969": "3", "\u0ce9": "3",
+            "\u06f3": "3", "\uabf3": "3", "\u0b69": "3", "\u0a69": "3",
+            "\u1c53": "3", "\u0c69": "3",
+
+            "\u09ea": "4", "\u0aea": "4", "\u096a": "4", "\u0cea": "4",
+            "\u06f4": "4", "\uabf4": "4", "\u0b6a": "4", "\u0a6a": "4",
+            "\u1c54": "4", "\u0c6a": "4",
+
+            "\u09eb": "5", "\u0aeb": "5", "\u096b": "5", "\u0ceb": "5",
+            "\u06f5": "5", "\uabf5": "5", "\u0b6b": "5", "\u0a6b": "5",
+            "\u1c55": "5", "\u0c6b": "5",
+
+            "\u09ec": "6", "\u0aec": "6", "\u096c": "6", "\u0cec": "6",
+            "\u06f6": "6", "\uabf6": "6", "\u0b6c": "6", "\u0a6c": "6",
+            "\u1c56": "6", "\u0c6c": "6",
+
+            "\u09ed": "7", "\u0aed": "7", "\u096d": "7", "\u0ced": "7",
+            "\u06f7": "7", "\uabf7": "7", "\u0b6d": "7", "\u0a6d": "7",
+            "\u1c57": "7", "\u0c6d": "7",
+
+            "\u09ee": "8", "\u0aee": "8", "\u096e": "8", "\u0cee": "8",
+            "\u06f8": "8", "\uabf8": "8", "\u0b6e": "8", "\u0a6e": "8",
+            "\u1c58": "8", "\u0c6e": "8",
+
+            "\u09ef": "9", "\u0aef": "9", "\u096f": "9", "\u0cef": "9",
+            "\u06f9": "9", "\uabf9": "9", "\u0b6f": "9", "\u0a6f": "9",
+            "\u1c59": "9", "\u0c6f": "9",
+        }
+        for k, v in digits_dict.items():
+            self._digits_translation_table[ord(k)] = v
+
+        # Also map ASCII '0'-'9'
+        for c in range(ord('0'), ord('9') + 1):
+            self._digits_translation_table[c] = chr(c)
+
+        ##############################
+        # PLACEHOLDER MAP QUEUE
+        ##############################
+        self._placeholder_entity_maps = Queue()
+
+        ##############################
+        # MOSES (as Python objects)
+        ##############################
+        self._en_tok = MosesTokenizer(lang="en")
+        self._en_normalizer = MosesPunctNormalizer()
+        self._en_detok = MosesDetokenizer(lang="en")
+
+        ##############################
+        # TRANSLITERATOR (Python object)
+        ##############################
+        self._xliterator = UnicodeIndicTransliterator()
+
+        ##############################
+        # Precompiled Patterns
+        ##############################
+        self._MULTISPACE_REGEX = re.compile(r"[ ]{2,}")
+        self._DIGIT_SPACE_PERCENT = re.compile(r"(\d) %")
+        self._DOUBLE_QUOT_PUNC = re.compile(r"\"([,\.]+)")
+        self._DIGIT_NBSP_DIGIT = re.compile(r"(\d) (\d)")
+        self._END_BRACKET_SPACE_PUNC_REGEX = re.compile(r"\) ([\.!:?;,])")
+
+        self._URL_PATTERN = re.compile(
+            r"\b(?<![\w/.])(?:(?:https?|ftp)://)?(?:(?:[\w-]+\.)+(?!\.))(?:[\w/\-?#&=%.]+)+(?!\.\w+)\b"
+        )
+        self._NUMERAL_PATTERN = re.compile(
+            r"(~?\d+\.?\d*\s?%?\s?-?\s?~?\d+\.?\d*\s?%|~?\d+%|\d+[-\/.,:']\d+[-\/.,:'+]\d+(?:\.\d+)?|\d+[-\/.:'+]\d+(?:\.\d+)?)"
+        )
+        self._EMAIL_PATTERN = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}")
+        self._OTHER_PATTERN = re.compile(r"[A-Za-z0-9]*[#|@]\w+")
+
+        # Combined punctuation replacements
+        self._PUNC_REPLACEMENTS = [
+            (re.compile(r"\r"), ""),
+            (re.compile(r"\(\s*"), "("),
+            (re.compile(r"\s*\)"), ")"),
+            (re.compile(r"\s:\s?"), ":"),
+            (re.compile(r"\s;\s?"), ";"),
+            (re.compile(r"[`´‘‚’]"), "'"),
+            (re.compile(r"[„“”«»]"), '"'),
+            (re.compile(r"[–—]"), "-"),
+            (re.compile(r"\.\.\."), "..."),
+            (re.compile(r" %"), "%"),
+            (re.compile(r"nº "), "nº "),
+            (re.compile(r" ºC"), " ºC"),
+            (re.compile(r" [?!;]"), lambda m: m.group(0).strip()),
+            (re.compile(r", "), ", "),
+        ]
+
+        self._INDIC_FAILURE_CASES = [
+            "آی ڈی ",
+            "ꯑꯥꯏꯗꯤ",
+            "आईडी",
+            "आई . डी . ",
+            "आई . डी .",
+            "आई. डी. ",
+            "आई. डी.",
+            "आय. डी. ",
+            "आय. डी.",
+            "आय . डी . ",
+            "आय . डी .",
+            "ऐटि",
+            "آئی ڈی ",
+            "ᱟᱭᱰᱤ ᱾",
+            "आयडी",
+            "ऐडि",
+            "आइडि",
+            "ᱟᱭᱰᱤ",
+        ]
+
+    # Internal Method: Apply punctuation replacements
+    cdef str _apply_punc_replacements(self, str text, list replacements) except *:
+        """
+        Apply a list of (pattern, replacement) in sequence to text.
+        """
+        cdef int i
+        cdef tuple pair
+        for i in range(len(replacements)):
+            pair = replacements[i]
+            text = pair[0].sub(pair[1], text)
+        return text
+
+    # Internal Method: Punctuation Normalization
+    cdef str _punc_norm(self, str text) except *:
+        """
+        Consolidate punctuation normalization in fewer passes.
+        """
+        # 1) Apply replacements
+        text = self._apply_punc_replacements(text, self._PUNC_REPLACEMENTS)
+
+        # 2) Additional patterns
+        text = self._MULTISPACE_REGEX.sub(" ", text)
+        text = self._END_BRACKET_SPACE_PUNC_REGEX.sub(r")\1", text)
+        text = self._DIGIT_SPACE_PERCENT.sub(r"\1%", text)
+        text = self._DOUBLE_QUOT_PUNC.sub(r'\1"', text)
+        text = self._DIGIT_NBSP_DIGIT.sub(r"\1.\2", text)
+        return text.strip()
+
+    # Internal Method: Wrap Text with Placeholders
+    cdef str _wrap_with_placeholders(self, str text) except *:
+        """
+        Wrap substrings with matched patterns in the text with placeholders.
+        Store the placeholder map in the queue for retrieval in postprocessing.
+        """
+        cdef int serial_no = 1
+        cdef dict placeholder_entity_map = {}
+        cdef list patterns = [
+            self._EMAIL_PATTERN,
+            self._URL_PATTERN,
+            self._NUMERAL_PATTERN,
+            self._OTHER_PATTERN,
+        ]
+        cdef object pattern
+        cdef set matches
+        cdef str match
+        cdef str base_placeholder
+        cdef int i
+
+        for pattern in patterns:
+            matches = set(pattern.findall(text))
+            for match in matches:
+                # Additional checks
+                if pattern is self._URL_PATTERN:
+                    if len(match.replace(".", "")) < 4:
+                        continue
+                if pattern is self._NUMERAL_PATTERN:
+                    if len(match.replace(" ", "").replace(".", "").replace(":", "")) < 4:
+                        continue
+
+                base_placeholder = f"<ID{serial_no}>"
+                # Map various placeholder formats to the matched text
+                placeholder_entity_map[f"<ID{serial_no}>"] = match
+                placeholder_entity_map[f"< ID{serial_no} >"] = match
+                placeholder_entity_map[f"[ID{serial_no}]"] = match
+                placeholder_entity_map[f"[ ID{serial_no} ]"] = match
+                placeholder_entity_map[f"[ID {serial_no}]"] = match
+                placeholder_entity_map[f"<ID{serial_no}]"] = match
+                placeholder_entity_map[f"< ID{serial_no}]"] = match
+                placeholder_entity_map[f"<ID{serial_no} ]"] = match
+
+                # Handle Indic failure cases
+                for i in range(len(self._INDIC_FAILURE_CASES)):
+                    indic_case = self._INDIC_FAILURE_CASES[i]
+                    placeholder_entity_map[f"<{indic_case}{serial_no}>"] = match
+                    placeholder_entity_map[f"< {indic_case}{serial_no} >"] = match
+                    placeholder_entity_map[f"< {indic_case} {serial_no} >"] = match
+                    placeholder_entity_map[f"<{indic_case} {serial_no}]"] = match
+                    placeholder_entity_map[f"< {indic_case} {serial_no} ]"] = match
+                    placeholder_entity_map[f"[{indic_case}{serial_no}]"] = match
+                    placeholder_entity_map[f"[{indic_case} {serial_no}]"] = match
+                    placeholder_entity_map[f"[ {indic_case}{serial_no} ]"] = match
+                    placeholder_entity_map[f"[ {indic_case} {serial_no} ]"] = match
+                    placeholder_entity_map[f"{indic_case} {serial_no}"] = match
+                    placeholder_entity_map[f"{indic_case}{serial_no}"] = match
+
+                # Replace the match with the base placeholder
+                text = text.replace(match, base_placeholder)
+                serial_no += 1
+
+        # Clean up any remaining placeholder artifacts
+        text = re.sub(r"\s+", " ", text).replace(">/", ">").replace("]/", "]")
+        self._placeholder_entity_maps.put(placeholder_entity_map)
+        return text
+
+    # Internal Method: Normalize Text
+    cdef str _normalize(self, str text) except *:
+        """
+        Normalizes numerals and optionally wraps placeholders.
+        """
+        # Single-pass digit translation
+        text = text.translate(self._digits_translation_table)
+
+        if self.inference:
+            text = self._wrap_with_placeholders(text)
+        return text
+
+    # Internal Method: Indic Tokenize and Transliterate
+    cdef str _do_indic_tokenize_and_transliterate(
+        self,
+        str sentence,
+        object normalizer,
+        str iso_lang,
+        bint transliterate
+    ) except *:
+        """
+        Helper method: normalizes, tokenizes, optionally transliterates from iso_lang -> 'hi'.
+        """
+        cdef str normed
+        cdef list tokens
+        cdef str joined
+        cdef str xlated
+
+        normed = normalizer.normalize(sentence.strip())
+        tokens = indic_tokenize.trivial_tokenize(normed, iso_lang)
+        joined = " ".join(tokens)
+        xlated = joined
+        if transliterate:
+            xlated = self._xliterator.transliterate(joined, iso_lang, "hi")
+            xlated = xlated.replace(" ् ", "्")
+        return xlated
+
+    # Internal Method: Preprocess a Single Sentence
+    cdef str _preprocess(
+        self,
+        str sent,
+        str src_lang,
+        str tgt_lang,
+        object normalizer,
+        bint is_target
+    ) except *:
+        """
+        Preprocess a single sentence: punctuation normalization, numeral normalization,
+        tokenization, transliteration, and adding language tags if necessary.
+        """
+        cdef str iso_lang = self._flores_codes.get(src_lang, "hi")
+        cdef str script_part = src_lang.split("_")[1]
+        cdef bint do_transliterate = True
+        cdef str e_strip
+        cdef str e_norm
+        cdef list e_tokens
+        cdef str processed_sent
+
+        # 1) Punctuation normalization
+        sent = self._punc_norm(sent)
+
+        # 2) Numerals & placeholders
+        sent = self._normalize(sent)
+
+        if script_part in ["Arab", "Aran", "Olck", "Mtei", "Latn"]:
+            do_transliterate = False
+
+        if iso_lang == "en":
+            # English path
+            e_strip = sent.strip()
+            e_norm = self._en_normalizer.normalize(e_strip)
+            e_tokens = self._en_tok.tokenize(e_norm, escape=False)
+            processed_sent = " ".join(e_tokens)
+        else:
+            # Indic path
+            processed_sent = self._do_indic_tokenize_and_transliterate(sent, normalizer, iso_lang, do_transliterate)
+
+        processed_sent = processed_sent.strip()
+        if not is_target:
+            return f"{src_lang} {tgt_lang} {processed_sent}"
+        else:
+            return processed_sent
+
+    # Internal Method: Postprocess a Single Sentence
+    cdef str _postprocess(self, object sent, str lang) except *:
+        """
+        Postprocess a single sentence:
+        1) Pull placeholder map from queue
+        2) Fix scripts for Perso-Arabic
+        3) Restore placeholders
+        4) Detokenize
+        """
+        cdef dict placeholder_entity_map
+        cdef str lang_code
+        cdef str script_code
+        cdef str iso_lang
+        cdef str k
+        cdef str v
+        cdef str xlated
+
+        # Unwrap if sent is a tuple or list
+        if isinstance(sent, (tuple, list)):
+            sent = sent[0]
+
+        placeholder_entity_map = self._placeholder_entity_maps.get()
+        lang_code, script_code = lang.split("_", 1)
+        iso_lang = self._flores_codes.get(lang, "hi")
+
+        # Fix for Perso-Arabic scripts
+        if script_code in ["Arab", "Aran"]:
+            sent = (
+                sent.replace(" ؟", "؟")
+                    .replace(" ۔", "۔")
+                    .replace(" ،", "،")
+                    .replace("ٮ۪", "ؠ")
+            )
+
+        # Oriya fix
+        if lang_code == "ory":
+            sent = sent.replace("ଯ଼", "ୟ")
+
+        # Restore placeholders
+        for k, v in placeholder_entity_map.items():
+            sent = sent.replace(k, v)
+
+        # Detokenize
+        if lang == "eng_Latn":
+            return self._en_detok.detokenize(sent.split(" "))
+        else:
+            xlated = self._xliterator.transliterate(sent, "hi", iso_lang)
+            return indic_detokenize.trivial_detokenize(xlated, iso_lang)
+
+    # Exposed Method: Preprocess a Batch of Sentences
+    cpdef list preprocess_batch(
+        self,
+        List[str] batch,
+        str src_lang,
+        str tgt_lang=None,
+        bint is_target=False,
+        bint visualize=False
+    ):
+        """
+        Preprocess an array of sentences (normalize, tokenize, transliterate).
+        This is exposed for external use.
+        """
+        cdef object normalizer = None
+        cdef str iso_code = self._flores_codes.get(src_lang, "hi")
+        cdef object iterator
+        cdef list results
+        cdef int i
+        cdef int n = len(batch)
+
+        if src_lang != "eng_Latn":
+            normalizer = IndicNormalizerFactory().get_normalizer(iso_code)
+
+        if visualize:
+            iterator = tqdm(batch, total=n, desc=f" | > Pre-processing {src_lang}", unit="line")
+        else:
+            iterator = batch
+
+        return [self._preprocess(s, src_lang, tgt_lang, normalizer, is_target) for s in iterator]
+
+    # Exposed Method: Postprocess a Batch of Sentences
+    cpdef list postprocess_batch(
+        self,
+        List[str] sents,
+        str lang="hin_Deva",
+        bint visualize=False
+    ):
+        """
+        Postprocess a batch of sentences:
+        Restore placeholders, fix script issues, and detokenize.
+        This is exposed for external use.
+        """
+        cdef object iterator
+        cdef list results
+        cdef int i
+        cdef int n = len(sents)
+
+        if visualize:
+            iterator = tqdm(sents, total=n, desc=f" | > Post-processing {lang}", unit="line")
+        else:
+            iterator = sents
+
+        results = [self._postprocess(s, lang) for s in iterator]
+        self._placeholder_entity_maps.queue.clear()
+        
+        return results

libs/IndicTransToolkit/IndicTransToolkit/version.py

@@ -0,0 +1 @@
+__version__ = "1.0.3"

libs/IndicTransToolkit/IndicTransToolkit/version.txt

@@ -0,0 +1 @@
+1.0.3

libs/IndicTransToolkit/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Varun Gumma.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE

libs/IndicTransToolkit/README.md

@@ -0,0 +1,97 @@
+# IndicTransToolkit
+
+## About
+The goal of this repository is to provide a simple, modular, and extendable toolkit for [IndicTrans2](https://github.com/AI4Bharat/IndicTrans2) and be compatible with the HuggingFace models released. Please refer to the `CHANGELOG.md` for latest developments.
+
+## Pre-requisites
+ - `Python 3.8+`
+ - [Indic NLP Library](https://github.com/VarunGumma/indic_nlp_library)
+ - Other requirements as listed in `requirements.txt`
+
+## Configuration
+ - Editable installation (Note, this may take a while):
+```bash 
+git clone https://github.com/VarunGumma/IndicTransToolkit
+cd IndicTransToolkit
+
+pip install --editable . --use-pep517 # required for pip >= 25.0
+
+# in case it fails, try:
+# pip install --editable . --use-pep517 --config-settings editable_mode=compat
+```
+
+## Examples
+For the training usecase, please refer [here](https://github.com/AI4Bharat/IndicTrans2/tree/main/huggingface_interface).
+
+### PreTainedTokenizer 
+```python
+import torch
+from IndicTransToolkit.processor import IndicProcessor # NOW IMPLEMENTED IN CYTHON !!
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+
+ip = IndicProcessor(inference=True)
+tokenizer = AutoTokenizer.from_pretrained("ai4bharat/indictrans2-en-indic-dist-200M", trust_remote_code=True)
+model = AutoModelForSeq2SeqLM.from_pretrained("ai4bharat/indictrans2-en-indic-dist-200M", trust_remote_code=True)
+
+sentences = [
+    "This is a test sentence.",
+    "This is another longer different test sentence.",
+    "Please send an SMS to 9876543210 and an email on [email protected] by 15th October, 2023.",
+]
+
+batch = ip.preprocess_batch(sentences, src_lang="eng_Latn", tgt_lang="hin_Deva", visualize=False) # set it to visualize=True to print a progress bar
+batch = tokenizer(batch, padding="longest", truncation=True, max_length=256, return_tensors="pt")
+
+with torch.inference_mode():
+    outputs = model.generate(**batch, num_beams=5, num_return_sequences=1, max_length=256)
+
+with tokenizer.as_target_tokenizer():
+    # This scoping is absolutely necessary, as it will instruct the tokenizer to tokenize using the target vocabulary.
+    # Failure to use this scoping will result in gibberish/unexpected predictions as the output will be de-tokenized with the source vocabulary instead.
+    outputs = tokenizer.batch_decode(outputs, skip_special_tokens=True, clean_up_tokenization_spaces=True)
+
+outputs = ip.postprocess_batch(outputs, lang="hin_Deva")
+print(outputs)
+
+>>> ['यह एक परीक्षण वाक्य है।', 'यह एक और लंबा अलग परीक्षण वाक्य है।', 'कृपया 9876543210 पर एक एस. एम. एस. भेजें और 15 अक्टूबर, 2023 तक [email protected] पर एक ईमेल भेजें।']
+```
+
+### Evaluation
+- `IndicEvaluator` is a python implementation of [compute_metrics.sh](https://github.com/AI4Bharat/IndicTrans2/blob/main/compute_metrics.sh). 
+- We have found that this python implementation gives slightly lower scores than the original `compute_metrics.sh`. So, please use this function cautiously, and feel free to raise a PR if you have found the bug/fix. 
+```python
+from IndicTransToolkit import IndicEvaluator
+
+# this method returns a dictionary with BLEU and ChrF2++ scores with appropriate signatures
+evaluator = IndicEvaluator()
+scores = evaluator.evaluate(tgt_lang=tgt_lang, preds=pred_file, refs=ref_file) 
+
+# alternatively, you can pass the list of predictions and references instead of files 
+# scores = evaluator.evaluate(tgt_lang=tgt_lang, preds=preds, refs=refs)
+```
+
+## Authors
+ - Varun Gumma ([email protected])
+ - Jay Gala ([email protected])
+ - Pranjal Agadh Chitale ([email protected])
+ - Raj Dabre ([email protected])
+
+
+## Bugs and Contribution
+Since this a bleeding-edge module, you may encounter broken stuff and import issues once in a while. In case you encounter any bugs or want additional functionalities, please feel free to raise `Issues`/`Pull Requests` or contact the authors. 
+
+
+## Citation
+If you use our codebase, or models, please do cite the following paper:
+```bibtex
+@article{
+    gala2023indictrans,
+    title={IndicTrans2: Towards High-Quality and Accessible Machine Translation Models for all 22 Scheduled Indian Languages},
+    author={Jay Gala and Pranjal A Chitale and A K Raghavan and Varun Gumma and Sumanth Doddapaneni and Aswanth Kumar M and Janki Atul Nawale and Anupama Sujatha and Ratish Puduppully and Vivek Raghavan and Pratyush Kumar and Mitesh M Khapra and Raj Dabre and Anoop Kunchukuttan},
+    journal={Transactions on Machine Learning Research},
+    issn={2835-8856},
+    year={2023},
+    url={https://openreview.net/forum?id=vfT4YuzAYA},
+    note={}
+}
+```

libs/IndicTransToolkit/build/lib.linux-x86_64-cpython-310/IndicTransToolkit/fast_processor.cpython-310-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d57d4239b3638a272e4b70292f10494ee4a0fee201a9d74c62fc35a3d263a45
+size 260304

libs/IndicTransToolkit/build/lib.linux-x86_64-cpython-310/IndicTransToolkit/processor.cpython-310-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1a101ecb27adaf367f00c90b3f8e96e7fbda3bf0560d48c368fec3750a040a4
+size 229200

libs/IndicTransToolkit/build/temp.linux-x86_64-cpython-310/IndicTransToolkit/fast_processor.o

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9e82df38b208dc0a9b468ff669c9da159c7deaabcb389fcfacd43e038504fec
+size 347184

libs/IndicTransToolkit/build/temp.linux-x86_64-cpython-310/IndicTransToolkit/processor.o

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d27c2cc00c97a89f97f7c28bc9175c5c403a0e2a372a0b39f1c5fe8609adda09
+size 303696

libs/IndicTransToolkit/pyproject.toml

@@ -0,0 +1,25 @@
+[build-system]
+requires = [
+    "setuptools>=68.2.2",
+    "wheel",
+    "Cython",
+]
+build-backend = "setuptools.build_meta"
+
+[tool.black]
+# Black configuration for code formatting
+line-length = 88
+target-version = ['py38']
+exclude = '''
+/(
+    \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | dist
+)/
+'''

libs/IndicTransToolkit/requirements.txt

@@ -0,0 +1,8 @@
+setuptools>=68.2.2
+torch
+cython
+sacremoses
+sentencepiece
+transformers
+sacrebleu
+indic-nlp-library-IT2 @ git+https://github.com/VarunGumma/indic_nlp_library.git

libs/IndicTransToolkit/setup.py

@@ -0,0 +1,61 @@
+import os
+import pathlib
+from sys import version_info, exit
+from setuptools import setup, find_packages
+from Cython.Build import cythonize
+from pkg_resources import parse_requirements
+
+def write_version_py():
+    version_txt_path = os.path.join("IndicTransToolkit", "version.txt")
+    with open(version_txt_path, "r", encoding="utf-8") as f:
+        version = f.read().strip()
+
+    version_py_path = os.path.join("IndicTransToolkit", "version.py")
+    with open(version_py_path, "w", encoding="utf-8") as f:
+        f.write(f'__version__ = "{version}"\n')
+    return version
+
+# Enforce Python >= 3.8
+if version_info < (3, 8):
+    exit("Sorry, Python >= 3.8 is required for IndicTransToolkit.")
+
+# Read long description from README
+with open("README.md", "r", errors="ignore", encoding="utf-8") as fh:
+    long_description = fh.read().strip()
+
+# Write version.py from version.txt
+version = write_version_py()
+
+# Parse requirements.txt
+req_file = pathlib.Path("requirements.txt")
+requirements = [str(req) for req in parse_requirements(req_file.open())]
+
+# Cython files to compile (adjust if your .pyx name differs)
+cython_extensions = cythonize(
+    [
+        "IndicTransToolkit/processor.pyx",
+    ],
+    compiler_directives={"language_level": "3", "boundscheck": False},
+)
+
+setup(
+    name="IndicTransToolkit",
+    version=version,
+    author="Varun Gumma",
+    author_email="[email protected]",
+    description="A simple, consistent, and extendable module for IndicTrans2 tokenizer compatible with HuggingFace models",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/VarunGumma/IndicTransToolkit",
+    packages=find_packages(),  # Auto-detect packages
+    license="MIT",
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],
+    python_requires=">=3.8",
+    install_requires=requirements,
+    ext_modules=cython_extensions,
+    zip_safe=False,
+)

libs/indic_nlp_library/.gitignore

@@ -0,0 +1,6 @@
+**/__pycache__/
+*.egg-info/
+dist/
+build/
+contrib/
+docs/

libs/indic_nlp_library/LICENSE

@@ -0,0 +1,9 @@
+MIT License
+
+Copyright (c) 2013-present Anoop Kunchukuttan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

libs/indic_nlp_library/README.md

@@ -0,0 +1,22 @@
+# Indic NLP Library
+This repository is a _de-bloated_ fork of the original [Indic NLP Library](https://github.com/anoopkunchukuttan/indic_nlp_library) and integrates [UrduHack](https://github.com/urduhack/urduhack) submodule and [Indic NLP Resources](https://github.com/anoopkunchukuttan/indic_nlp_resources) directly. This allows to work with Urdu normalization and tokenization without needing to install [urduhack](https://pypi.org/project/urduhack/) and `indic_nlp_resources` separately, which can be an issue sometimes as it is `TensorFlow` based. This repository is mainly created and mainted for [IndicTrans2](https://github.com/AI4Bharat/IndicTrans2) and [IndicTransTokenizer](https://github.com/VarunGumma/IndicTransTokenizer)
+
+For any queries, please get in touch with the original authors/maintainers of the respective libraries:
+
+- `Indic NLP Library`: [anoopkunchukuttan](https://github.com/anoopkunchukuttan)
+- `Indic NLP Resources`: [anoopkunchukuttan](https://github.com/anoopkunchukuttan) 
+- `UrduHack`: [UrduHack](https://github.com/urduhack)
+
+## Usage:
+```
+git clone https://github.com/VarunGumma/indic_nlp_library.git
+
+cd indic_nlp_library
+pip install --editable ./
+```
+
+## Updates:
+- Integrated `urduhack` directly into the repository.
+- Renamed `master` branch as `main`.
+- Integrated `indic_nlp_resources` directly into the repository.
+- _De-bloated_ the repository.

libs/indic_nlp_library/RESOURCES/script/all_script_phonetic_data.csv

@@ -0,0 +1,113 @@
+Unicode,Relative Offset,Devanagari,ITRANS,Notes,Valid Vector Representation,is_vowel,is_consonant,nukta,halanta,anusvara,misc,short_vowel,long_vowel,weak,medium,strong,independent_vowel,dependent_vowel,plosive,fricative,Central-approximant,Lateral-approximant,flap,velar,palatal,retroflex,dental,labial,aspirated,not_aspirated,voiced,unvoiced,nasal,not_nasal,front,central,back,close,close-mid,open-mid,open,rounded,not_rounded
+900,0,ऀ,ऀ,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+901,1,ँ,.n,,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0
+902,2,ं,.n,,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0
+903,3,ः,H,Should represent as pure aspiration and not as a vowel,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+904,4,ऄ,ऄ,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+905,5,अ,a,,1,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,1,0,0,1,0,0,0,1,0,1
+906,6,आ,A,,1,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,1,0,0,1,0,0,0,1,0,1
+907,7,इ,i,,1,1,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,1,1,0,0,1,0,0,0,0,1
+908,8,ई,I,,1,1,0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,1,1,0,0,1,0,0,0,0,1
+909,9,उ,u,,1,1,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,1,0,0,1,0,0,1,1,0,0,0,1,0
+90a,10,ऊ,uu,,1,1,0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,1,0,0,1,0,0,1,1,0,0,0,1,0
+90b,11,ऋ,R^i,,1,1,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,1,0,0,1,0,0,1,0,1,0,0,0,1
+90c,12,ऌ,LLi,,1,1,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,1,1,0,0,1,1,0,0,1,0,0,0,1,0
+90d,13,ऍ,ऍ,Nasalized e,1,1,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,1,1,0,0,0,0,1,1,0,1,0,1,0,0,0,1,0,0,0,1
+90e,14,ऎ,.e,,1,1,0,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,1,1,0,0,0,1,0,0,0,1
+90f,15,ए,e,,1,1,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,1,1,0,0,0,1,0,0,0,1
+910,16,ऐ,ai,,1,1,0,0,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,1,1,0,1,1,0,0,1,0,1
+911,17,ऑ,ऑ,Nasalized o,1,1,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,1,0,0,0,1,0,1,1,0,1,0,0,0,1,0,1,0,0,1,0
+912,18,ऒ,.o,,1,1,0,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,1,0,0,0,1,0,1,1,0,0,1,0,0,1,0,1,0,0,1,0
+913,19,ओ,o,,1,1,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,1,0,0,0,1,0,1,1,0,0,1,0,0,1,0,1,0,0,1,0
+914,20,औ,au,,1,1,0,0,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,1,0,0,0,1,0,1,1,0,0,1,0,0,1,1,0,0,1,1,1
+915,21,क,ka,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,0,0,0
+916,22,ख,kha,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0
+917,23,ग,ga,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+918,24,घ,gha,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0
+919,25,ङ,~Na,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0
+91a,26,च,ca,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,0,0,0
+91b,27,छ,Cha,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0
+91c,28,ज,ja,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+91d,29,झ,jha,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0
+91e,30,ञ,JNa,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0
+91f,31,ट,Ta,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,1,0,1,0,1,0,0,0,0,0,0,0,0,0
+920,32,ठ,Tha,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0
+921,33,ड,Da,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+922,34,ढ,Dha,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0
+923,35,ण,Na,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0
+924,36,त,ta,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,0,1,0,1,0,0,0,0,0,0,0,0,0
+925,37,थ,tha,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0
+926,38,द,da,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+927,39,ध,dha,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0
+928,40,न,na,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0
+929,41,ऩ,ऩ,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0
+92a,42,प,pa,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,1,0,1,0,1,0,0,0,0,0,0,0,0,0
+92b,43,फ,pha,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0
+92c,44,ब,ba,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+92d,45,भ,bha,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0
+92e,46,म,ma,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0
+92f,47,य,ya,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+930,48,र,ra,alveolar or dental- approximated by dental/ can also be considered a rhotic consonant (flap ie tap),1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+931,49,ऱ,Ra,retroflex (trill),1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+932,50,ल,la,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+933,51,ळ,La,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+934,52,ऴ,zha,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+935,53,व,va,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+936,54,श,sha,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0
+937,55,ष,Sha,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0
+938,56,स,sa,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0
+939,57,ह,ha,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0
+93a,58,ऺ,ऺ,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+93b,59,ऻ,ऻ,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+93c,60,़,़,,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+93d,61,ऽ,.a,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+93e,62,ा,A,,1,1,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,1,0,0,1,0,0,0,1,0,1
+93f,63,ि,i,,1,1,0,0,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,1,1,0,0,1,0,0,0,0,1
+940,64,ी,I,,1,1,0,0,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,1,1,0,0,1,0,0,0,0,1
+941,65,ु,u,,1,1,0,0,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,1,1,0,0,1,0,0,1,1,0,0,0,1,0
+942,66,ू,uu,,1,1,0,0,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,1,1,0,0,1,0,0,1,1,0,0,0,1,0
+943,67,ृ,R^i,,1,1,0,0,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,1,1,0,0,1,0,0,1,0,1,0,0,0,1
+944,68,ॄ,R^I,,1,1,0,0,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,1,1,0,0,1,0,0,1,0,1,0,0,0,1
+945,69,ॅ,ॅ,Nasalized e,1,1,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,1,1,0,0,0,0,1,1,0,1,0,1,0,0,0,1,0,0,0,1
+946,70,ॆ,.e,,1,1,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,1,1,0,0,0,1,0,0,0,1
+947,71,े,e,,1,1,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,1,1,0,0,0,1,0,0,0,1
+948,72,ै,ai,,1,1,0,0,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,1,1,0,1,1,0,0,1,0,1
+949,73,ॉ,ॉ,Nasalized o,1,1,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,1,0,0,0,1,0,1,1,0,1,0,0,0,1,0,1,0,0,1,0
+94a,74,ॊ,.o,,1,1,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,1,0,1,1,0,0,1,0,0,1,0,1,0,0,1,0
+94b,75,ो,o,,1,1,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,1,0,0,0,1,0,1,1,0,0,1,0,0,1,0,1,0,0,1,0
+94c,76,ौ,au,,1,1,0,0,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,1,1,0,0,1,0,0,1,1,0,0,1,1,1
+94d,77,्,,,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+94e,78,ॎ,ॎ,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+94f,79,ॏ,ॏ,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+950,80,ॐ,AUM,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+951,81,॑,॑,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+952,82,॒,॒,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+953,83,॓,॓,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+954,84,॔,॔,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+955,85,ॕ,ॕ,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+956,86,ॖ,ॖ,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+957,87,ॗ,ॗ,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+958,88,क़,क़,will be decomposed to canonical representation: consonant+nukta,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+959,89,ख़,ख़,will be decomposed to canonical representation: consonant+nukta,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+95a,90,ग़,ग़,will be decomposed to canonical representation: consonant+nukta,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+95b,91,ज़,ज़,will be decomposed to canonical representation: consonant+nukta,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+95c,92,ड़,ड़,will be decomposed to canonical representation: consonant+nukta,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+95d,93,ढ़,ढ़,will be decomposed to canonical representation: consonant+nukta,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+95e,94,फ़,फ़,will be decomposed to canonical representation: consonant+nukta,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+95f,95,य़,य़,will be decomposed to canonical representation: consonant+nukta,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+960,96,ॠ,R^I,,1,1,0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,1,0,0,1,0,0,1,0,1,0,0,0,1
+961,97,ॡ,L^I,,1,1,0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,1,1,0,0,1,1,0,0,1,0,0,0,1,0
+962,98,ॢ,LLi,,1,1,0,0,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,1,1,0,0,1,1,0,0,1,0,0,0,1,0
+963,99,ॣ,L^I,,1,1,0,0,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,1,1,0,0,1,1,0,0,1,0,0,0,1,0
+964,100,।,0,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+965,101,॥,..,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+966,102,०,0,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+967,103,१,1,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+968,104,२,2,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+969,105,३,3,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+96a,106,४,4,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+96b,107,५,5,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+96c,108,६,6,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+96d,109,७,7,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+96e,110,८,8,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+96f,111,९,9,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0

libs/indic_nlp_library/RESOURCES/script/english_arpabet_list.csv

@@ -0,0 +1,46 @@
+AO
+AA
+IY
+UW
+EH
+IH
+UH
+AH
+AX
+AE
+EY
+AY
+OW
+AW
+OY
+P
+B
+T
+D
+K
+G
+CH
+JH
+F
+V
+TH
+DH
+S
+Z
+SH
+ZH
+HH
+M
+EM
+N
+EN
+NG
+ENG
+L
+EL
+R
+DX
+NX
+Y
+W
+Q

libs/indic_nlp_library/RESOURCES/script/english_script_phonetic_data.csv

@@ -0,0 +1,47 @@
+Unicode,Relative Offset,Devanagari,ITRANS,Notes,Valid Vector Representation,is_vowel,is_consonant,nukta,halanta,anusvara,misc,short_vowel,long_vowel,weak,medium,strong,independent_vowel,dependent_vowel,plosive,fricative,Central-approximant,Lateral-approximant,flap,velar,palatal,retroflex,dental,labial,aspirated,not_aspirated,voiced,unvoiced,nasal,not_nasal,front,central,back,close,close-mid,open-mid,open,rounded,not_rounded
+900,0,,AO,,1,1,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,1,0,0,0,1,0,1,1,0,1,0,0,0,1,0,1,0,0,1,0
+901,1,,AA,,1,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,1,0,0,1,0,0,0,1,0,1
+902,2,,IY,,1,1,0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,1,1,0,0,1,0,0,0,0,1
+903,3,,UW,,1,1,0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,1,0,0,1,0,0,1,1,0,0,0,1,0
+904,4,ए,EH,,1,1,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,1,1,0,0,0,1,0,0,0,1
+905,5,इ,IH,,1,1,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,1,1,0,0,1,0,0,0,0,1
+906,6,उ,UH,,1,1,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,1,0,0,1,0,0,1,1,0,0,0,1,0
+907,7,अ,AH,,1,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,1,0,0,1,0,0,0,1,0,1
+908,8,अ,AX,,1,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,1,0,0,1,0,0,0,1,0,1
+909,9,ऍ,AE,,1,1,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,1,1,0,0,0,0,1,1,0,1,0,1,0,0,0,1,0,0,0,1
+90a,10,,EY,,1,1,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,1,1,0,0,0,1,0,0,0,1
+90b,11,ऐ,AY,,1,1,0,0,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,1,1,0,1,1,0,0,1,0,1
+90c,12,ओ,OW,,1,1,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,1,0,0,0,1,0,1,1,0,0,1,0,0,1,0,1,0,0,1,0
+90d,13,औ,AW,,1,1,0,0,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,1,0,0,0,1,0,1,1,0,0,1,0,0,1,1,0,0,1,1,1
+90e,14,,OY,,1,1,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,1,1,0,0,1,0,1,1,0,1,0,0,0,1,0,1,0,0,1,0
+90f,15,,P,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,1,0,1,0,1,0,0,0,0,0,0,0,0,0
+910,16,,B,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+911,17,,T,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,0,1,0,1,0,0,0,0,0,0,0,0,0
+912,18,,D,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+913,19,,K,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,0,0,0
+914,20,,G,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+915,21,,CH,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,0,0,0
+916,22,,JH,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+917,23,,F,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,1,0,1,0,1,0,0,0,0,0,0,0,0,0
+918,24,,V,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+919,25,,TH,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,1,0,1,0,1,0,0,0,0,0,0,0,0,0
+91a,26,,DH,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+91b,27,,S,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0
+91c,28,,Z,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+91d,29,,SH,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0
+91e,30,,ZH,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0
+91f,31,,HH,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0
+920,32,,M,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0
+921,33,,EM,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0
+922,34,,N,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0
+923,35,,EN,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0
+924,36,,NG,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0
+925,37,,ENG,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0
+926,38,,L,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+927,39,,EL,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+928,40,,R,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+929,41,,DX,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+92a,42,,NX,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0
+92b,43,,Y,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+92c,44,,W,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+92d,45,,Q,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0

libs/indic_nlp_library/RESOURCES/script/tamil_script_phonetic_data.csv

@@ -0,0 +1,113 @@
+Unicode,Relative Offset,Devanagari,ITRANS,Notes,Valid Vector Representation,is_vowel,is_consonant,nukta,halanta,anusvara,misc,short_vowel,long_vowel,weak,medium,strong,independent_vowel,dependent_vowel,plosive,fricative,Central-approximant,Lateral-approximant,flap,velar,palatal,retroflex,dental,labial,aspirated,not_aspirated,voiced,unvoiced,nasal,not_nasal,front,central,back,close,close-mid,open-mid,open,rounded,not_rounded
+900,0,ऀ,ऀ,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+901,1,ँ,.n,,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0
+902,2,ं,.n,,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0
+903,3,ः,H,Should represent as pure aspiration and not as a vowel,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+904,4,ऄ,ऄ,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+905,5,अ,a,,1,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,1,0,0,1,0,0,0,1,0,1
+906,6,आ,A,,1,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,1,0,0,1,0,0,0,1,0,1
+907,7,इ,i,,1,1,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,1,1,0,0,1,0,0,0,0,1
+908,8,ई,I,,1,1,0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,1,1,0,0,1,0,0,0,0,1
+909,9,उ,u,,1,1,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,1,0,0,1,0,0,1,1,0,0,0,1,0
+90a,10,ऊ,uu,,1,1,0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,1,0,0,1,0,0,1,1,0,0,0,1,0
+90b,11,ऋ,R^i,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+90c,12,ऌ,LLi,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+90d,13,ऍ,ऍ,Nasalized e,1,1,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,1,1,0,0,0,0,1,1,0,1,0,1,0,0,0,1,0,0,0,1
+90e,14,ऎ,.e,,1,1,0,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,1,1,0,0,0,1,0,0,0,1
+90f,15,ए,e,,1,1,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,1,1,0,0,0,1,0,0,0,1
+910,16,ऐ,ai,,1,1,0,0,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,1,1,0,1,1,0,0,1,0,1
+911,17,ऑ,ऑ,Nasalized o,1,1,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,1,0,0,0,1,0,1,1,0,1,0,0,0,1,0,1,0,0,1,0
+912,18,ऒ,.o,,1,1,0,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,1,0,0,0,1,0,1,1,0,0,1,0,0,1,0,1,0,0,1,0
+913,19,ओ,o,,1,1,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,1,0,0,0,1,0,1,1,0,0,1,0,0,1,0,1,0,0,1,0
+914,20,औ,au,,1,1,0,0,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,1,0,0,0,1,0,1,1,0,0,1,0,0,1,1,0,0,1,1,1
+915,21,क,ka,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,1,1,1,0,1,0,0,0,0,0,0,0,0,0
+916,22,ख,kha,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+917,23,ग,ga,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+918,24,घ,gha,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+919,25,ङ,~Na,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0
+91a,26,च,ca,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,1,1,1,1,0,1,0,0,0,0,0,0,0,0,0
+91b,27,छ,Cha,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+91c,28,ज,ja,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+91d,29,झ,jha,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+91e,30,ञ,JNa,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0
+91f,31,ट,Ta,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,1,1,1,1,0,1,0,0,0,0,0,0,0,0,0
+920,32,ठ,Tha,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+921,33,ड,Da,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+922,34,ढ,Dha,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+923,35,ण,Na,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0
+924,36,त,ta,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,1,1,1,1,0,1,0,0,0,0,0,0,0,0,0
+925,37,थ,tha,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+926,38,द,da,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+927,39,ध,dha,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+928,40,न,na,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0
+929,41,ऩ,ऩ,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0
+92a,42,प,pa,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,1,1,1,1,0,1,0,0,0,0,0,0,0,0,0
+92b,43,फ,pha,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+92c,44,ब,ba,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+92d,45,भ,bha,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+92e,46,म,ma,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0
+92f,47,य,ya,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+930,48,र,ra,alveolar or dental- approximated by dental/ can also be considered a rhotic consonant (flap ie tap),1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+931,49,ऱ,Ra,retroflex (trill),1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+932,50,ल,la,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+933,51,ळ,La,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+934,52,ऴ,zha,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+935,53,व,va,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0
+936,54,श,sha,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0
+937,55,ष,Sha,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0
+938,56,स,sa,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0
+939,57,ह,ha,,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0
+93a,58,ऺ,ऺ,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+93b,59,ऻ,ऻ,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+93c,60,़,़,,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+93d,61,ऽ,.a,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+93e,62,ा,A,,1,1,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,1,0,0,1,0,0,0,1,0,1
+93f,63,ि,i,,1,1,0,0,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,1,1,0,0,1,0,0,0,0,1
+940,64,ी,I,,1,1,0,0,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,1,1,0,0,1,0,0,0,0,1
+941,65,ु,u,,1,1,0,0,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,1,1,0,0,1,0,0,1,1,0,0,0,1,0
+942,66,ू,uu,,1,1,0,0,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,1,1,0,0,1,0,0,1,1,0,0,0,1,0
+943,67,ृ,R^i,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+944,68,ॄ,R^I,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+945,69,ॅ,ॅ,Nasalized e,1,1,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,1,1,0,0,0,0,1,1,0,1,0,1,0,0,0,1,0,0,0,1
+946,70,ॆ,.e,,1,1,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,1,1,0,0,0,1,0,0,0,1
+947,71,े,e,,1,1,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,1,1,0,0,0,1,0,0,0,1
+948,72,ै,ai,,1,1,0,0,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,1,1,0,1,1,0,0,1,0,1
+949,73,ॉ,ॉ,Nasalized o,1,1,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,1,0,0,0,1,0,1,1,0,1,0,0,0,1,0,1,0,0,1,0
+94a,74,ॊ,.o,,1,1,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,1,0,1,1,0,0,1,0,0,1,0,1,0,0,1,0
+94b,75,ो,o,,1,1,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,1,0,0,0,1,0,1,1,0,0,1,0,0,1,0,1,0,0,1,0
+94c,76,ौ,au,,1,1,0,0,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,1,1,0,0,1,0,0,1,1,0,0,1,1,1
+94d,77,्,,,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+94e,78,ॎ,ॎ,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+94f,79,ॏ,ॏ,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+950,80,ॐ,AUM,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+951,81,॑,॑,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+952,82,॒,॒,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+953,83,॓,॓,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+954,84,॔,॔,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+955,85,ॕ,ॕ,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+956,86,ॖ,ॖ,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+957,87,ॗ,ॗ,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+958,88,क़,क़,will be decomposed to canonical representation: consonant+nukta,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+959,89,ख़,ख़,will be decomposed to canonical representation: consonant+nukta,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+95a,90,ग़,ग़,will be decomposed to canonical representation: consonant+nukta,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+95b,91,ज़,ज़,will be decomposed to canonical representation: consonant+nukta,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+95c,92,ड़,ड़,will be decomposed to canonical representation: consonant+nukta,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+95d,93,ढ़,ढ़,will be decomposed to canonical representation: consonant+nukta,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+95e,94,फ़,फ़,will be decomposed to canonical representation: consonant+nukta,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+95f,95,य़,य़,will be decomposed to canonical representation: consonant+nukta,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+960,96,ॠ,R^I,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1
+961,97,ॡ,L^I,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0
+962,98,ॢ,LLi,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+963,99,ॣ,L^I,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+964,100,।,0,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+965,101,॥,..,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+966,102,०,0,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+967,103,१,1,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+968,104,२,2,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+969,105,३,3,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+96a,106,४,4,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+96b,107,५,5,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+96c,108,६,6,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+96d,109,७,7,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+96e,110,८,8,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+96f,111,९,9,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0

libs/indic_nlp_library/RESOURCES/transliterate/offset_itrans_map.csv

@@ -0,0 +1,129 @@
+offset_hex,devnag_char,itrans
+0x0,ऀ,ऀ
+0x1,ँ,ँ
+0x2,ं,.m
+0x3,ः,H
+0x4,ऄ,ऄ
+0x5,अ,a
+0x6,आ,aa
+0x7,इ,i
+0x8,ई,ii
+0x9,उ,u
+0xa,ऊ,uu
+0xb,ऋ,R^i
+0xc,ऌ,L^i
+0xd,ऍ,ऍ
+0xe,ऎ,.e
+0xf,ए,e
+0x10,ऐ,ai
+0x11,ऑ,ऑ
+0x12,ऒ,.o
+0x13,ओ,o
+0x14,औ,au
+0x15,क,ka
+0x16,ख,kha
+0x17,ग,ga
+0x18,घ,gha
+0x19,ङ,~Na
+0x1a,च,cha
+0x1b,छ,Cha
+0x1c,ज,ja
+0x1d,झ,jha
+0x1e,ञ,~na
+0x1f,ट,Ta
+0x20,ठ,Tha
+0x21,ड,Da
+0x22,ढ,Dha
+0x23,ण,Na
+0x24,त,ta
+0x25,थ,tha
+0x26,द,da
+0x27,ध,dha
+0x28,न,na
+0x29,ऩ,*na
+0x2a,प,pa
+0x2b,फ,pha
+0x2c,ब,ba
+0x2d,भ,bha
+0x2e,म,ma
+0x2f,य,ya
+0x30,र,ra
+0x31,ऱ,Ra
+0x32,ल,la
+0x33,ळ,lda
+0x34,ऴ,zha
+0x35,व,va
+0x36,श,sha
+0x37,ष,Sha
+0x38,स,sa
+0x39,ह,ha
+0x3a,ऺ,ऺ
+0x3b,ऻ,ऻ
+0x3c,़,़
+0x3d,ऽ,.a
+0x3e,ा,aa
+0x3f,ि,i
+0x40,ी,ii
+0x41,ु,u
+0x42,ू,uu
+0x43,ृ,R^i
+0x44,ॄ,R^I
+0x45,ॅ,ॅ
+0x46,ॆ,.e
+0x47,े,e
+0x48,ै,ai
+0x49,ॉ,ॉ
+0x4a,ॊ,.o
+0x4b,ो,o
+0x4c,ौ,au
+0x4d,्,
+0x4e,ॎ,ॎ
+0x4f,ॏ,ॏ
+0x50,ॐ,AUM
+0x51,॑,॑
+0x52,॒,॒
+0x53,॓,॓
+0x54,॔,॔
+0x55,ॕ,ॕ
+0x56,ॖ,ॖ
+0x57,ॗ,ॗ
+0x58,क़,क़
+0x59,ख़,ख़
+0x5a,ग़,ग़
+0x5b,ज़,ज़
+0x5c,ड़,ड़
+0x5d,ढ़,ढ़
+0x5e,फ़,फ़
+0x5f,य़,य़
+0x60,ॠ,R^I
+0x61,ॡ,L^I
+0x62,ॢ,L^i
+0x63,ॣ,L^I
+0x64,।,.
+0x65,॥,..
+0x66,०,0
+0x67,१,1
+0x68,२,2
+0x69,३,3
+0x6a,४,4
+0x6b,५,5
+0x6c,६,6
+0x6d,७,7
+0x6e,८,8
+0x6f,९,9
+0x70,॰,॰
+0x71,ॱ,ॱ
+0x72,ॲ,ॲ
+0x73,ॳ,ॳ
+0x74,ॴ,ॴ
+0x75,ॵ,ॵ
+0x76,ॶ,ॶ
+0x77,ॷ,ॷ
+0x78,ॸ,ॸ
+0x79,ॹ,ॹ
+0x7a,ॺ,ॺ
+0x7b,ॻ,ॻ
+0x7c,ॼ,ॼ
+0x7d,ॽ,ॽ
+0x7e,ॾ,ॾ
+0x7f,ॿ,a

libs/indic_nlp_library/indic_nlp_library_IT2.egg-info/PKG-INFO

@@ -0,0 +1,52 @@
+Metadata-Version: 2.2
+Name: indic_nlp_library_IT2
+Version: 0.0.2
+Summary: The goal of the Indic NLP Library is to build Python based libraries for common text processing and Natural Language Processing in Indian languages. This fork is specialized for IndicTrans2.
+Home-page: https://github.com/VarunGumma/indic_nlp_library
+Author: Varun Gumma
+Author-email: [email protected]
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: sphinx-argparse
+Requires-Dist: sphinx_rtd_theme
+Requires-Dist: morfessor
+Requires-Dist: pandas
+Requires-Dist: numpy
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Indic NLP Library
+This repository is a _de-bloated_ fork of the original [Indic NLP Library](https://github.com/anoopkunchukuttan/indic_nlp_library) and integrates [UrduHack](https://github.com/urduhack/urduhack) submodule and [Indic NLP Resources](https://github.com/anoopkunchukuttan/indic_nlp_resources) directly. This allows to work with Urdu normalization and tokenization without needing to install [urduhack](https://pypi.org/project/urduhack/) and `indic_nlp_resources` separately, which can be an issue sometimes as it is `TensorFlow` based. This repository is mainly created and mainted for [IndicTrans2](https://github.com/AI4Bharat/IndicTrans2) and [IndicTransTokenizer](https://github.com/VarunGumma/IndicTransTokenizer)
+
+For any queries, please get in touch with the original authors/maintainers of the respective libraries:
+
+- `Indic NLP Library`: [anoopkunchukuttan](https://github.com/anoopkunchukuttan)
+- `Indic NLP Resources`: [anoopkunchukuttan](https://github.com/anoopkunchukuttan) 
+- `UrduHack`: [UrduHack](https://github.com/urduhack)
+
+## Usage:
+```
+git clone https://github.com/VarunGumma/indic_nlp_library.git
+
+cd indic_nlp_library
+pip install --editable ./
+```
+
+## Updates:
+- Integrated `urduhack` directly into the repository.
+- Renamed `master` branch as `main`.
+- Integrated `indic_nlp_resources` directly into the repository.
+- _De-bloated_ the repository.

libs/indic_nlp_library/indic_nlp_library_IT2.egg-info/SOURCES.txt

@@ -0,0 +1,40 @@
+LICENSE
+README.md
+setup.py
+indic_nlp_library_IT2.egg-info/PKG-INFO
+indic_nlp_library_IT2.egg-info/SOURCES.txt
+indic_nlp_library_IT2.egg-info/dependency_links.txt
+indic_nlp_library_IT2.egg-info/requires.txt
+indic_nlp_library_IT2.egg-info/top_level.txt
+indicnlp/__init__.py
+indicnlp/common.py
+indicnlp/langinfo.py
+indicnlp/loader.py
+indicnlp/version.py
+indicnlp/normalize/__init__.py
+indicnlp/normalize/indic_normalize.py
+indicnlp/script/__init__.py
+indicnlp/script/english_script.py
+indicnlp/script/indic_scripts.py
+indicnlp/script/phonetic_sim.py
+indicnlp/tokenize/__init__.py
+indicnlp/tokenize/indic_detokenize.py
+indicnlp/tokenize/indic_tokenize.py
+indicnlp/tokenize/sentence_tokenize.py
+indicnlp/transliterate/__init__.py
+indicnlp/transliterate/acronym_transliterator.py
+indicnlp/transliterate/script_unifier.py
+indicnlp/transliterate/unicode_transliterate.py
+indicnlp/urduhack/__init__.py
+indicnlp/urduhack/stop_words.py
+indicnlp/urduhack/urdu_characters.py
+indicnlp/urduhack/normalization/__init__.py
+indicnlp/urduhack/normalization/character.py
+indicnlp/urduhack/normalization/regexes.py
+indicnlp/urduhack/preprocessing/__init__.py
+indicnlp/urduhack/preprocessing/character.py
+indicnlp/urduhack/preprocessing/regexes.py
+indicnlp/urduhack/preprocessing/util.py
+indicnlp/urduhack/tokenization/__init__.py
+indicnlp/urduhack/tokenization/eos.py
+indicnlp/urduhack/tokenization/tokenizer.py

libs/indic_nlp_library/indic_nlp_library_IT2.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

libs/indic_nlp_library/indic_nlp_library_IT2.egg-info/requires.txt

@@ -0,0 +1,5 @@
+sphinx-argparse
+sphinx_rtd_theme
+morfessor
+pandas
+numpy

libs/indic_nlp_library/indic_nlp_library_IT2.egg-info/top_level.txt

@@ -0,0 +1 @@
+indicnlp