Harveen | Adding code

IndicTrans_training.ipynb

@@ -0,0 +1,752 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "view-in-github"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/gowtham1997/indicTrans-1/blob/main/IndicTrans_training.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 1,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "FdyHSnoj7Iun",
+        "outputId": "d0624c60-68c4-470f-9ade-c517e3296044"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "/content/training\n"
+          ]
+        }
+      ],
+      "source": [
+        "# create a seperate folder to store everything\n",
+        "!mkdir training\n",
+        "%cd training"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 2,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "y55OfxBz8QeP",
+        "outputId": "6d0ab016-0f96-4671-ddee-f06b50506dcd"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Cloning into 'indicTrans'...\n",
+            "remote: Enumerating objects: 432, done.\u001b[K\n",
+            "remote: Counting objects: 100% (139/139), done.\u001b[K\n",
+            "remote: Compressing objects: 100% (34/34), done.\u001b[K\n",
+            "remote: Total 432 (delta 122), reused 105 (delta 105), pack-reused 293\u001b[K\n",
+            "Receiving objects: 100% (432/432), 1.43 MiB | 14.11 MiB/s, done.\n",
+            "Resolving deltas: 100% (248/248), done.\n",
+            "/content/training/indicTrans\n",
+            "Cloning into 'indic_nlp_library'...\n",
+            "remote: Enumerating objects: 1325, done.\u001b[K\n",
+            "remote: Counting objects: 100% (147/147), done.\u001b[K\n",
+            "remote: Compressing objects: 100% (103/103), done.\u001b[K\n",
+            "remote: Total 1325 (delta 84), reused 89 (delta 41), pack-reused 1178\u001b[K\n",
+            "Receiving objects: 100% (1325/1325), 9.57 MiB | 10.51 MiB/s, done.\n",
+            "Resolving deltas: 100% (688/688), done.\n",
+            "Cloning into 'indic_nlp_resources'...\n",
+            "remote: Enumerating objects: 133, done.\u001b[K\n",
+            "remote: Counting objects: 100% (7/7), done.\u001b[K\n",
+            "remote: Compressing objects: 100% (7/7), done.\u001b[K\n",
+            "remote: Total 133 (delta 0), reused 2 (delta 0), pack-reused 126\u001b[K\n",
+            "Receiving objects: 100% (133/133), 149.77 MiB | 34.05 MiB/s, done.\n",
+            "Resolving deltas: 100% (51/51), done.\n",
+            "Checking out files: 100% (28/28), done.\n",
+            "Cloning into 'subword-nmt'...\n",
+            "remote: Enumerating objects: 580, done.\u001b[K\n",
+            "remote: Counting objects: 100% (4/4), done.\u001b[K\n",
+            "remote: Compressing objects: 100% (4/4), done.\u001b[K\n",
+            "remote: Total 580 (delta 0), reused 1 (delta 0), pack-reused 576\u001b[K\n",
+            "Receiving objects: 100% (580/580), 237.41 KiB | 5.28 MiB/s, done.\n",
+            "Resolving deltas: 100% (349/349), done.\n",
+            "/content/training\n"
+          ]
+        }
+      ],
+      "source": [
+        "# clone the repo for running finetuning\n",
+        "!git clone https://github.com/AI4Bharat/indicTrans.git\n",
+        "%cd indicTrans\n",
+        "# clone requirements repositories\n",
+        "!git clone https://github.com/anoopkunchukuttan/indic_nlp_library.git\n",
+        "!git clone https://github.com/anoopkunchukuttan/indic_nlp_resources.git\n",
+        "!git clone https://github.com/rsennrich/subword-nmt.git\n",
+        "%cd .."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 3,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "ziWWl-1a8SMw",
+        "outputId": "d7908a62-9573-4693-e7cb-44aeeebaaa15"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Reading package lists... Done\n",
+            "Building dependency tree       \n",
+            "Reading state information... Done\n",
+            "The following NEW packages will be installed:\n",
+            "  tree\n",
+            "0 upgraded, 1 newly installed, 0 to remove and 39 not upgraded.\n",
+            "Need to get 40.7 kB of archives.\n",
+            "After this operation, 105 kB of additional disk space will be used.\n",
+            "Get:1 http://archive.ubuntu.com/ubuntu bionic/universe amd64 tree amd64 1.7.0-5 [40.7 kB]\n",
+            "Fetched 40.7 kB in 0s (133 kB/s)\n",
+            "debconf: unable to initialize frontend: Dialog\n",
+            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76, <> line 1.)\n",
+            "debconf: falling back to frontend: Readline\n",
+            "debconf: unable to initialize frontend: Readline\n",
+            "debconf: (This frontend requires a controlling tty.)\n",
+            "debconf: falling back to frontend: Teletype\n",
+            "dpkg-preconfigure: unable to re-open stdin: \n",
+            "Selecting previously unselected package tree.\n",
+            "(Reading database ... 160772 files and directories currently installed.)\n",
+            "Preparing to unpack .../tree_1.7.0-5_amd64.deb ...\n",
+            "Unpacking tree (1.7.0-5) ...\n",
+            "Setting up tree (1.7.0-5) ...\n",
+            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
+            "Collecting sacremoses\n",
+            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n",
+            "\u001b[K     |████████████████████████████████| 901kB 4.0MB/s \n",
+            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (1.1.5)\n",
+            "Collecting mock\n",
+            "  Downloading https://files.pythonhosted.org/packages/5c/03/b7e605db4a57c0f6fba744b11ef3ddf4ddebcada35022927a2b5fc623fdf/mock-4.0.3-py3-none-any.whl\n",
+            "Collecting sacrebleu\n",
+            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7e/57/0c7ca4e31a126189dab99c19951910bd081dea5bbd25f24b77107750eae7/sacrebleu-1.5.1-py3-none-any.whl (54kB)\n",
+            "\u001b[K     |████████████████████████████████| 61kB 7.4MB/s \n",
+            "\u001b[?25hCollecting tensorboardX\n",
+            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/42/36/2b147652c40c3a858efa0afbf7b8236fae968e88ff530511a4cfa299a506/tensorboardX-2.3-py2.py3-none-any.whl (124kB)\n",
+            "\u001b[K     |████████████████████████████████| 133kB 24.0MB/s \n",
+            "\u001b[?25hRequirement already satisfied: pyarrow in /usr/local/lib/python3.7/dist-packages (3.0.0)\n",
+            "Collecting indic-nlp-library\n",
+            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/84/d4/495bb43b88a2a6d04b09c29fc5115f24872af74cd8317fe84026abd4ddb1/indic_nlp_library-0.81-py3-none-any.whl (40kB)\n",
+            "\u001b[K     |████████████████████████████████| 40kB 5.4MB/s \n",
+            "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses) (1.15.0)\n",
+            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from sacremoses) (2019.12.20)\n",
+            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses) (7.1.2)\n",
+            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from sacremoses) (4.41.1)\n",
+            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses) (1.0.1)\n",
+            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas) (2018.9)\n",
+            "Requirement already satisfied: numpy>=1.15.4 in /usr/local/lib/python3.7/dist-packages (from pandas) (1.19.5)\n",
+            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas) (2.8.1)\n",
+            "Collecting portalocker==2.0.0\n",
+            "  Downloading https://files.pythonhosted.org/packages/89/a6/3814b7107e0788040870e8825eebf214d72166adf656ba7d4bf14759a06a/portalocker-2.0.0-py2.py3-none-any.whl\n",
+            "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorboardX) (3.12.4)\n",
+            "Collecting morfessor\n",
+            "  Downloading https://files.pythonhosted.org/packages/39/e6/7afea30be2ee4d29ce9de0fa53acbb033163615f849515c0b1956ad074ee/Morfessor-2.0.6-py3-none-any.whl\n",
+            "Collecting sphinx-argparse\n",
+            "  Downloading https://files.pythonhosted.org/packages/06/2b/dfad6a1831c3aeeae25d8d3d417224684befbf45e10c7f2141631616a6ed/sphinx-argparse-0.2.5.tar.gz\n",
+            "Collecting sphinx-rtd-theme\n",
+            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ac/24/2475e8f83519b54b2148d4a56eb1111f9cec630d088c3ffc214492c12107/sphinx_rtd_theme-0.5.2-py2.py3-none-any.whl (9.1MB)\n",
+            "\u001b[K     |████████████████████████████████| 9.2MB 21.7MB/s \n",
+            "\u001b[?25hRequirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from protobuf>=3.8.0->tensorboardX) (57.0.0)\n",
+            "Requirement already satisfied: sphinx>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from sphinx-argparse->indic-nlp-library) (1.8.5)\n",
+            "Collecting docutils<0.17\n",
+            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/81/44/8a15e45ffa96e6cf82956dd8d7af9e666357e16b0d93b253903475ee947f/docutils-0.16-py2.py3-none-any.whl (548kB)\n",
+            "\u001b[K     |████████████████████████████████| 552kB 38.5MB/s \n",
+            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (20.9)\n",
+            "Requirement already satisfied: imagesize in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (1.2.0)\n",
+            "Requirement already satisfied: requests>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.23.0)\n",
+            "Requirement already satisfied: sphinxcontrib-websupport in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (1.2.4)\n",
+            "Requirement already satisfied: Pygments>=2.0 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.6.1)\n",
+            "Requirement already satisfied: snowballstemmer>=1.1 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.1.0)\n",
+            "Requirement already satisfied: babel!=2.0,>=1.3 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.9.1)\n",
+            "Requirement already satisfied: alabaster<0.8,>=0.7 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (0.7.12)\n",
+            "Requirement already satisfied: Jinja2>=2.3 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.11.3)\n",
+            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.4.7)\n",
+            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (1.24.3)\n",
+            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.10)\n",
+            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (3.0.4)\n",
+            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2021.5.30)\n",
+            "Requirement already satisfied: sphinxcontrib-serializinghtml in /usr/local/lib/python3.7/dist-packages (from sphinxcontrib-websupport->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (1.1.5)\n",
+            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from Jinja2>=2.3->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.0.1)\n",
+            "Building wheels for collected packages: sphinx-argparse\n",
+            "  Building wheel for sphinx-argparse (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+            "  Created wheel for sphinx-argparse: filename=sphinx_argparse-0.2.5-cp37-none-any.whl size=11552 sha256=0f3830a0bf7a6cfa99000091da945e9dd814b2f1e1f9ca5d773f99aaa0d3a4a5\n",
+            "  Stored in directory: /root/.cache/pip/wheels/2a/18/1b/4990a1859da4edc77ab312bc2986c08d2733fb5713d06e44f5\n",
+            "Successfully built sphinx-argparse\n",
+            "\u001b[31mERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.\u001b[0m\n",
+            "Installing collected packages: sacremoses, mock, portalocker, sacrebleu, tensorboardX, morfessor, sphinx-argparse, docutils, sphinx-rtd-theme, indic-nlp-library\n",
+            "  Found existing installation: docutils 0.17.1\n",
+            "    Uninstalling docutils-0.17.1:\n",
+            "      Successfully uninstalled docutils-0.17.1\n",
+            "Successfully installed docutils-0.16 indic-nlp-library-0.81 mock-4.0.3 morfessor-2.0.6 portalocker-2.0.0 sacrebleu-1.5.1 sacremoses-0.0.45 sphinx-argparse-0.2.5 sphinx-rtd-theme-0.5.2 tensorboardX-2.3\n",
+            "Cloning into 'fairseq'...\n",
+            "remote: Enumerating objects: 28410, done.\u001b[K\n",
+            "remote: Counting objects: 100% (229/229), done.\u001b[K\n",
+            "remote: Compressing objects: 100% (127/127), done.\u001b[K\n",
+            "remote: Total 28410 (delta 114), reused 187 (delta 99), pack-reused 28181\u001b[K\n",
+            "Receiving objects: 100% (28410/28410), 11.96 MiB | 24.45 MiB/s, done.\n",
+            "Resolving deltas: 100% (21310/21310), done.\n",
+            "/content/training/fairseq\n",
+            "Obtaining file:///content/training/fairseq\n",
+            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
+            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
+            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
+            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
+            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+f887152) (2019.12.20)\n",
+            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+f887152) (4.41.1)\n",
+            "Collecting omegaconf<2.1\n",
+            "  Downloading https://files.pythonhosted.org/packages/d0/eb/9d63ce09dd8aa85767c65668d5414958ea29648a0eec80a4a7d311ec2684/omegaconf-2.0.6-py3-none-any.whl\n",
+            "Requirement already satisfied: numpy; python_version >= \"3.7\" in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+f887152) (1.19.5)\n",
+            "Requirement already satisfied: sacrebleu>=1.4.12 in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+f887152) (1.5.1)\n",
+            "Requirement already satisfied: cython in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+f887152) (0.29.23)\n",
+            "Collecting hydra-core<1.1\n",
+            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/52/e3/fbd70dd0d3ce4d1d75c22d56c0c9f895cfa7ed6587a9ffb821d6812d6a60/hydra_core-1.0.6-py3-none-any.whl (123kB)\n",
+            "\u001b[K     |████████████████████████████████| 133kB 4.7MB/s \n",
+            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+f887152) (1.9.0+cu102)\n",
+            "Requirement already satisfied: cffi in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+f887152) (1.14.5)\n",
+            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from omegaconf<2.1->fairseq==1.0.0a0+f887152) (3.7.4.3)\n",
+            "Collecting PyYAML>=5.1.*\n",
+            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7a/a5/393c087efdc78091afa2af9f1378762f9821c9c1d7a22c5753fb5ac5f97a/PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636kB)\n",
+            "\u001b[K     |████████████████████████████████| 645kB 32.4MB/s \n",
+            "\u001b[?25hRequirement already satisfied: portalocker==2.0.0 in /usr/local/lib/python3.7/dist-packages (from sacrebleu>=1.4.12->fairseq==1.0.0a0+f887152) (2.0.0)\n",
+            "Requirement already satisfied: importlib-resources; python_version < \"3.9\" in /usr/local/lib/python3.7/dist-packages (from hydra-core<1.1->fairseq==1.0.0a0+f887152) (5.1.4)\n",
+            "Collecting antlr4-python3-runtime==4.8\n",
+            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/56/02/789a0bddf9c9b31b14c3e79ec22b9656185a803dc31c15f006f9855ece0d/antlr4-python3-runtime-4.8.tar.gz (112kB)\n",
+            "\u001b[K     |████████████████████████████████| 112kB 53.0MB/s \n",
+            "\u001b[?25hRequirement already satisfied: pycparser in /usr/local/lib/python3.7/dist-packages (from cffi->fairseq==1.0.0a0+f887152) (2.20)\n",
+            "Requirement already satisfied: zipp>=3.1.0; python_version < \"3.10\" in /usr/local/lib/python3.7/dist-packages (from importlib-resources; python_version < \"3.9\"->hydra-core<1.1->fairseq==1.0.0a0+f887152) (3.4.1)\n",
+            "Building wheels for collected packages: antlr4-python3-runtime\n",
+            "  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+            "  Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.8-cp37-none-any.whl size=141231 sha256=52f59bfe6322a04598da6960d2d5675a581273a45e4391e04cf1240c97346019\n",
+            "  Stored in directory: /root/.cache/pip/wheels/e3/e2/fa/b78480b448b8579ddf393bebd3f47ee23aa84c89b6a78285c8\n",
+            "Successfully built antlr4-python3-runtime\n",
+            "Installing collected packages: PyYAML, omegaconf, antlr4-python3-runtime, hydra-core, fairseq\n",
+            "  Found existing installation: PyYAML 3.13\n",
+            "    Uninstalling PyYAML-3.13:\n",
+            "      Successfully uninstalled PyYAML-3.13\n",
+            "  Running setup.py develop for fairseq\n",
+            "Successfully installed PyYAML-5.4.1 antlr4-python3-runtime-4.8 fairseq hydra-core-1.0.6 omegaconf-2.0.6\n",
+            "/content/training\n"
+          ]
+        }
+      ],
+      "source": [
+        "! sudo apt install tree\n",
+        "\n",
+        "# Install the necessary libraries\n",
+        "!pip install sacremoses pandas mock sacrebleu tensorboardX pyarrow indic-nlp-library\n",
+        "# Install fairseq from source\n",
+        "!git clone https://github.com/pytorch/fairseq.git\n",
+        "%cd fairseq\n",
+        "# !git checkout da9eaba12d82b9bfc1442f0e2c6fc1b895f4d35d\n",
+        "!pip install --editable ./\n",
+        "%cd .."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 1,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "tmfGYkd58UiO",
+        "outputId": "3b83bcf6-bbbf-4e49-c2bb-7d0fb999297d"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "^C\n"
+          ]
+        },
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "--2021-12-18 21:31:57--  https://storage.googleapis.com/samanantar-public/benchmarks.zip\n",
+            "Resolving storage.googleapis.com (storage.googleapis.com)... 172.217.160.144, 216.58.196.176, 142.250.71.16, ...\n",
+            "Connecting to storage.googleapis.com (storage.googleapis.com)|172.217.160.144|:443... connected.\n",
+            "HTTP request sent, awaiting response... 200 OK\n",
+            "Length: 7301872 (7.0M) [application/zip]\n",
+            "Saving to: 'benchmarks.zip'\n",
+            "\n",
+            "     0K .......... .......... .......... .......... ..........  0%  774K 9s\n",
+            "    50K .......... .......... .......... .......... ..........  1% 2.10M 6s\n",
+            "   100K .......... .......... .......... .......... ..........  2% 2.46M 5s\n",
+            "   150K .......... .......... .......... .......... ..........  2% 2.68M 4s\n",
+            "   200K .......... .......... .......... .......... ..........  3% 1.44M 4s\n",
+            "   250K .......... .......... .......... .......... ..........  4% 2.48M 4s\n",
+            "   300K .......... .......... .......... .......... ..........  4% 3.41M 4s\n",
+            "   350K .......... .......... .......... .......... ..........  5% 2.22M 4s\n",
+            "   400K .......... .......... .......... .......... ..........  6% 1.20M 4s\n",
+            "   450K .......... .......... .......... .......... ..........  7% 2.65M 4s\n",
+            "   500K .......... .......... .......... .......... ..........  7% 2.97M 3s\n",
+            "   550K .......... .......... .......... .......... ..........  8%  887K 4s\n",
+            "   600K .......... .......... .......... .......... ..........  9% 2.90M 4s\n",
+            "   650K .......... .......... .......... .......... ..........  9% 2.76M 4s\n",
+            "   700K .......... .......... .......... .......... .......... 10%  980K 4s\n",
+            "   750K .......... .......... .......... .......... .......... 11% 2.55M 4s\n",
+            "   800K .......... .......... .......... .......... .......... 11% 2.86M 3s\n",
+            "   850K .......... .......... .......... .......... .......... 12% 3.04M 3s\n",
+            "   900K .......... .......... .......... .......... .......... 13% 1.01M 3s\n",
+            "   950K .......... .......... .......... .......... .......... 14% 3.35M 3s\n",
+            "  1000K .......... .......... .......... .......... .......... 14% 5.04M 3s\n",
+            "  1050K .......... .......... .......... .......... .......... 15% 14.5M 3s\n",
+            "  1100K .......... .......... .......... .......... .......... 16% 1.01M 3s\n",
+            "  1150K .......... .......... .......... .......... .......... 16% 4.48M 3s\n",
+            "  1200K .......... .......... .......... .......... .......... 17% 4.34M 3s\n",
+            "  1250K .......... .......... .......... .......... .......... 18% 2.90M 3s\n",
+            "  1300K .......... .......... .......... .......... .......... 18% 1.14M 3s\n",
+            "  1350K .......... .......... .......... .......... .......... 19% 3.00M 3s\n",
+            "  1400K .......... .......... .......... .......... .......... 20% 5.09M 3s\n",
+            "  1450K .......... .......... .......... .......... .......... 21% 1.91M 3s\n",
+            "  1500K .......... .......... .......... .......... .......... 21% 7.70M 3s\n",
+            "  1550K .......... .......... .......... .......... .......... 22% 1.27M 3s\n",
+            "  1600K .......... .......... .......... .......... .......... 23% 3.06M 3s\n",
+            "  1650K .......... .......... .......... .......... .......... 23% 4.11M 3s\n",
+            "  1700K .......... .......... .......... .......... .......... 24% 3.34M 3s\n",
+            "  1750K .......... .......... .......... .......... .......... 25% 4.13M 2s\n",
+            "  1800K .......... .......... .......... .......... .......... 25% 7.95M 2s\n",
+            "  1850K .......... .......... .......... .......... .......... 26% 3.69M 2s\n",
+            "  1900K .......... .......... .......... .......... .......... 27% 4.00M 2s\n",
+            "  1950K .......... .......... .......... .......... .......... 28% 3.50M 2s\n",
+            "  2000K .......... .......... .......... .......... .......... 28% 4.04M 2s\n",
+            "  2050K .......... .......... .......... .......... .......... 29% 3.31M 2s\n",
+            "  2100K .......... .......... .......... .......... .......... 30% 2.49M 2s\n",
+            "  2150K .......... .......... .......... .......... .......... 30% 4.19M 2s\n",
+            "  2200K .......... .......... .......... .......... .......... 31% 5.18M 2s\n",
+            "  2250K .......... .......... .......... .......... .......... 32% 9.49M 2s\n",
+            "  2300K .......... .......... .......... .......... .......... 32% 8.67M 2s\n",
+            "  2350K .......... .......... .......... .......... .......... 33% 4.88M 2s\n",
+            "  2400K .......... .......... .......... .......... .......... 34% 4.56M 2s\n",
+            "  2450K .......... .......... .......... .......... .......... 35% 4.94M 2s\n",
+            "  2500K .......... .......... .......... .......... .......... 35% 4.38M 2s\n",
+            "  2550K .......... .......... .......... .......... .......... 36% 3.78M 2s\n",
+            "  2600K .......... .......... .......... .......... .......... 37% 4.95M 2s\n",
+            "  2650K .......... .......... .......... .......... .......... 37% 5.50M 2s\n",
+            "  2700K .......... .......... .......... .......... .......... 38% 5.23M 2s\n",
+            "  2750K .......... .......... .......... .......... .......... 39% 3.77M 2s\n",
+            "  2800K .......... .......... .......... .......... .......... 39% 10.7M 2s\n",
+            "  2850K .......... .......... .......... .......... .......... 40% 7.16M 2s\n",
+            "  2900K .......... .......... .......... .......... .......... 41% 5.36M 2s\n",
+            "  2950K .......... .......... .......... .......... .......... 42% 6.80M 1s\n",
+            "  3000K .......... .......... .......... .......... .......... 42% 6.57M 1s\n",
+            "  3050K .......... .......... .......... .......... .......... 43% 7.21M 1s\n",
+            "  3100K .......... .......... .......... .......... .......... 44% 6.66M 1s\n",
+            "  3150K .......... .......... .......... .......... .......... 44% 6.42M 1s\n",
+            "  3200K .......... .......... .......... .......... .......... 45% 8.02M 1s\n",
+            "  3250K .......... .......... .......... .......... .......... 46% 5.96M 1s\n",
+            "  3300K .......... .......... .......... .......... .......... 46% 5.13M 1s\n",
+            "  3350K .......... .......... .......... .......... .......... 47% 5.19M 1s\n",
+            "  3400K .......... .......... .......... .......... .......... 48% 7.64M 1s\n",
+            "  3450K .......... .......... .......... .......... .......... 49% 6.11M 1s\n",
+            "  3500K .......... .......... .......... .......... .......... 49% 4.01M 1s\n",
+            "  3550K .......... .......... .......... .......... .......... 50% 4.52M 1s\n",
+            "  3600K .......... .......... .......... .......... .......... 51% 6.72M 1s\n",
+            "  3650K .......... .......... .......... .......... .......... 51% 5.45M 1s\n",
+            "  3700K .......... .......... .......... .......... .......... 52% 4.37M 1s\n",
+            "  3750K .......... .......... .......... .......... .......... 53% 5.39M 1s\n",
+            "  3800K .......... .......... .......... .......... .......... 53% 7.40M 1s\n",
+            "  3850K .......... .......... .......... .......... .......... 54% 6.70M 1s\n",
+            "  3900K .......... .......... .......... .......... .......... 55% 5.14M 1s\n",
+            "  3950K .......... .......... .......... .......... .......... 56% 5.02M 1s\n",
+            "  4000K .......... .......... .......... .......... .......... 56% 6.70M 1s\n",
+            "  4050K .......... .......... .......... .......... .......... 57% 6.76M 1s\n",
+            "  4100K .......... .......... .......... .......... .......... 58% 2.52M 1s\n",
+            "  4150K .......... .......... .......... .......... .......... 58%  887K 1s\n",
+            "  4200K .......... .......... .......... .......... .......... 59% 9.25M 1s\n",
+            "  4250K .......... .......... .......... .......... .......... 60% 1.27M 1s\n",
+            "  4300K .......... .......... .......... .......... .......... 61% 5.72M 1s\n",
+            "  4350K .......... .......... .......... .......... .......... 61% 4.48M 1s\n",
+            "  4400K .......... .......... .......... .......... .......... 62% 5.20M 1s\n",
+            "  4450K .......... .......... .......... .......... .......... 63% 6.21M 1s\n",
+            "  4500K .......... .......... .......... .......... .......... 63% 7.94M 1s\n",
+            "  4550K .......... .......... .......... .......... .......... 64% 4.76M 1s\n",
+            "  4600K .......... .......... .......... .......... .......... 65% 4.74M 1s\n",
+            "  4650K .......... .......... .......... .......... .......... 65% 6.94M 1s\n",
+            "  4700K .......... .......... .......... .......... .......... 66% 5.62M 1s\n",
+            "  4750K .......... .......... .......... .......... .......... 67% 4.44M 1s\n",
+            "  4800K .......... .......... .......... .......... .......... 68% 6.02M 1s\n",
+            "  4850K .......... .......... .......... .......... .......... 68% 6.61M 1s\n",
+            "  4900K .......... .......... .......... .......... .......... 69% 3.04M 1s\n",
+            "  4950K .......... .......... .......... .......... .......... 70% 5.34M 1s\n",
+            "  5000K .......... .......... .......... .......... .......... 70% 3.03M 1s\n",
+            "  5050K .......... .......... .......... .......... .......... 71% 19.8M 1s\n",
+            "  5100K .......... .......... .......... .......... .......... 72% 6.17M 1s\n",
+            "  5150K .......... .......... .......... .......... .......... 72% 5.58M 1s\n",
+            "  5200K .......... .......... .......... .......... .......... 73% 7.38M 1s\n",
+            "  5250K .......... .......... .......... .......... .......... 74% 7.11M 1s\n",
+            "  5300K .......... .......... .......... .......... .......... 75% 6.24M 1s\n",
+            "  5350K .......... .......... .......... .......... .......... 75% 4.62M 1s\n",
+            "  5400K .......... .......... .......... .......... .......... 76% 7.64M 0s\n",
+            "  5450K .......... .......... .......... .......... .......... 77% 6.06M 0s\n",
+            "  5500K .......... .......... .......... .......... .......... 77% 5.56M 0s\n",
+            "  5550K .......... .......... .......... .......... .......... 78% 2.96M 0s\n",
+            "  5600K .......... .......... .......... .......... .......... 79% 6.17M 0s\n",
+            "  5650K .......... .......... .......... .......... .......... 79% 9.58M 0s\n",
+            "  5700K .......... .......... .......... .......... .......... 80% 2.58M 0s\n",
+            "  5750K .......... .......... .......... .......... .......... 81% 4.23M 0s\n",
+            "  5800K .......... .......... .......... .......... .......... 82% 5.70M 0s\n",
+            "  5850K .......... .......... .......... .......... .......... 82% 4.72M 0s\n",
+            "  5900K .......... .......... .......... .......... .......... 83% 6.52M 0s\n",
+            "  5950K .......... .......... .......... .......... .......... 84% 5.86M 0s\n",
+            "  6000K .......... .......... .......... .......... .......... 84% 5.22M 0s\n",
+            "  6050K .......... .......... .......... .......... .......... 85% 5.50M 0s\n",
+            "  6100K .......... .......... .......... .......... .......... 86% 6.29M 0s\n",
+            "  6150K .......... .......... .......... .......... .......... 86% 6.93M 0s\n",
+            "  6200K .......... .......... .......... .......... .......... 87% 5.50M 0s\n",
+            "  6250K .......... .......... .......... .......... .......... 88% 5.82M 0s\n",
+            "  6300K .......... .......... .......... .......... .......... 89% 6.76M 0s\n",
+            "  6350K .......... .......... .......... .......... .......... 89% 3.73M 0s\n",
+            "  6400K .......... .......... .......... .......... .......... 90% 5.98M 0s\n",
+            "  6450K .......... .......... .......... .......... .......... 91% 5.78M 0s\n",
+            "  6500K .......... .......... .......... .......... .......... 91% 5.60M 0s\n",
+            "  6550K .......... .......... .......... .......... .......... 92% 4.84M 0s\n",
+            "  6600K .......... .......... .......... .......... .......... 93% 7.25M 0s\n",
+            "  6650K .......... .......... .......... .......... .......... 93% 2.60M 0s\n",
+            "  6700K .......... .......... .......... .......... .......... 94% 6.02M 0s\n",
+            "  6750K .......... .......... .......... .......... .......... 95% 6.57M 0s\n",
+            "  6800K .......... .......... .......... .......... .......... 96% 8.30M 0s\n",
+            "  6850K .......... .......... .......... .......... .......... 96% 14.4M 0s\n",
+            "  6900K .......... .......... .......... .......... .......... 97% 4.58M 0s\n",
+            "  6950K .......... .......... .......... .......... .......... 98% 3.31M 0s\n",
+            "  7000K .......... .......... .......... .......... .......... 98% 6.88M 0s\n",
+            "  7050K .......... .......... .......... .......... .......... 99% 4.40M 0s\n",
+            "  7100K .......... .......... ..........                      100% 15.1M=1.9s\n",
+            "\n",
+            "2021-12-18 21:32:01 (3.64 MB/s) - 'benchmarks.zip' saved [7301872/7301872]\n",
+            "\n"
+          ]
+        },
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Archive:  samanatar-en-indic-v0.2.zip\n"
+          ]
+        },
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "  End-of-central-directory signature not found.  Either this file is not\n",
+            "  a zipfile, or it constitutes one disk of a multi-part archive.  In the\n",
+            "  latter case the central directory and zipfile comment will be found on\n",
+            "  the last disk(s) of this archive.\n",
+            "unzip:  cannot find zipfile directory in one of samanatar-en-indic-v0.2.zip or\n",
+            "        samanatar-en-indic-v0.2.zip.zip, and cannot find samanatar-en-indic-v0.2.zip.ZIP, period.\n"
+          ]
+        },
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Archive:  benchmarks.zip\n",
+            "   creating: benchmarks/\n",
+            "   creating: benchmarks/pmi/\n",
+            "   creating: benchmarks/pmi/en-as/\n",
+            "  inflating: benchmarks/pmi/en-as/dev.as  \n",
+            "  inflating: benchmarks/pmi/en-as/dev.en  \n",
+            "  inflating: benchmarks/pmi/en-as/test.as  \n",
+            "  inflating: benchmarks/pmi/en-as/test.en  \n",
+            "   creating: benchmarks/wat2021-devtest/\n",
+            "  inflating: benchmarks/wat2021-devtest/dev.gu  \n",
+            "  inflating: benchmarks/wat2021-devtest/dev.en  \n",
+            "  inflating: benchmarks/wat2021-devtest/test.bn  \n",
+            "  inflating: benchmarks/wat2021-devtest/dev.bn  \n",
+            "  inflating: benchmarks/wat2021-devtest/test.hi  \n",
+            "  inflating: benchmarks/wat2021-devtest/dev.kn  \n",
+            "  inflating: benchmarks/wat2021-devtest/dev.ta  \n",
+            "  inflating: benchmarks/wat2021-devtest/test.pa  \n",
+            "  inflating: benchmarks/wat2021-devtest/test.en  \n",
+            "  inflating: benchmarks/wat2021-devtest/test.mr  \n",
+            "  inflating: benchmarks/wat2021-devtest/test.kn  \n",
+            "  inflating: benchmarks/wat2021-devtest/dev.ml  \n",
+            "  inflating: benchmarks/wat2021-devtest/test.ta  \n",
+            "  inflating: benchmarks/wat2021-devtest/test.gu  \n",
+            "  inflating: benchmarks/wat2021-devtest/dev.or  \n",
+            "  inflating: benchmarks/wat2021-devtest/test.or  \n",
+            "  inflating: benchmarks/wat2021-devtest/test.te  \n",
+            "  inflating: benchmarks/wat2021-devtest/dev.mr  \n",
+            "  inflating: benchmarks/wat2021-devtest/test.ml  \n",
+            "  inflating: benchmarks/wat2021-devtest/dev.pa  \n",
+            "  inflating: benchmarks/wat2021-devtest/dev.te  \n",
+            "  inflating: benchmarks/wat2021-devtest/dev.hi  \n",
+            "   creating: benchmarks/wat2020-devtest/\n",
+            "   creating: benchmarks/wat2020-devtest/en-bn/\n",
+            "  inflating: benchmarks/wat2020-devtest/en-bn/dev.en  \n",
+            "  inflating: benchmarks/wat2020-devtest/en-bn/test.bn  \n",
+            "  inflating: benchmarks/wat2020-devtest/en-bn/dev.bn  \n",
+            "  inflating: benchmarks/wat2020-devtest/en-bn/test.en  \n",
+            "   creating: benchmarks/wat2020-devtest/en-ta/\n",
+            "  inflating: benchmarks/wat2020-devtest/en-ta/dev.en  \n",
+            "  inflating: benchmarks/wat2020-devtest/en-ta/dev.ta  \n",
+            "  inflating: benchmarks/wat2020-devtest/en-ta/test.en  \n",
+            "  inflating: benchmarks/wat2020-devtest/en-ta/test.ta  \n",
+            "   creating: benchmarks/wat2020-devtest/en-mr/\n",
+            "  inflating: benchmarks/wat2020-devtest/en-mr/dev.en  \n",
+            "  inflating: benchmarks/wat2020-devtest/en-mr/test.en  \n",
+            "  inflating: benchmarks/wat2020-devtest/en-mr/test.mr  \n",
+            "  inflating: benchmarks/wat2020-devtest/en-mr/dev.mr  \n",
+            "   creating: benchmarks/wat2020-devtest/en-te/\n",
+            "  inflating: benchmarks/wat2020-devtest/en-te/dev.en  \n",
+            "  inflating: benchmarks/wat2020-devtest/en-te/test.en  \n",
+            "  inflating: benchmarks/wat2020-devtest/en-te/test.te  \n",
+            "  inflating: benchmarks/wat2020-devtest/en-te/dev.te  \n",
+            "   creating: benchmarks/wat2020-devtest/en-hi/\n",
+            "  inflating: benchmarks/wat2020-devtest/en-hi/dev.en  \n",
+            "  inflating: benchmarks/wat2020-devtest/en-hi/test.hi  \n",
+            "  inflating: benchmarks/wat2020-devtest/en-hi/test.en  \n",
+            "  inflating: benchmarks/wat2020-devtest/en-hi/dev.hi  \n",
+            "   creating: benchmarks/wat2020-devtest/en-gu/\n",
+            "  inflating: benchmarks/wat2020-devtest/en-gu/dev.gu  \n",
+            "  inflating: benchmarks/wat2020-devtest/en-gu/dev.en  \n",
+            "  inflating: benchmarks/wat2020-devtest/en-gu/test.en  \n",
+            "  inflating: benchmarks/wat2020-devtest/en-gu/test.gu  \n",
+            "   creating: benchmarks/wat2020-devtest/en-ml/\n",
+            "  inflating: benchmarks/wat2020-devtest/en-ml/dev.en  \n",
+            "  inflating: benchmarks/wat2020-devtest/en-ml/test.en  \n",
+            "  inflating: benchmarks/wat2020-devtest/en-ml/dev.ml  \n",
+            "  inflating: benchmarks/wat2020-devtest/en-ml/test.ml  \n",
+            "   creating: benchmarks/ufal-ta/\n",
+            "   creating: benchmarks/ufal-ta/en-ta/\n",
+            "  inflating: benchmarks/ufal-ta/en-ta/dev.en  \n",
+            "  inflating: benchmarks/ufal-ta/en-ta/dev.ta  \n",
+            "  inflating: benchmarks/ufal-ta/en-ta/test.en  \n",
+            "  inflating: benchmarks/ufal-ta/en-ta/test.ta  \n",
+            "   creating: benchmarks/wmt-news/\n",
+            "   creating: benchmarks/wmt-news/en-ta/\n",
+            "  inflating: benchmarks/wmt-news/en-ta/dev.en  \n",
+            "  inflating: benchmarks/wmt-news/en-ta/dev.ta  \n",
+            "  inflating: benchmarks/wmt-news/en-ta/test.en  \n",
+            "  inflating: benchmarks/wmt-news/en-ta/test.ta  \n",
+            "   creating: benchmarks/wmt-news/en-hi/\n",
+            "  inflating: benchmarks/wmt-news/en-hi/dev.en  \n",
+            "  inflating: benchmarks/wmt-news/en-hi/test.hi  \n",
+            "  inflating: benchmarks/wmt-news/en-hi/test.en  \n",
+            "  inflating: benchmarks/wmt-news/en-hi/dev.hi  \n",
+            "   creating: benchmarks/wmt-news/en-gu/\n",
+            "  inflating: benchmarks/wmt-news/en-gu/test.en  \n",
+            "  inflating: benchmarks/wmt-news/en-gu/test.gu  \n"
+          ]
+        }
+      ],
+      "source": [
+        "## for the latest samanantar dataset v0.3 -> please use this link: https://storage.googleapis.com/samanantar-public/V0.3/source_wise_splits.zip\n",
+        "# This v0.3 dataset has source wise splits to indicate where the data has been collected from\n",
+        "# For preprocessing simplicity we will use v0.2( which just uses raw text files without source information) in this tutorial\n",
+        "# \n",
+        "# \n",
+        "#  lets now download the indictrans data v0.2 dataset\n",
+        "! wget https://storage.googleapis.com/samanantar-public/V0.2/data/en2indic/samanatar-en-indic-v0.2.zip\n",
+        "\n",
+        "\n",
+        "\n",
+        "# lets also download the benchmarks for dev and test set\n",
+        "\n",
+        "! wget https://storage.googleapis.com/samanantar-public/benchmarks.zip\n",
+        "\n",
+        "# training data is organized as en-X folders where each folder contains two text files containing parallel data for en-X lang pair.\n",
+        "\n",
+        "# final_data\n",
+        "# ├── en-as\n",
+        "# │   ├── train.as\n",
+        "# │   └── train.en\n",
+        "# ├── en-bn\n",
+        "# │   ├── train.bn\n",
+        "# │   └── train.en\n",
+        "# ├── en-gu\n",
+        "# │   ├── train.en\n",
+        "# │   └── train.gu\n",
+        "# ├── en-hi\n",
+        "# │   ├── train.en\n",
+        "# │   └── train.hi\n",
+        "# ├── en-kn\n",
+        "# │   ├── train.en\n",
+        "# │   └── train.kn\n",
+        "# ├── en-ml\n",
+        "# │   ├── train.en\n",
+        "# │   └── train.ml\n",
+        "# ├── en-mr\n",
+        "# │   ├── train.en\n",
+        "# │   └── train.mr\n",
+        "# ├── en-or\n",
+        "# │   ├── train.en\n",
+        "# │   └── train.or\n",
+        "# ├── en-pa\n",
+        "# │   ├── train.en\n",
+        "# │   └── train.pa\n",
+        "# ├── en-ta\n",
+        "# │   ├── train.en\n",
+        "# │   └── train.ta\n",
+        "# └── en-te\n",
+        "#     ├── train.en\n",
+        "#     └── train.te\n",
+        "\n",
+        "\n",
+        "! unzip samanatar-en-indic-v0.2.zip\n",
+        "\n",
+        "# benchmarks folder consists of all the benchmarks we report in the paper - pmi, ufal-ta, wat2020, wat2021, wmt-news\n",
+        "\n",
+        "! unzip benchmarks.zip"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MR_2GQoa84Jn"
+      },
+      "outputs": [],
+      "source": [
+        "# create an experiment dir to store train data, devtest data. \n",
+        "# This folder will also store vocabulary files (created with subword_nmt for bpe), fairseq bin files (for training), model checkpoints.\n",
+        "\n",
+        "# for this example we will be training indic to en translation model. We will name our exp_dir as indic-en-exp\n",
+        "! mkdir indic-en-exp\n",
+        "# copying all the train folders to exp_dir\n",
+        "! cp -r final_data/* indic-en-exp\n",
+        "\n",
+        "! mkdir -p indic-en-exp/devtest\n",
+        "\n",
+        "# copying all benchmarks to devtest folder in exp_dir\n",
+        "! cp -r benchmarks/* indic-en-exp/devtest\n",
+        "\n",
+        "# folder to store combined devtest data (based on the domains you want to test, you can combine multiple benchmarks dev datasets, remove duplicates)\n",
+        "! mkdir -p indic-en-exp/devtest/all\n",
+        "\n",
+        "# in this tutorial, for simplicity, we will just use wat2020 devtest for dev and test set\n",
+        "! cp -r indic-en-exp/devtest/wat2020-devtest/* indic-en-exp/devtest/all\n",
+        "\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lorcT8wkFPtQ"
+      },
+      "outputs": [],
+      "source": [
+        "% cd indicTrans"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vhvYXUc1FaVn"
+      },
+      "outputs": [],
+      "source": [
+        "# prepare_data_joint_training.sh takes experiment dir, src_lang, tgt_lang as input \n",
+        "# This does preprocessing, building vocab, binarization for joint training\n",
+        "\n",
+        "# The learning  and applying vocabulary will take a while if the dataset is huge. To make it faster, run it on a multicore system\n",
+        "\n",
+        "! bash prepare_data_joint_training.sh '../indic-en-exp' 'indic' 'en'"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "p1i3fRQzF2-x"
+      },
+      "outputs": [],
+      "source": [
+        "# Training the model\n",
+        "\n",
+        "# pls refer to fairseq documentaion to know more about each of these options (https://fairseq.readthedocs.io/en/latest/command_line_tools.html)\n",
+        "\n",
+        "\n",
+        "# some notable args:\n",
+        "# --max-updates         -> maximum update steps the model will be trained for\n",
+        "# --arch=transformer_4x -> we use a custom transformer model and name it transformer_4x (4 times the parameter size of transformer  base)\n",
+        "# --user_dir            -> we define the custom transformer arch in model_configs folder and pass it as an argument to user_dir for fairseq to register this architechture\n",
+        "# --lr                  -> learning rate. From our limited experiments, we find that lower learning rates like 3e-5 works best for finetuning.\n",
+        "# --max_tokens          -> this is max tokens per batch. You should limit to lower values if you get oom errors.\n",
+        "# --update-freq         -> gradient accumulation steps\n",
+        "\n",
+        "\n",
+        "!( fairseq-train ../indic-en-exp/final_bin \\\n",
+        "--max-source-positions=210 \\\n",
+        "--max-target-positions=210 \\\n",
+        "--max-update=<max_updates> \\\n",
+        "--save-interval=1 \\\n",
+        "--arch=transformer_4x \\\n",
+        "--criterion=label_smoothed_cross_entropy \\\n",
+        "--source-lang=SRC \\\n",
+        "--lr-scheduler=inverse_sqrt \\\n",
+        "--target-lang=TGT \\\n",
+        "--label-smoothing=0.1 \\\n",
+        "--optimizer adam \\\n",
+        "--adam-betas \"(0.9, 0.98)\" \\\n",
+        "--clip-norm 1.0 \\\n",
+        "--warmup-init-lr 1e-07 \\\n",
+        "--lr 0.0005 \\\n",
+        "--warmup-updates 4000 \\\n",
+        "--dropout 0.2 \\\n",
+        "--save-dir ../indic-en-exp/model \\\n",
+        "--keep-last-epochs 5 \\\n",
+        "--patience 5 \\\n",
+        "--skip-invalid-size-inputs-valid-test \\\n",
+        "--fp16 \\\n",
+        "--user-dir model_configs \\\n",
+        "--wandb-project <wandb_project_name> \\\n",
+        "--update-freq=<grad_accumulation_steps> \\\n",
+        "--distributed-world-size <num_gpus> \\\n",
+        "--max-tokens <max_tokens_in_a_batch> )"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "authorship_tag": "ABX9TyO6AA5gXphZ5kJ6h+dgeSqb",
+      "collapsed_sections": [],
+      "include_colab_link": true,
+      "name": "IndicTrans_training.ipynb",
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.7.7"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Gowtham.R
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

api.py

@@ -0,0 +1,86 @@
+import time
+
+from fairseq import checkpoint_utils, distributed_utils, options, tasks, utils
+from inference.engine import Model
+from flask import Flask, request
+from flask import jsonify
+from flask_cors import CORS, cross_origin
+import webvtt
+from io import StringIO
+
+
+app = Flask(__name__)
+cors = CORS(app)
+app.config['CORS_HEADERS'] = 'Content-Type'
+
+indic2en_model = Model(expdir='../models/v3/indic-en')
+en2indic_model = Model(expdir='../models/v3/en-indic')
+m2m_model = Model(expdir='../models/m2m')
+
+language_dict = {
+    'Assamese': 'as',
+    'Hindi' : 'hi',
+    'Marathi' : 'mr',
+    'Tamil' : 'ta',
+    'Bengali' : 'bn',
+    'Kannada' : 'kn',
+    'Oriya' : 'or',
+    'Telugu' : 'te',
+    'Gujarati' : 'gu',
+    'Malayalam' : 'ml',
+    'Punjabi' : 'pa',
+}
+
+def get_inference_params():
+    model_type = request.form['model_type']
+    source_language = request.form['source_language']
+    target_language = request.form['target_language']
+
+    if model_type == 'indic-en':
+        model = indic2en_model
+        source_lang = language_dict[source_language]
+        assert target_language == 'English'
+        target_lang = 'en'
+    elif model_type == 'en-indic':
+        model = en2indic_model
+        assert source_language == 'English'
+        source_lang = 'en'
+        target_lang = language_dict[target_language]
+    elif model_type == 'm2m':
+        model = m2m_model
+        source_lang = language_dict[source_language]
+        target_lang = language_dict[target_language]
+    
+    return model, source_lang, target_lang
+
+@app.route('/', methods=['GET'])
+def main():
+    return "IndicTrans API"
+
+@app.route("/translate", methods=['POST'])
+@cross_origin()
+def infer_indic_en():
+    model, source_lang, target_lang = get_inference_params()
+    source_text = request.form['text']
+
+    start_time = time.time()
+    target_text = model.translate_paragraph(source_text, source_lang, target_lang)
+    end_time = time.time()
+    return {'text':target_text, 'duration':round(end_time-start_time, 2)}
+
+@app.route("/translate_vtt", methods=['POST'])
+@cross_origin()
+def infer_vtt_indic_en():
+    model, source_lang, target_lang = get_inference_params()
+    source_text = request.form['text']
+    captions = webvtt.read_buffer(StringIO(source_text))
+    source_sentences = [caption.text.replace('\r', '').replace('\n', ' ') for caption in captions]
+
+    start_time = time.time()
+    target_sentences = model.batch_translate(source_sentences, source_lang, target_lang)
+    end_time = time.time()
+
+    for i in range(len(target_sentences)):
+        captions[i].text = target_sentences[i]
+
+    return {'text': captions.content, 'duration':round(end_time-start_time, 2)}

app.py

@@ -0,0 +1,36 @@
+import os
+#import gradio as gr
+
+os.system('wget -q https://storage.googleapis.com/vakyaansh-open-models/translation_models/en-indic.zip')
+os.system('unzip /home/user/app/en-indic.zip')
+os.system('pip uninstall -y numpy')
+os.system('pip install numpy')
+#os.system('pip uninstall -y numba')
+#os.system('pip install numba==0.53')
+
+from fairseq import checkpoint_utils, distributed_utils, options, tasks, utils
+import gradio as grd
+from inference.engine import Model
+indic2en_model = Model(expdir='en-indic')
+
+INDIC = {"Assamese": "as", "Bengali": "bn", "Gujarati": "gu", "Hindi": "hi","Kannada": "kn","Malayalam": "ml", "Marathi": "mr", "Odia": "or","Punjabi": "pa","Tamil": "ta", "Telugu" : "te"}
+
+
+def translate(text, lang):
+  return indic2en_model.translate_paragraph(text, 'en', INDIC[lang])
+
+
+
+languages = list(INDIC.keys())
+
+#print(translate('helo how are you'))
+ddwn = grd.inputs.Dropdown(languages, type="value", default="Hindi", label="Select Target Language")
+txt = grd.inputs.Textbox( lines=5, placeholder="Enter Text to translate", default="", label="Enter Text in English")
+txt_ouptut = grd.outputs.Textbox(type="auto", label="Translated text in Target Language")
+
+example=[['I want to translate this sentence in Hindi','Hindi'],
+        ['I am feeling very good today.', 'Bengali']]
+
+supp = ','.join(languages)
+iface = grd.Interface(fn=translate, inputs=[txt,ddwn] , outputs=txt_ouptut, title='Translation for 11 Indic Languages', description = 'This is a demo based on IndicTrans. Languages Supported: '+supp, article = 'Original repo [link](https://github.com/AI4Bharat/indicTrans) by AI4Bharat. <b>Note: This space can only perform translation from English to Indic languages. Support for other combinations will be provided soon.', examples=example)
+iface.launch(enable_queue=True)

apply_bpe_traindevtest_notag.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+expdir=$1  # EXPDIR
+
+SUBWORD_NMT_DIR="subword-nmt"
+
+data_dir="$expdir/data"
+mkdir -p $expdir/bpe
+
+for dset in `echo train dev test`
+do
+    echo $dset
+    in_dset_dir="$data_dir/$dset"
+    out_dset_dir="$expdir/bpe/$dset"
+    # out_dset_dir="$expdir/final/$dset"
+    echo "Apply joint vocab to SRC corpus"
+    # for very large datasets, use gnu-parallel to speed up applying bpe
+    # uncomment the below line if the apply bpe is slow
+
+    # parallel --pipe --keep-order \
+    python $SUBWORD_NMT_DIR/subword_nmt/apply_bpe.py \
+        -c $expdir/vocab/bpe_codes.32k.SRC_TGT \
+        --vocabulary $expdir/vocab/vocab.SRC \
+        --vocabulary-threshold 5 \
+        --num-workers "-1" \
+        < $in_dset_dir.SRC \
+        > $out_dset_dir.SRC
+    echo "Apply joint vocab to TGT corpus"
+
+    # for very large datasets, use gnu-parallel to speed up applying bpe
+    # uncomment the below line if the apply bpe is slow
+
+    # parallel --pipe --keep-order \
+    python $SUBWORD_NMT_DIR/subword_nmt/apply_bpe.py \
+        -c $expdir/vocab/bpe_codes.32k.SRC_TGT \
+        --vocabulary $expdir/vocab/vocab.TGT \
+        --vocabulary-threshold 5 \
+        --num-workers "-1" \
+        < $in_dset_dir.TGT \
+        > $out_dset_dir.TGT
+done

apply_single_bpe_traindevtest_notag.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+expdir=$1  # EXPDIR
+
+SUBWORD_NMT_DIR="subword-nmt"
+
+data_dir="$expdir/data"
+mkdir -p $expdir/bpe
+
+for dset in `echo train dev test`
+do
+    echo $dset
+    in_dset_dir="$data_dir/$dset"
+    out_dset_dir="$expdir/bpe/$dset"
+    # out_dset_dir="$expdir/final/$dset"
+    echo "Apply to SRC corpus"
+    # for very large datasets, use gnu-parallel to speed up applying bpe
+    # uncomment the below line if the apply bpe is slow
+
+    # parallel --pipe --keep-order \
+    python $SUBWORD_NMT_DIR/subword_nmt/apply_bpe.py \
+        -c $expdir/vocab/bpe_codes.32k.SRC \
+        --vocabulary $expdir/vocab/vocab.SRC \
+        --vocabulary-threshold 5 \
+        --num-workers "-1" \
+        < $in_dset_dir.SRC \
+        > $out_dset_dir.SRC
+    echo "Apply to TGT corpus"
+    # for very large datasets, use gnu-parallel to speed up applying bpe
+    # uncomment the below line if the apply bpe is slow
+
+    # parallel --pipe --keep-order \
+    python $SUBWORD_NMT_DIR/subword_nmt/apply_bpe.py \
+        -c $expdir/vocab/bpe_codes.32k.TGT \
+        --vocabulary $expdir/vocab/vocab.TGT \
+        --vocabulary-threshold 5 \
+        --num-workers "-1" \
+        < $in_dset_dir.TGT \
+        > $out_dset_dir.TGT
+done

binarize_training_exp.sh

@@ -0,0 +1,24 @@
+#/bin/bash
+
+exp_dir=$1
+src_lang=$2
+tgt_lang=$3
+
+# use cpu_count to get num_workers instead of setting it manually when running in different
+# instances
+num_workers=`python -c "import multiprocessing; print(multiprocessing.cpu_count())"`
+
+data_dir=$exp_dir/final
+out_data_dir=$exp_dir/final_bin
+
+rm -rf $out_data_dir
+
+fairseq-preprocess \
+    --source-lang $src_lang --target-lang $tgt_lang \
+    --trainpref $data_dir/train \
+    --validpref $data_dir/dev \
+    --testpref $data_dir/test \
+    --destdir $out_data_dir \
+    --workers $num_workers \
+    --thresholdtgt 5 \
+    --thresholdsrc 5

compute_bleu.sh

@@ -0,0 +1,28 @@
+pred_fname=$1
+ref_fname=$2
+src_lang=$3
+tgt_lang=$4
+
+# we compute and report tokenized bleu scores.
+# For computing BLEU scores, systems should output detokenized outputs. Your MT system might be doing it out of the box if you are using SentencePiece - nothing to do in that case.
+# If you are using BPE then:
+# 1. For English, you can use MosesDetokenizer (either the scripts in moses or the sacremoses python package)
+# 2. For Indian languages, you can use the IndicNLP library detokenizer (note: please don't skip this step, since detok/tokenizer are not guaranteed to be reversible**.
+# ^ both 1. and 2. are scripts/postprocess_translate.py
+
+
+# For computing BLEU, we use sacrebleu:
+# For English output: sacrebleu reffile < outputfile. This internally tokenizes using mteval-v13a
+# For Indian language output, we need tokenized output and reference since we don't know how well the sacrebleu tokenizer works for Indic input.
+# Hence we tokenize both preds and target files with IndicNLP tokenizer and then run: sacrebleu --tokenize none reffile < outputfile
+if [ $tgt_lang == 'en' ]; then
+    # indic to en models
+    sacrebleu $ref_fname < $pred_fname
+else
+    # indicnlp tokenize predictions and reference files before evaluation
+    input_size=`python scripts/preprocess_translate.py $ref_fname $ref_fname.tok $tgt_lang`
+    input_size=`python scripts/preprocess_translate.py $pred_fname $pred_fname.tok $tgt_lang`
+
+    # since we are tokenizing with indicnlp separately, we are setting tokenize to none here
+    sacrebleu --tokenize none $ref_fname.tok < $pred_fname.tok
+fi

indicTrans_python_interface.ipynb

@@ -0,0 +1,462 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "view-in-github"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/gowtham1997/indicTrans-1/blob/main/indicTrans_python_interface.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 1,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "CjfzxXZLHed_",
+        "outputId": "69a66b95-41b2-4413-82d1-0caacbddb3f3"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Cloning into 'indicTrans-1'...\n",
+            "remote: Enumerating objects: 486, done.\u001b[K\n",
+            "remote: Counting objects: 100% (189/189), done.\u001b[K\n",
+            "remote: Compressing objects: 100% (67/67), done.\u001b[K\n",
+            "remote: Total 486 (delta 154), reused 134 (delta 121), pack-reused 297\u001b[K\n",
+            "Receiving objects: 100% (486/486), 1.48 MiB | 17.61 MiB/s, done.\n",
+            "Resolving deltas: 100% (281/281), done.\n",
+            "/content/indicTrans\n",
+            "Cloning into 'indic_nlp_library'...\n",
+            "remote: Enumerating objects: 1325, done.\u001b[K\n",
+            "remote: Counting objects: 100% (147/147), done.\u001b[K\n",
+            "remote: Compressing objects: 100% (103/103), done.\u001b[K\n",
+            "remote: Total 1325 (delta 84), reused 89 (delta 41), pack-reused 1178\u001b[K\n",
+            "Receiving objects: 100% (1325/1325), 9.57 MiB | 13.55 MiB/s, done.\n",
+            "Resolving deltas: 100% (688/688), done.\n",
+            "Cloning into 'indic_nlp_resources'...\n",
+            "remote: Enumerating objects: 133, done.\u001b[K\n",
+            "remote: Counting objects: 100% (7/7), done.\u001b[K\n",
+            "remote: Compressing objects: 100% (7/7), done.\u001b[K\n",
+            "remote: Total 133 (delta 0), reused 2 (delta 0), pack-reused 126\u001b[K\n",
+            "Receiving objects: 100% (133/133), 149.77 MiB | 33.48 MiB/s, done.\n",
+            "Resolving deltas: 100% (51/51), done.\n",
+            "Checking out files: 100% (28/28), done.\n",
+            "Cloning into 'subword-nmt'...\n",
+            "remote: Enumerating objects: 580, done.\u001b[K\n",
+            "remote: Counting objects: 100% (4/4), done.\u001b[K\n",
+            "remote: Compressing objects: 100% (4/4), done.\u001b[K\n",
+            "remote: Total 580 (delta 0), reused 1 (delta 0), pack-reused 576\u001b[K\n",
+            "Receiving objects: 100% (580/580), 237.41 KiB | 18.26 MiB/s, done.\n",
+            "Resolving deltas: 100% (349/349), done.\n",
+            "/content\n"
+          ]
+        }
+      ],
+      "source": [
+        "# clone the repo for running evaluation\n",
+        "!git clone https://github.com/AI4Bharat/indicTrans.git\n",
+        "%cd indicTrans\n",
+        "# clone requirements repositories\n",
+        "!git clone https://github.com/anoopkunchukuttan/indic_nlp_library.git\n",
+        "!git clone https://github.com/anoopkunchukuttan/indic_nlp_resources.git\n",
+        "!git clone https://github.com/rsennrich/subword-nmt.git\n",
+        "%cd .."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 2,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "IeYW2BJhlJvx",
+        "outputId": "3357bc85-44d8-43b0-8c64-eef9f18be716"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Collecting sacremoses\n",
+            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n",
+            "\r\u001b[K     |▍                               | 10kB 14.0MB/s eta 0:00:01\r\u001b[K     |▊                               | 20kB 18.8MB/s eta 0:00:01\r\u001b[K     |█                               | 30kB 22.5MB/s eta 0:00:01\r\u001b[K     |█▌                              | 40kB 25.7MB/s eta 0:00:01\r\u001b[K     |█▉                              | 51kB 27.6MB/s eta 0:00:01\r\u001b[K     |██▏                             | 61kB 29.2MB/s eta 0:00:01\r\u001b[K     |██▋                             | 71kB 27.3MB/s eta 0:00:01\r\u001b[K     |███                             | 81kB 27.7MB/s eta 0:00:01\r\u001b[K     |███▎                            | 92kB 28.8MB/s eta 0:00:01\r\u001b[K     |███▋                            | 102kB 29.9MB/s eta 0:00:01\r\u001b[K     |████                            | 112kB 29.9MB/s eta 0:00:01\r\u001b[K     |████▍                           | 122kB 29.9MB/s eta 0:00:01\r\u001b[K     |████▊                           | 133kB 29.9MB/s eta 0:00:01\r\u001b[K     |█████▏                          | 143kB 29.9MB/s eta 0:00:01\r\u001b[K     |█████▌                          | 153kB 29.9MB/s eta 0:00:01\r\u001b[K     |█████▉                          | 163kB 29.9MB/s eta 0:00:01\r\u001b[K     |██████▎                         | 174kB 29.9MB/s eta 0:00:01\r\u001b[K     |██████▋                         | 184kB 29.9MB/s eta 0:00:01\r\u001b[K     |███████                         | 194kB 29.9MB/s eta 0:00:01\r\u001b[K     |███████▎                        | 204kB 29.9MB/s eta 0:00:01\r\u001b[K     |███████▊                        | 215kB 29.9MB/s eta 0:00:01\r\u001b[K     |████████                        | 225kB 29.9MB/s eta 0:00:01\r\u001b[K     |████████▍                       | 235kB 29.9MB/s eta 0:00:01\r\u001b[K     |████████▉                       | 245kB 29.9MB/s eta 0:00:01\r\u001b[K     |█████████▏                      | 256kB 29.9MB/s eta 0:00:01\r\u001b[K     |█████████▌                      | 266kB 29.9MB/s eta 0:00:01\r\u001b[K     |█████████▉                      | 276kB 29.9MB/s eta 0:00:01\r\u001b[K     |██████████▎                     | 286kB 29.9MB/s eta 0:00:01\r\u001b[K     |██████████▋                     | 296kB 29.9MB/s eta 0:00:01\r\u001b[K     |███████████                     | 307kB 29.9MB/s eta 0:00:01\r\u001b[K     |███████████▍                    | 317kB 29.9MB/s eta 0:00:01\r\u001b[K     |███████████▊                    | 327kB 29.9MB/s eta 0:00:01\r\u001b[K     |████████████                    | 337kB 29.9MB/s eta 0:00:01\r\u001b[K     |████████████▌                   | 348kB 29.9MB/s eta 0:00:01\r\u001b[K     |████████████▉                   | 358kB 29.9MB/s eta 0:00:01\r\u001b[K     |█████████████▏                  | 368kB 29.9MB/s eta 0:00:01\r\u001b[K     |█████████████▌                  | 378kB 29.9MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 389kB 29.9MB/s eta 0:00:01\r\u001b[K     |██████████████▎                 | 399kB 29.9MB/s eta 0:00:01\r\u001b[K     |██████████████▋                 | 409kB 29.9MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 419kB 29.9MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 430kB 29.9MB/s eta 0:00:01\r\u001b[K     |███████████████▊                | 440kB 29.9MB/s eta 0:00:01\r\u001b[K     |████████████████                | 450kB 29.9MB/s eta 0:00:01\r\u001b[K     |████████████████▌               | 460kB 29.9MB/s eta 0:00:01\r\u001b[K     |████████████████▉               | 471kB 29.9MB/s eta 0:00:01\r\u001b[K     |█████████████████▏              | 481kB 29.9MB/s eta 0:00:01\r\u001b[K     |█████████████████▋              | 491kB 29.9MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 501kB 29.9MB/s eta 0:00:01\r\u001b[K     |██████████████████▎             | 512kB 29.9MB/s eta 0:00:01\r\u001b[K     |██████████████████▊             | 522kB 29.9MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 532kB 29.9MB/s eta 0:00:01\r\u001b[K     |███████████████████▍            | 542kB 29.9MB/s eta 0:00:01\r\u001b[K     |███████████████████▊            | 552kB 29.9MB/s eta 0:00:01\r\u001b[K     |████████████████████▏           | 563kB 29.9MB/s eta 0:00:01\r\u001b[K     |████████████████████▌           | 573kB 29.9MB/s eta 0:00:01\r\u001b[K     |████████████████████▉           | 583kB 29.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████▎          | 593kB 29.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████▋          | 604kB 29.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 614kB 29.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████▎         | 624kB 29.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████▊         | 634kB 29.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 645kB 29.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████▍        | 655kB 29.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 665kB 29.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████▏       | 675kB 29.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████▌       | 686kB 29.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 696kB 29.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▎      | 706kB 29.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▋      | 716kB 29.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 727kB 29.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▍     | 737kB 29.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▊     | 747kB 29.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 757kB 29.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▌    | 768kB 29.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▉    | 778kB 29.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▏   | 788kB 29.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▌   | 798kB 29.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 808kB 29.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▎  | 819kB 29.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▋  | 829kB 29.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 839kB 29.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▍ | 849kB 29.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▊ | 860kB 29.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▏| 870kB 29.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▌| 880kB 29.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▉| 890kB 29.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 901kB 29.9MB/s \n",
+            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (1.1.5)\n",
+            "Collecting mock\n",
+            "  Downloading https://files.pythonhosted.org/packages/5c/03/b7e605db4a57c0f6fba744b11ef3ddf4ddebcada35022927a2b5fc623fdf/mock-4.0.3-py3-none-any.whl\n",
+            "Collecting sacrebleu\n",
+            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7e/57/0c7ca4e31a126189dab99c19951910bd081dea5bbd25f24b77107750eae7/sacrebleu-1.5.1-py3-none-any.whl (54kB)\n",
+            "\u001b[K     |████████████████████████████████| 61kB 7.5MB/s \n",
+            "\u001b[?25hCollecting tensorboardX\n",
+            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/42/36/2b147652c40c3a858efa0afbf7b8236fae968e88ff530511a4cfa299a506/tensorboardX-2.3-py2.py3-none-any.whl (124kB)\n",
+            "\u001b[K     |████████████████████████████████| 133kB 47.5MB/s \n",
+            "\u001b[?25hRequirement already satisfied: pyarrow in /usr/local/lib/python3.7/dist-packages (3.0.0)\n",
+            "Collecting indic-nlp-library\n",
+            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/84/d4/495bb43b88a2a6d04b09c29fc5115f24872af74cd8317fe84026abd4ddb1/indic_nlp_library-0.81-py3-none-any.whl (40kB)\n",
+            "\u001b[K     |████████████████████████████████| 40kB 5.2MB/s \n",
+            "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from sacremoses) (4.41.1)\n",
+            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses) (1.15.0)\n",
+            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses) (1.0.1)\n",
+            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from sacremoses) (2019.12.20)\n",
+            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses) (7.1.2)\n",
+            "Requirement already satisfied: numpy>=1.15.4 in /usr/local/lib/python3.7/dist-packages (from pandas) (1.19.5)\n",
+            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas) (2.8.1)\n",
+            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas) (2018.9)\n",
+            "Collecting portalocker==2.0.0\n",
+            "  Downloading https://files.pythonhosted.org/packages/89/a6/3814b7107e0788040870e8825eebf214d72166adf656ba7d4bf14759a06a/portalocker-2.0.0-py2.py3-none-any.whl\n",
+            "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorboardX) (3.12.4)\n",
+            "Collecting sphinx-rtd-theme\n",
+            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ac/24/2475e8f83519b54b2148d4a56eb1111f9cec630d088c3ffc214492c12107/sphinx_rtd_theme-0.5.2-py2.py3-none-any.whl (9.1MB)\n",
+            "\u001b[K     |████████████████████████████████| 9.2MB 42.0MB/s \n",
+            "\u001b[?25hCollecting morfessor\n",
+            "  Downloading https://files.pythonhosted.org/packages/39/e6/7afea30be2ee4d29ce9de0fa53acbb033163615f849515c0b1956ad074ee/Morfessor-2.0.6-py3-none-any.whl\n",
+            "Collecting sphinx-argparse\n",
+            "  Downloading https://files.pythonhosted.org/packages/06/2b/dfad6a1831c3aeeae25d8d3d417224684befbf45e10c7f2141631616a6ed/sphinx-argparse-0.2.5.tar.gz\n",
+            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from protobuf>=3.8.0->tensorboardX) (57.0.0)\n",
+            "Requirement already satisfied: sphinx in /usr/local/lib/python3.7/dist-packages (from sphinx-rtd-theme->indic-nlp-library) (1.8.5)\n",
+            "Collecting docutils<0.17\n",
+            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/81/44/8a15e45ffa96e6cf82956dd8d7af9e666357e16b0d93b253903475ee947f/docutils-0.16-py2.py3-none-any.whl (548kB)\n",
+            "\u001b[K     |████████████████████████████████| 552kB 31.5MB/s \n",
+            "\u001b[?25hRequirement already satisfied: sphinxcontrib-websupport in /usr/local/lib/python3.7/dist-packages (from sphinx->sphinx-rtd-theme->indic-nlp-library) (1.2.4)\n",
+            "Requirement already satisfied: snowballstemmer>=1.1 in /usr/local/lib/python3.7/dist-packages (from sphinx->sphinx-rtd-theme->indic-nlp-library) (2.1.0)\n",
+            "Requirement already satisfied: Jinja2>=2.3 in /usr/local/lib/python3.7/dist-packages (from sphinx->sphinx-rtd-theme->indic-nlp-library) (2.11.3)\n",
+            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from sphinx->sphinx-rtd-theme->indic-nlp-library) (20.9)\n",
+            "Requirement already satisfied: alabaster<0.8,>=0.7 in /usr/local/lib/python3.7/dist-packages (from sphinx->sphinx-rtd-theme->indic-nlp-library) (0.7.12)\n",
+            "Requirement already satisfied: imagesize in /usr/local/lib/python3.7/dist-packages (from sphinx->sphinx-rtd-theme->indic-nlp-library) (1.2.0)\n",
+            "Requirement already satisfied: Pygments>=2.0 in /usr/local/lib/python3.7/dist-packages (from sphinx->sphinx-rtd-theme->indic-nlp-library) (2.6.1)\n",
+            "Requirement already satisfied: requests>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from sphinx->sphinx-rtd-theme->indic-nlp-library) (2.23.0)\n",
+            "Requirement already satisfied: babel!=2.0,>=1.3 in /usr/local/lib/python3.7/dist-packages (from sphinx->sphinx-rtd-theme->indic-nlp-library) (2.9.1)\n",
+            "Requirement already satisfied: sphinxcontrib-serializinghtml in /usr/local/lib/python3.7/dist-packages (from sphinxcontrib-websupport->sphinx->sphinx-rtd-theme->indic-nlp-library) (1.1.5)\n",
+            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from Jinja2>=2.3->sphinx->sphinx-rtd-theme->indic-nlp-library) (2.0.1)\n",
+            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->sphinx->sphinx-rtd-theme->indic-nlp-library) (2.4.7)\n",
+            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx->sphinx-rtd-theme->indic-nlp-library) (2.10)\n",
+            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx->sphinx-rtd-theme->indic-nlp-library) (3.0.4)\n",
+            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx->sphinx-rtd-theme->indic-nlp-library) (1.24.3)\n",
+            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx->sphinx-rtd-theme->indic-nlp-library) (2021.5.30)\n",
+            "Building wheels for collected packages: sphinx-argparse\n",
+            "  Building wheel for sphinx-argparse (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+            "  Created wheel for sphinx-argparse: filename=sphinx_argparse-0.2.5-cp37-none-any.whl size=11552 sha256=d8cbdca000085e2e2c122c305bb21aa76a9600012ded8e06c300e03d1c4d1e32\n",
+            "  Stored in directory: /root/.cache/pip/wheels/2a/18/1b/4990a1859da4edc77ab312bc2986c08d2733fb5713d06e44f5\n",
+            "Successfully built sphinx-argparse\n",
+            "\u001b[31mERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.\u001b[0m\n",
+            "Installing collected packages: sacremoses, mock, portalocker, sacrebleu, tensorboardX, docutils, sphinx-rtd-theme, morfessor, sphinx-argparse, indic-nlp-library\n",
+            "  Found existing installation: docutils 0.17.1\n",
+            "    Uninstalling docutils-0.17.1:\n",
+            "      Successfully uninstalled docutils-0.17.1\n",
+            "Successfully installed docutils-0.16 indic-nlp-library-0.81 mock-4.0.3 morfessor-2.0.6 portalocker-2.0.0 sacrebleu-1.5.1 sacremoses-0.0.45 sphinx-argparse-0.2.5 sphinx-rtd-theme-0.5.2 tensorboardX-2.3\n",
+            "Collecting mosestokenizer\n",
+            "  Downloading https://files.pythonhosted.org/packages/4b/b3/c0af235b16c4f44a2828ef017f7947d1262b2646e440f85c6a2ff26a8c6f/mosestokenizer-1.1.0.tar.gz\n",
+            "Collecting subword-nmt\n",
+            "  Downloading https://files.pythonhosted.org/packages/74/60/6600a7bc09e7ab38bc53a48a20d8cae49b837f93f5842a41fe513a694912/subword_nmt-0.3.7-py2.py3-none-any.whl\n",
+            "Requirement already satisfied: docopt in /usr/local/lib/python3.7/dist-packages (from mosestokenizer) (0.6.2)\n",
+            "Collecting openfile\n",
+            "  Downloading https://files.pythonhosted.org/packages/93/e6/805db6867faacb488b44ba8e0829ef4de151dd0499f3c5da5f4ad11698a7/openfile-0.0.7-py3-none-any.whl\n",
+            "Collecting uctools\n",
+            "  Downloading https://files.pythonhosted.org/packages/04/cb/70ed842d9a43460eedaa11f7503b4ab6537b43b63f0d854d59d8e150fac1/uctools-1.3.0.tar.gz\n",
+            "Collecting toolwrapper\n",
+            "  Downloading https://files.pythonhosted.org/packages/41/7b/34bf8fb69426d8a18bcc61081e9d126f4fcd41c3c832072bef39af1602cd/toolwrapper-2.1.0.tar.gz\n",
+            "Building wheels for collected packages: mosestokenizer, uctools, toolwrapper\n",
+            "  Building wheel for mosestokenizer (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+            "  Created wheel for mosestokenizer: filename=mosestokenizer-1.1.0-cp37-none-any.whl size=49120 sha256=4fc04046040e73bd5d13c606ebbfc65ac38c7d073f7fc0b0e4cc1d4215b595f3\n",
+            "  Stored in directory: /root/.cache/pip/wheels/a2/e7/48/48d5e0f9c0cd5def2dfd7cb8543945f906448ed1313de24a29\n",
+            "  Building wheel for uctools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+            "  Created wheel for uctools: filename=uctools-1.3.0-cp37-none-any.whl size=6163 sha256=c5a865107c59f98c4da5d18ddc754fa141ab494574187281de1502561c6a004e\n",
+            "  Stored in directory: /root/.cache/pip/wheels/06/b6/8f/935d5bf5bca85d47c6f5ec31641879bba057d336ab36b1e773\n",
+            "  Building wheel for toolwrapper (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+            "  Created wheel for toolwrapper: filename=toolwrapper-2.1.0-cp37-none-any.whl size=3356 sha256=41a3e12078d5681e8467701735208d880ba588b0f5dbfb3b99c4e04bc643eccc\n",
+            "  Stored in directory: /root/.cache/pip/wheels/84/ea/29/e02f3b855bf19344972092873a1091b329309bbc3d3d0cbaef\n",
+            "Successfully built mosestokenizer uctools toolwrapper\n",
+            "Installing collected packages: openfile, uctools, toolwrapper, mosestokenizer, subword-nmt\n",
+            "Successfully installed mosestokenizer-1.1.0 openfile-0.0.7 subword-nmt-0.3.7 toolwrapper-2.1.0 uctools-1.3.0\n",
+            "Cloning into 'fairseq'...\n",
+            "remote: Enumerating objects: 28410, done.\u001b[K\n",
+            "remote: Counting objects: 100% (229/229), done.\u001b[K\n",
+            "remote: Compressing objects: 100% (127/127), done.\u001b[K\n",
+            "remote: Total 28410 (delta 114), reused 187 (delta 99), pack-reused 28181\u001b[K\n",
+            "Receiving objects: 100% (28410/28410), 11.96 MiB | 24.16 MiB/s, done.\n",
+            "Resolving deltas: 100% (21310/21310), done.\n",
+            "/content/fairseq\n",
+            "Obtaining file:///content/fairseq\n",
+            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
+            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
+            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
+            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
+            "Requirement already satisfied: cffi in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+f887152) (1.14.5)\n",
+            "Collecting hydra-core<1.1\n",
+            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/52/e3/fbd70dd0d3ce4d1d75c22d56c0c9f895cfa7ed6587a9ffb821d6812d6a60/hydra_core-1.0.6-py3-none-any.whl (123kB)\n",
+            "\u001b[K     |████████████████████████████████| 133kB 11.6MB/s \n",
+            "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+f887152) (4.41.1)\n",
+            "Collecting omegaconf<2.1\n",
+            "  Downloading https://files.pythonhosted.org/packages/d0/eb/9d63ce09dd8aa85767c65668d5414958ea29648a0eec80a4a7d311ec2684/omegaconf-2.0.6-py3-none-any.whl\n",
+            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+f887152) (2019.12.20)\n",
+            "Requirement already satisfied: numpy; python_version >= \"3.7\" in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+f887152) (1.19.5)\n",
+            "Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+f887152) (1.9.0+cu102)\n",
+            "Requirement already satisfied: sacrebleu>=1.4.12 in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+f887152) (1.5.1)\n",
+            "Requirement already satisfied: cython in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+f887152) (0.29.23)\n",
+            "Requirement already satisfied: pycparser in /usr/local/lib/python3.7/dist-packages (from cffi->fairseq==1.0.0a0+f887152) (2.20)\n",
+            "Collecting antlr4-python3-runtime==4.8\n",
+            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/56/02/789a0bddf9c9b31b14c3e79ec22b9656185a803dc31c15f006f9855ece0d/antlr4-python3-runtime-4.8.tar.gz (112kB)\n",
+            "\u001b[K     |████████████████████████████████| 112kB 33.5MB/s \n",
+            "\u001b[?25hRequirement already satisfied: importlib-resources; python_version < \"3.9\" in /usr/local/lib/python3.7/dist-packages (from hydra-core<1.1->fairseq==1.0.0a0+f887152) (5.1.4)\n",
+            "Collecting PyYAML>=5.1.*\n",
+            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7a/a5/393c087efdc78091afa2af9f1378762f9821c9c1d7a22c5753fb5ac5f97a/PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636kB)\n",
+            "\u001b[K     |████████████████████████████████| 645kB 30.2MB/s \n",
+            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from omegaconf<2.1->fairseq==1.0.0a0+f887152) (3.7.4.3)\n",
+            "Requirement already satisfied: portalocker==2.0.0 in /usr/local/lib/python3.7/dist-packages (from sacrebleu>=1.4.12->fairseq==1.0.0a0+f887152) (2.0.0)\n",
+            "Requirement already satisfied: zipp>=3.1.0; python_version < \"3.10\" in /usr/local/lib/python3.7/dist-packages (from importlib-resources; python_version < \"3.9\"->hydra-core<1.1->fairseq==1.0.0a0+f887152) (3.4.1)\n",
+            "Building wheels for collected packages: antlr4-python3-runtime\n",
+            "  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+            "  Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.8-cp37-none-any.whl size=141231 sha256=69960f774a6fdb385fed1a63fb02ae50b57299408cfd6fb33be60d686be878b7\n",
+            "  Stored in directory: /root/.cache/pip/wheels/e3/e2/fa/b78480b448b8579ddf393bebd3f47ee23aa84c89b6a78285c8\n",
+            "Successfully built antlr4-python3-runtime\n",
+            "Installing collected packages: antlr4-python3-runtime, PyYAML, omegaconf, hydra-core, fairseq\n",
+            "  Found existing installation: PyYAML 3.13\n",
+            "    Uninstalling PyYAML-3.13:\n",
+            "      Successfully uninstalled PyYAML-3.13\n",
+            "  Running setup.py develop for fairseq\n",
+            "Successfully installed PyYAML-5.4.1 antlr4-python3-runtime-4.8 fairseq hydra-core-1.0.6 omegaconf-2.0.6\n",
+            "/content\n"
+          ]
+        }
+      ],
+      "source": [
+        "# Install the necessary libraries\n",
+        "!pip install sacremoses pandas mock sacrebleu tensorboardX pyarrow indic-nlp-library\n",
+        "! pip install mosestokenizer subword-nmt\n",
+        "# Install fairseq from source\n",
+        "!git clone https://github.com/pytorch/fairseq.git\n",
+        "%cd fairseq\n",
+        "# !git checkout da9eaba12d82b9bfc1442f0e2c6fc1b895f4d35d\n",
+        "!pip install --editable ./\n",
+        "\n",
+        "%cd .."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 1,
+      "metadata": {
+        "id": "TktUu9NW_PLq"
+      },
+      "outputs": [],
+      "source": [
+        "# this step is only required if you are running the code on colab\n",
+        "# restart the runtime after running prev cell (to update). See this -> https://stackoverflow.com/questions/57838013/modulenotfounderror-after-successful-pip-install-in-google-colaboratory\n",
+        "\n",
+        "# this import will not work without restarting runtime\n",
+        "from fairseq import checkpoint_utils, distributed_utils, options, tasks, utils"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 9,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "E_4JxNdRlPQB",
+        "outputId": "82ab5e2f-d560-4f4e-bf3f-f1ca0a8d31b8"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "--2021-06-27 12:43:16--  https://storage.googleapis.com/samanantar-public/V0.2/models/indic-en.zip\n",
+            "Resolving storage.googleapis.com (storage.googleapis.com)... 172.217.13.240, 172.217.15.80, 142.251.33.208, ...\n",
+            "Connecting to storage.googleapis.com (storage.googleapis.com)|172.217.13.240|:443... connected.\n",
+            "HTTP request sent, awaiting response... 200 OK\n",
+            "Length: 4551079075 (4.2G) [application/zip]\n",
+            "Saving to: ‘indic-en.zip’\n",
+            "\n",
+            "indic-en.zip        100%[===================>]   4.24G  28.8MB/s    in 83s     \n",
+            "\n",
+            "2021-06-27 12:44:39 (52.1 MB/s) - ‘indic-en.zip’ saved [4551079075/4551079075]\n",
+            "\n",
+            "Archive:  indic-en.zip\n",
+            "   creating: indic-en/\n",
+            "   creating: indic-en/vocab/\n",
+            "  inflating: indic-en/vocab/bpe_codes.32k.SRC  \n",
+            "  inflating: indic-en/vocab/vocab.SRC  \n",
+            "  inflating: indic-en/vocab/vocab.TGT  \n",
+            "  inflating: indic-en/vocab/bpe_codes.32k.TGT  \n",
+            "   creating: indic-en/final_bin/\n",
+            "  inflating: indic-en/final_bin/dict.TGT.txt  \n",
+            "  inflating: indic-en/final_bin/dict.SRC.txt  \n",
+            "   creating: indic-en/model/\n",
+            "  inflating: indic-en/model/checkpoint_best.pt  \n",
+            "/content/indicTrans\n"
+          ]
+        }
+      ],
+      "source": [
+        "# download the indictrans model\n",
+        "\n",
+        "\n",
+        "# downloading the indic-en model\n",
+        "!wget https://storage.googleapis.com/samanantar-public/V0.3/models/indic-en.zip\n",
+        "!unzip indic-en.zip\n",
+        "\n",
+        "# downloading the en-indic model\n",
+        "# !wget https://storage.googleapis.com/samanantar-public/V0.3/models/en-indic.zip\n",
+        "# !unzip en-indic.zip\n",
+        "\n",
+        "# # downloading the indic-indic model\n",
+        "# !wget https://storage.googleapis.com/samanantar-public/V0.3/models/m2m.zip\n",
+        "# !unzip m2m.zip\n",
+        "\n",
+        "%cd indicTrans"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 10,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "yTnWbHqY01-B",
+        "outputId": "0d075f51-097b-46ad-aade-407a4437aa62"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Initializing vocab and bpe\n",
+            "Initializing model for translation\n"
+          ]
+        }
+      ],
+      "source": [
+        "from indicTrans.inference.engine import Model\n",
+        "\n",
+        "indic2en_model = Model(expdir='../indic-en')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 11,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "QTp2NOgQ__sB",
+        "outputId": "e015a71e-8206-4e1d-cb3e-11ecb4d44f76"
+      },
+      "outputs": [
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "100%|██████████| 3/3 [00:00<00:00, 1225.21it/s]\n",
+            "/usr/local/lib/python3.7/dist-packages/torch/_tensor.py:575: UserWarning: floor_divide is deprecated, and will be removed in a future version of pytorch. It currently rounds toward 0 (like the 'trunc' function NOT 'floor'). This results in incorrect rounding for negative values.\n",
+            "To keep the current behavior, use torch.div(a, b, rounding_mode='trunc'), or for actual floor division, use torch.div(a, b, rounding_mode='floor'). (Triggered internally at  /pytorch/aten/src/ATen/native/BinaryOps.cpp:467.)\n",
+            "  return torch.floor_divide(self, other)\n"
+          ]
+        },
+        {
+          "data": {
+            "text/plain": [
+              "['He seems to know us.',\n",
+              " 'I couldnt find it anywhere.',\n",
+              " 'If someone in your neighbourhood develops these symptoms, staying at home can help prevent the spread of the coronavirus infection.']"
+            ]
+          },
+          "execution_count": 11,
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "ta_sents = ['அவனுக்கு நம்மைப் தெரியும் என்று தோன்றுகிறது',\n",
+        "            \"இது எங்கே இருக்கு என்று என்னால் கண்டுபிடிக்க முடியவில்லை.\",\n",
+        "            'உங்களுக்கு உங்கள் அருகில் இருக்கும் ஒருவருக்கோ இத்தகைய அறிகுறிகள் தென்பட்டால், வீட்டிலேயே இருப்பது, கொரோனா வைரஸ் தொற்று பிறருக்கு வராமல் தடுக்க உதவும்.']\n",
+        "\n",
+        "\n",
+        "indic2en_model.batch_translate(ta_sents, 'ta', 'en')\n",
+        "\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 13,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 68
+        },
+        "id": "VFXrCNZGEN7Z",
+        "outputId": "f72aad17-1cc0-4774-a7ee-5b3a5d954de3"
+      },
+      "outputs": [
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "100%|██████████| 4/4 [00:00<00:00, 1496.76it/s]\n"
+          ]
+        },
+        {
+          "data": {
+            "application/vnd.google.colaboratory.intrinsic+json": {
+              "type": "string"
+            },
+            "text/plain": [
+              "'The pandemic has resulted in worldwide social and economic disruption. The world is facing the worst recession since the global financial crisis. This led to the postponement or cancellation of sporting, religious, political and cultural events. Due to the fear, there was shortage of supply as more people purchased items like masks, sanitizers etc.'"
+            ]
+          },
+          "execution_count": 13,
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "\n",
+        "ta_paragraph = \"\"\"இத்தொற்றுநோய் உலகளாவிய சமூக மற்றும் பொருளாதார சீர்குலைவை ஏற்படுத்தியுள்ளது.இதனால் பெரும் பொருளாதார மந்தநிலைக்குப் பின்னர் உலகளவில் மிகப்பெரிய மந்தநிலை ஏற்பட்டுள்ளது. இது விளையாட்டு,மத, அரசியல் மற்றும் கலாச்சார நிகழ்வுகளை ஒத்திவைக்க அல்லது ரத்து செய்ய வழிவகுத்தது.\n",
+        "அச்சம் காரணமாக முகக்கவசம், கிருமிநாசினி உள்ளிட்ட பொருட்களை அதிக நபர்கள் வாங்கியதால் விநியோகப் பற்றாக்குறை ஏற்பட்டது.\"\"\"\n",
+        "\n",
+        "indic2en_model.translate_paragraph(ta_paragraph, 'ta', 'en')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Hi_D7s_VIjis"
+      },
+      "outputs": [],
+      "source": []
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "authorship_tag": "ABX9TyM3t8oQYMhBUuq4/Pyhcr0+",
+      "collapsed_sections": [],
+      "include_colab_link": true,
+      "name": "indicTrans_python_interface.ipynb",
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

indic_nlp_library/LICENSE

@@ -0,0 +1,9 @@
+MIT License
+
+Copyright (c) 2013-present Anoop Kunchukuttan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

indic_nlp_library/README.md

@@ -0,0 +1,142 @@
+# Indic NLP Library
+
+The goal of the Indic NLP Library is to build Python based libraries for common text processing and Natural Language Processing in Indian languages. Indian languages share a lot of similarity in terms of script, phonology, language syntax, etc. and this library is an attempt to provide a general solution to very commonly required toolsets for Indian language text.
+
+The library provides the following functionalities:
+
+- Text Normalization
+- Script Information
+- Word Tokenization and Detokenization
+- Sentence Splitting 
+- Word Segmentation
+- Syllabification
+- Script Conversion
+- Romanization
+- Indicization
+- Transliteration
+- Translation
+
+The data resources required by the Indic NLP Library are hosted in a different repository. These resources are required for some modules. You can download from the [Indic NLP Resources](https://github.com/anoopkunchukuttan/indic_nlp_resources) project. 
+
+**If you are interested in Indian language NLP resources, you should check the [Indic NLP Catalog](https://github.com/indicnlpweb/indicnlp_catalog) for pointers.**
+
+## Pre-requisites
+
+- Python 3.x 
+   - (For Python 2.x version check the tag `PYTHON_2.7_FINAL_JAN_2019`. Not actively supporting Python 2.x anymore, but will try to maintain as much compatibility as possible)
+- [Indic NLP Resources](https://github.com/anoopkunchukuttan/indic_nlp_resources)
+- [Urduhack](https://github.com/urduhack/urduhack): Needed only if Urdu normalization is required. It has other dependencies like Tensorflow.
+- Other dependencies are listed in setup.py
+
+
+## Configuration
+
+- Installation from pip:
+
+    `pip install indic-nlp-library`
+
+- If you want to use the project from the github repo, add the project to the Python Path: 
+
+    - Clone this repository
+    - Install dependencies: `pip install -r requirements.txt`
+    - Run: `export PYTHONPATH=$PYTHONPATH:<project base directory>`
+
+- In either case, export the path to the _Indic NLP Resources_ directory
+
+    Run: `export INDIC_RESOURCES_PATH=<path to Indic NLP resources>` 
+
+## Usage 
+
+You can use the Python API to access all the features of the library. Many of the most common operations are also accessible via a unified commandline API. 
+
+### Getting Started
+
+Check [this IPython Notebook](http://nbviewer.ipython.org/url/anoopkunchukuttan.github.io/indic_nlp_library/doc/indic_nlp_examples.ipynb) for examples to use the Python API.
+  - You can find the Python 2.x Notebook [here](http://nbviewer.ipython.org/url/anoopkunchukuttan.github.io/indic_nlp_library/doc/indic_nlp_examples_2_7.ipynb) 
+
+### Documentation
+
+You can find detailed documentation  [HERE](https://indic-nlp-library.readthedocs.io/en/latest)
+
+This documents the Python API as well as the commandline reference. 
+
+## Citing
+
+If you use this library, please include the following citation: 
+
+```
+@misc{kunchukuttan2020indicnlp,
+author = "Anoop Kunchukuttan",
+title = "{The IndicNLP Library}",
+year = "2020",
+howpublished={\url{https://github.com/anoopkunchukuttan/indic_nlp_library/blob/master/docs/indicnlp.pdf}}
+}
+```
+You can find the document [HERE](docs/indicnlp.pdf)
+
+## Website
+
+`http://anoopkunchukuttan.github.io/indic_nlp_library`
+
+## Author
+Anoop Kunchukuttan ([[email protected]]([email protected]))
+
+## Companies, Organizations, Projects using IndicNLP Library
+
+- [AI4Bharat-IndicNLPSuite](https://indicnlp.ai4bharat.org)
+- [The Classical Language Toolkit](http://cltk.org)
+- [Microsoft NLP Recipes](https://github.com/microsoft/nlp-recipes)
+- [Facebook M2M-100](https://github.com/pytorch/fairseq/tree/master/examples/m2m_100)
+
+## Revision Log
+
+
+0.81 : 26 May 2021 
+    
+    - Bug fix in version number extraction
+
+0.80 : 24 May 2021 
+
+    - Improved sentence splitting
+    - Bug fixes
+    - Support for Urdu Normalizer
+
+0.71 : 03 Sep 2020 
+
+    - Improved documentation
+    - Bug fixes
+
+0.7 : 02 Apr 2020:
+
+    - Unified commandline 
+    - Improved documentation
+    - Added setup.py
+
+0.6 : 16 Dec 2019:
+
+    - New romanizer and indicizer
+    - Script Unifiers
+    - Improved script normalizers
+    - Added contrib directory for sample uses
+    - changed to MIT license 
+
+0.5 : 03 Jun 2019: 
+
+    - Improved word tokenizer to handle dates and numbers. 
+    - Added sentence splitter that can handle common prefixes/honorofics and uses some heuristics.
+    - Added detokenizer
+    - Added acronym transliterator that can convert English acronyms to Brahmi-derived scripts
+
+0.4 : 28 Jan 2019: Ported to Python 3, and lots of feature additions since last release; primarily around script information, script similarity and syllabification.
+
+0.3 : 21 Oct 2014: Supports morph-analysis between Indian languages
+
+0.2 : 13 Jun 2014: Supports transliteration between Indian languages and tokenization of Indian languages 
+
+0.1 : 12 Mar 2014: Initial version. Supports text normalization.
+
+## LICENSE
+
+Indic NLP Library is released under the MIT license
+
+

indic_nlp_library/contrib/README.md

@@ -0,0 +1,7 @@
+# Contrib
+
+Contains additional utilities and applications using Indic NLP library core
+
+- `indic_scraper_project_sample.ipynb`: A simple pipeline for building monolingual corpora for Indian languages from crawled web content, Wikipedia, etc. An extensible framework which allows incorporation of website specific extractors, whereas generic NLP tasks like tokenization, sentence splitting, normalization, etc. are handled by the framework.
+- `correct_moses_tokenizer.py`: This script corrects the incorrect tokenization done by Moses tokenizer.  The Moses tokenizer splits on nukta and halant characters.
+- `hindi_to_kannada_transliterator.py`: This script transliterates Hindi to Kannada. It removes/remaps characters only found in Hindi. It also adds halanta to words ending with consonant - as is the convention in Kannada.

indic_nlp_library/contrib/correct_moses_tokenizer.py

@@ -0,0 +1,29 @@
+import sys
+from indicnlp import langinfo
+from indicnlp import loader 
+
+if __name__ == '__main__':
+    """
+        This script corrects the incorrect tokenization done by Moses tokenizer.
+        The Moses tokenizer splits on nukta and halant characters
+        Usage: python correct_moses_tokenizer.py <infname> <outfname> <langcode>
+    """
+
+    loader.load()
+
+    infname=sys.argv[1]
+    outfname=sys.argv[2]
+    lang=sys.argv[3]
+
+    halant_char=langinfo.offset_to_char(langinfo.HALANTA_OFFSET,lang)
+    nukta_char=langinfo.offset_to_char(langinfo.NUKTA_OFFSET,lang)
+
+    with open(infname,'r',encoding='utf-8') as infile, \
+         open(outfname,'w',encoding='utf-8') as outfile:
+        for line in infile:
+            outfile.write(
+                    line.replace(
+                        ' {} '.format(halant_char), halant_char).replace(
+                        ' {} '.format(nukta_char), nukta_char).replace(
+                        ' {}{}'.format(nukta_char,halant_char),'{}{}'.format(nukta_char,halant_char))    
+                    )

indic_nlp_library/contrib/hindi_to_kannada_transliterator.py

@@ -0,0 +1,62 @@
+import sys
+from indicnlp import common
+common.set_resources_path(INDIC_NLP_RESOURCES)
+
+from indicnlp import loader
+from indicnlp.normalize import indic_normalize
+from indicnlp.transliterate import unicode_transliterate
+
+if __name__ == '__main__': 
+    """
+    This script transliterates Hindi to Kannada. It removes/remaps 
+    characters only found in Hindi. It also adds halanta to words ending
+    with consonant - as is the convention in Kannada
+    """
+
+    infname=sys.argv[1]  # one sentence/word per line. Sentences should be space-tokenized
+    outfname=sys.agv[2]
+    loader.load()
+
+    normalizer_factory=indic_normalize.IndicNormalizerFactory()
+    normalizer=normalizer_factory.get_normalizer('hi')
+
+    with open(infname,'r',encoding='utf-8') as infile, \
+         open(outfname,'w',encoding='utf-8') as outfile:
+        for line in infile: 
+            line=line.strip()
+            line=normalizer.normalize(line)
+    
+            ## replace chandrabindus with anusvara
+            line=line.replace('\u0900','\u0902')
+            line=line.replace('\u0901','\u0902')
+    
+            ### replace chandra e and o diacritics with e and o respectively
+            #line=line.replace('\u0945','\u0947')
+            #line=line.replace('\u0949','\u094b')
+   
+            ### replace chandra e and o diacritics with a diacritic
+            ## this seems to be general usage
+            line=line.replace('\u0945','\u093e')
+            line=line.replace('\u0949','\u093e')
+   
+            ## remove nukta 
+            line=line.replace('\u093c','')
+
+            ## add halant if word ends with consonant
+            #if isc.is_consonant(isc.get_phonetic_feature_vector(line[-1],'hi')):
+            #    line=line+'\u094d'
+            words=line.split(' ')
+            outwords=[]
+            for word in line.split(' '):
+                if isc.is_consonant(isc.get_phonetic_feature_vector(word[-1],'hi')):
+                    word=word+'\u094d'
+            outwords.append(word)
+            line=' '.join(outwords)
+
+    
+            ## script conversion 
+            line=unicode_transliterate.UnicodeIndicTransliterator.transliterate(line,'hi','kn')
+            
+            outfile.write(line+'\n')
+
+

indic_nlp_library/contrib/indic_scraper_project_sample.ipynb

@@ -0,0 +1,569 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Pre-requisites\n",
+    "\n",
+    "- Python 3.5+\n",
+    "- Python packages: \n",
+    "    - `pip install bs4 pandas mmh3`\n",
+    "- [Indic NLP Library](https://github.com/anoopkunchukuttan/indic_nlp_library)\n",
+    "- [Indic NLP Resources](https://github.com/anoopkunchukuttan/indic_nlp_resources)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Initialize the Indic NLP Library\n",
+    "\n",
+    "Run the cell below to initialize the Indic NLP Library"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# The path to the local git repo for Indic NLP Library\n",
+    "INDIC_NLP_LIB_HOME=\"/disk1/src/indic_nlp_library\"\n",
+    "\n",
+    "# The path to the local git repo for Indic NLP Resources\n",
+    "INDIC_NLP_RESOURCES=\"/disk1/src/indic_nlp_resources\"\n",
+    "\n",
+    "import sys\n",
+    "sys.path.append('{}/src'.format(INDIC_NLP_LIB_HOME))\n",
+    "\n",
+    "from indicnlp import common\n",
+    "common.set_resources_path(INDIC_NLP_RESOURCES)\n",
+    "\n",
+    "from indicnlp import loader\n",
+    "loader.load()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from bs4 import BeautifulSoup\n",
+    "import os\n",
+    "import string\n",
+    "import indicnlp\n",
+    "from indicnlp.tokenize import indic_tokenize\n",
+    "from indicnlp.normalize import indic_normalize\n",
+    "from indicnlp.transliterate import unicode_transliterate\n",
+    "from indicnlp.tokenize import sentence_tokenize\n",
+    "import re\n",
+    "import collections\n",
+    "import random\n",
+    "import mmh3"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Common Functions"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def preprocess_sent(text,lang,normalizer):\n",
+    "    \"\"\"\n",
+    "        Pre-process text (normalization and tokenization)\n",
+    "        \n",
+    "        text: text string to preprocess\n",
+    "        lang: language code (2-letter ISO code)\n",
+    "        normalizer: normalizer object for language\n",
+    "        \n",
+    "        returns the processed text string\n",
+    "    \"\"\"\n",
+    "    return ' '.join(indic_tokenize.trivial_tokenize(normalizer.normalize(text.replace('\\n',' ')),lang)) \n",
+    "\n",
+    "def sent_split(text,lang):\n",
+    "    \"\"\"\n",
+    "        Sentence splitter\n",
+    "        \n",
+    "        text: text to sentence split \n",
+    "        lang: language\n",
+    "        \n",
+    "        returns list of sentences \n",
+    "    \"\"\"\n",
+    "    return  sentence_tokenize.sentence_split(text,lang)\n",
+    "\n",
+    "def extract_all_content(indir,lang,\n",
+    "                        article_extract_fn,\n",
+    "                        preprocess_fn=preprocess_sent,\n",
+    "                        narticles=-1,\n",
+    "                        start_artid=0):\n",
+    "    \"\"\"\n",
+    "    This method reads all files from the input directory, extracts text content from each file,\n",
+    "    and pre-processes the text. This method is a generator. \n",
+    "    For each sentence, the method yields a tuple of the format: \n",
+    "    \n",
+    "    (artid, fname, paraid, sentid, processed_text)\n",
+    "    \n",
+    "    indir: path to input directoryo containing files to be parsed \n",
+    "    \n",
+    "    lang: language to the files in the input directory\n",
+    "    \n",
+    "    article_extract_fn: the function to extract text content from each file. \n",
+    "        Signature of the function: get_article_contents(fname,lang,encoding) \n",
+    "          `fname` is name of the file, `lang` is langcode, \n",
+    "          `encoding` is text-encoding (default=utf-8). \n",
+    "        The function yields a tuple (paraid, sentid, extracted_text) \n",
+    "        for each  sentence.\n",
+    "        \n",
+    "    preprocess_fn: pre-processing function to apply to the extracted text. \n",
+    "        The function takes a string as input and returns processed string as output.\n",
+    "        \n",
+    "    narticles: extract and process the first `narticles` from input directory. \n",
+    "        if narticles=-1 (default), all files are extracted\n",
+    "        \n",
+    "    start_artid: the start of the article id to assign to extracted articles (default=0)\n",
+    "    \n",
+    "    \"\"\"\n",
+    "\n",
+    "    fnames = os.listdir(indir)\n",
+    "    if narticles>0:\n",
+    "        fnames=fnames[:narticles]\n",
+    "    nsent=0\n",
+    "\n",
+    "    normalizer_factory=indic_normalize.IndicNormalizerFactory()\n",
+    "    normalizer=normalizer_factory.get_normalizer(lang)\n",
+    "             \n",
+    "    print('Number of articles: {}'.format(len(fnames)))\n",
+    "    for artid, fname in enumerate(fnames,start_artid):\n",
+    "#         print(fname)\n",
+    "        if artid%100 == 0:\n",
+    "            print('({}|{})'.format(artid,nsent),end=' ... ')\n",
+    "        \n",
+    "        try:\n",
+    "            fpath=os.sep.join([indir,fname])\n",
+    "            for paraid, sentid, sent in article_extract_fn(fpath,lang):\n",
+    "                nsent+=1\n",
+    "                yield( ( artid, fname, paraid, sentid, preprocess_fn(sent,lang,normalizer) ) )\n",
+    "        except:\n",
+    "            print('Cannot parse {}'.format(fname))\n",
+    "                \n",
+    "def write_corpus(corpus_iterator,content_fname,article_mapping_fname,delimiter=' ||| ', encoding='utf-8'):\n",
+    "    \"\"\"\n",
+    "    Writes the extracted corpus to a file. The extracted data is organized in terms of articles, paragraphs \n",
+    "    and sentences. The following is the format of the output file: \n",
+    "        - one line per sentence\n",
+    "        - format of line: article_id, para_id, sent_id, sentence\n",
+    "    In addition to the content file mention, a metadata file which maps the article id to the filename is also written. \n",
+    "    \n",
+    "    corpus_iterator: iterator over the corpus, yielding tuple (artid, fname, paraid, sentid, processed_text). \n",
+    "        The function `extract_all_content` yields a generator in this format. \n",
+    "    content_fname: output content file to write the extracted data to in the format mentioned above\n",
+    "    article_mapping_fname: output metadata file to write article id to filename mapping.\n",
+    "    delimiter=' ||| ': delimiter for the content file. The default delimiter is the same \n",
+    "                        as used in the Moses phrase table\n",
+    "    encoding: text encoding default - 'utf-8'\n",
+    "    \n",
+    "    \"\"\"\n",
+    "    \n",
+    "    artid_name_mapping={}\n",
+    "    with open(content_fname,'w',encoding=encoding) as contentfile:\n",
+    "        for artid, fname, paraid, sentid, text in corpus_iterator:\n",
+    "            contentfile.write(delimiter.join([str(artid), str(paraid), str(sentid), text]) + '\\n')\n",
+    "            artid_name_mapping[artid]=fname\n",
+    "\n",
+    "    with open(article_mapping_fname,'w',encoding=encoding) as artmappingfile:\n",
+    "        for artid, name in sorted(artid_name_mapping.items(),key=lambda x: x[0]):\n",
+    "            artmappingfile.write('{} {} {}\\n'.format(artid,delimiter,name))\n",
+    "\n",
+    "def convert_txt_to_csv_format(infname, outfname, encoding='utf-8'):\n",
+    "    \"\"\"\n",
+    "    convert txt file to csv format. This method is used when the text file is directly available.\n",
+    "    The input file has one sentence per line. Assumed to be preprocessed (tokenized, normalized)\n",
+    "    \n",
+    "    \"\"\"\n",
+    "    with open(infname,'r',encoding=encoding) as infile, \\\n",
+    "         open(outfname,'w',encoding=encoding) as outfile: \n",
+    "        for i, line in enumerate(infile):\n",
+    "            outfile.write('0 ||| 0 ||| {} ||| {}\\n'.format(i,line.strip()))\n",
+    "            \n",
+    "def preprocess_convert_txt_to_csv_format(infname, outfname, lang, encoding='utf-8'):\n",
+    "    \"\"\"\n",
+    "    Convert raw text file to csv format\n",
+    "    \"\"\"\n",
+    "    \n",
+    "    normalizer_factory=indic_normalize.IndicNormalizerFactory()\n",
+    "    normalizer=normalizer_factory.get_normalizer(lang)\n",
+    "    \n",
+    "    with open(infname,'r',encoding=encoding) as infile, \\\n",
+    "         open(outfname,'w',encoding=encoding) as outfile: \n",
+    "        i=0\n",
+    "        for line in infile:\n",
+    "            sents = sent_split(line.strip(),lang)\n",
+    "            for sent in sents:\n",
+    "                outfile.write('0 ||| 0 ||| {} ||| {}\\n'.format(i,\n",
+    "                                                    preprocess_sent(sent.strip(), lang, normalizer)) )\n",
+    "                i=i+1\n",
+    "\n",
+    "def print_txt(infnames, outfname, encoding='utf-8'):\n",
+    "    \"\"\"\n",
+    "    Extract only the text from the content csv file. The output file has one sentence per file.\n",
+    "    \"\"\"\n",
+    "    with open(outfname,'w',encoding=encoding) as outfile: \n",
+    "        for infname in filter(lambda x: os.path.isfile(x),infnames):\n",
+    "            with open(infname,'r',encoding=encoding) as infile:\n",
+    "                for i, line in enumerate(infile):\n",
+    "                    fields=line.strip().split('|||')\n",
+    "                    if len(fields) >=4:\n",
+    "                        outfile.write('{}\\n'.format(fields[3].strip()))\n",
+    "                        \n",
+    "# def dedup_and_print_txt(infnames, outfname, encoding='utf-8'):\n",
+    "    \n",
+    "#     total=0\n",
+    "#     unique=0\n",
+    "#     hash_codes=set()\n",
+    "    \n",
+    "#     with open(outfname,'w',encoding=encoding) as outfile: \n",
+    "#         for infname in filter(lambda x: os.path.isfile(x),infnames):\n",
+    "#             with open(infname,'r',encoding=encoding) as infile:\n",
+    "#                 for i, line in enumerate(infile):\n",
+    "#                     fields=line.strip().split('|||')\n",
+    "#                     if len(fields) >=4:\n",
+    "#                         sent=fields[3].strip()\n",
+    "#                         total+=1\n",
+    "#                         hs=hash(sent)\n",
+    "#                         if hs not in hash_codes:\n",
+    "#                             outfile.write('{}\\n'.format(sent))\n",
+    "#                             hash_codes.add(hs)\n",
+    "#                             unique+=1\n",
+    "    \n",
+    "#     print('Total: {}'.format(total))\n",
+    "#     print('Unique: {}'.format(unique))\n",
+    "\n",
+    "def dedup_shuffle_and_print_txt(infnames, outfname, max_buf_size=100000,encoding='utf-8'):\n",
+    "    \"\"\"\n",
+    "    The method creates a sentence level corpora from multiple content csv files.\n",
+    "    All sentences are extracted, they are de-duplicated using murmurhash and shuffled\n",
+    "    before writing the entire corpus to the output file. The output file has one sentence per line.\n",
+    "\n",
+    "    \"\"\"\n",
+    "    \n",
+    "    total=0\n",
+    "    unique=0\n",
+    "    hash_codes=set()\n",
+    "    sent_buffer=[]\n",
+    "    \n",
+    "    with open(outfname,'w',encoding=encoding) as outfile: \n",
+    "        for infname in filter(lambda x: os.path.isfile(x),infnames):\n",
+    "            print('Processing: {}'.format(infname))\n",
+    "            with open(infname,'r',encoding=encoding) as infile:\n",
+    "                for i, line in enumerate(infile):\n",
+    "                    fields=line.strip().split('|||')\n",
+    "                    if len(fields) >=4:\n",
+    "                        sent=fields[3].strip()\n",
+    "                        total+=1\n",
+    "#                         hs=hash(sent)\n",
+    "                        hs=mmh3.hash128(sent)\n",
+    "                        if hs not in hash_codes:\n",
+    "#                             outfile.write('{}\\n'.format(sent))\n",
+    "                            sent_buffer.append(sent)\n",
+    "                            hash_codes.add(hs)\n",
+    "                            unique+=1\n",
+    "                    if len(sent_buffer)>=max_buf_size:\n",
+    "                        random.shuffle(sent_buffer)\n",
+    "                        for sent in sent_buffer: \n",
+    "                            outfile.write('{}\\n'.format(sent))\n",
+    "                        sent_buffer.clear()\n",
+    "                \n",
+    "        if len(sent_buffer)>0:\n",
+    "            random.shuffle(sent_buffer)\n",
+    "            for sent in sent_buffer: \n",
+    "                outfile.write('{}\\n'.format(sent))\n",
+    "            sent_buffer.clear()                    \n",
+    "                        \n",
+    "    print('Total: {}'.format(total))\n",
+    "    print('Unique: {}'.format(unique))\n",
+    "\n",
+    "def extract_wikiextractor_file(infname, outfname, lang, \n",
+    "                               encoding='utf-8', delimiter=' ||| ', preprocess_fn=preprocess_sent):\n",
+    "    \"\"\"\n",
+    "    Extract text content into a content csv file from wikipedia article page. \n",
+    "    The wikipedia article page is the output from `wikiextractor` [https://github.com/attardi/wikiextractor] \n",
+    "    \n",
+    "    \"\"\"\n",
+    "    normalizer_factory=indic_normalize.IndicNormalizerFactory()\n",
+    "    normalizer=normalizer_factory.get_normalizer(lang)\n",
+    "    \n",
+    "    with open(infname,'r',encoding=encoding) as infile, \\\n",
+    "         open(outfname,'w',encoding=encoding) as outfile: \n",
+    "        artid=-1\n",
+    "        paraid=0\n",
+    "        for line in infile:\n",
+    "            if line.find('<doc')==0:\n",
+    "                artid+=1\n",
+    "                paraid=0\n",
+    "                continue\n",
+    "            if line.find('</doc')==0:\n",
+    "                continue\n",
+    "            if len(line.strip())>0:\n",
+    "                for sentid, sent in enumerate(sent_split(line.strip(),lang)):\n",
+    "                    sent=sent.strip()\n",
+    "                    if sent!='':\n",
+    "                        sent = preprocess_fn(sent,lang,normalizer)\n",
+    "                        outfile.write(delimiter.join([str(artid), str(paraid), str(sentid), sent]) + '\\n')\n",
+    "                paraid+=1\n",
+    "\n",
+    "                \n",
+    "def extract_leipzig_corpus(infname,outfname,lang,encoding='utf-8'):\n",
+    "    \"\"\"\n",
+    "    Extractor for files form the Leipzig corpus\n",
+    "    [http://wortschatz.uni-leipzig.de/en/download/]\n",
+    "    \n",
+    "    \"\"\"\n",
+    "    normalizer_factory=indic_normalize.IndicNormalizerFactory()\n",
+    "    normalizer=normalizer_factory.get_normalizer(lang)    \n",
+    "\n",
+    "    with open(infname,'r',encoding=encoding) as infile, \\\n",
+    "         open(outfname,'w',encoding=encoding) as outfile: \n",
+    "        for i, line in enumerate(infile):\n",
+    "            outfile.write('0 ||| 0 ||| {} ||| {}\\n'.format(i,preprocess_sent(line,lang,normalizer)))                \n",
+    "                \n",
+    "def dataset_stats(fname):\n",
+    "    \"\"\"\n",
+    "    Extracts dataset statistics from the final extracted file. This input file contains\n",
+    "    one sentence per line. The sentences are tokenized.\n",
+    "    \"\"\"\n",
+    "\n",
+    "    all_puncs=set(string.punctuation+'\\u0964\\u0965')\n",
+    "    \n",
+    "    sent_count=0\n",
+    "    token_cnt=0\n",
+    "    true_token_cnt=0\n",
+    "    tokens=set()\n",
+    "    \n",
+    "    with open(fname,'r',encoding='utf-8') as infile:\n",
+    "        for line in infile:\n",
+    "            sent_count+=1\n",
+    "            a=line.strip().split(' ')\n",
+    "            token_cnt+=len(a)\n",
+    "            b=list(filter(lambda x: x not in all_puncs,a))\n",
+    "            true_token_cnt+=len(b)\n",
+    "            tokens.update(b)\n",
+    "    \n",
+    "    print('== Stats ==')\n",
+    "    print('Sent count: {}'.format(sent_count))\n",
+    "    print('Token count: {}'.format(token_cnt))\n",
+    "    print('True Token count: {}'.format(true_token_cnt))\n",
+    "    print('Unique Token count: {}'.format(len(tokens)))\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Marathi"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Wikipedia"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Wikipedia extraction commands using wikiextractor\n",
+    "\n",
+    "```\n",
+    "### This uses WikiExtractor (https://github.com/attardi/wikiextractor)\n",
+    "\n",
+    "x=/disk1/crawl_project/ta/wikipedia\n",
+    "mkdir $x\n",
+    "cd $x\n",
+    "wget https://dumps.wikimedia.org/tawiki/20190501/tawiki-20190501-pages-articles-multistream.xml.bz2\n",
+    "cd /disk1/src/wikiextractor\n",
+    "python3 WikiExtractor.py -cb 250k -o $x/extracted $x/tawiki-20190501-pages-articles-multistream.xml.bz2\n",
+    "cd -\n",
+    "find extracted -name '*bz2' -exec bunzip2 -c {} \\; > text.xml\n",
+    "rm text.xml\n",
+    "rm tawiki-20190501-pages-articles-multistream.xml.bz2\n",
+    "rm -rf extracted\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "mrwiki-20190401-pages-articles-multistream.xml.bz2\n",
+    "\n",
+    "INFO: Finished 1-process extraction of 53715 articles in 123.6s (434.7 art/s)\n",
+    "\n",
+    "INFO: total of page: 102025, total of articl page: 53715; total of used articl page: 53715"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Post-processing output generated by wikiextractor"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## tex.xml is extracted as shown in commanfs above\n",
+    "extract_wikiextractor_file('text.xml',\n",
+    "                           'content_fname1.csv',\n",
+    "                           'mr')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Loksatta"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "**Extractor function for Marathi Loksatta page**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_article_contents_mr_loksatta(fname,lang,encoding='utf-8'):\n",
+    "    with open(fname,'r',encoding=encoding) as infile: \n",
+    "        soup = BeautifulSoup(infile)\n",
+    "        for elem in soup.find_all('div'):\n",
+    "            if 'itemprop' in elem.attrs and 'articleBody' in elem['itemprop']:\n",
+    "                filtered_paras=list(filter(lambda x: x.name=='p' and len(x.attrs)==0,elem.children))\n",
+    "                paraid=0\n",
+    "                for blockid, block in enumerate(filtered_paras):\n",
+    "#                     print('Para: {}'.format(blockid))\n",
+    "#                     print(list(block.strings))\n",
+    "                    text=' '.join(block.strings)\n",
+    "                    if blockid==0 and text.find(':')>=0 and text.find(':')<20:\n",
+    "                        text=':'.join(text.split(':')[1:])\n",
+    "                    for para_text in text.split('\\n'): \n",
+    "                        for sentid, sent in enumerate(sent_split(para_text,lang)):\n",
+    "                            sent=sent.strip()\n",
+    "                            if sent!='':\n",
+    "    #                             print('{}: {}'.format(sentid, sent))\n",
+    "                                yield((paraid,sentid,sent))\n",
+    "    #                             yield((paraid,sentid,preprocess_sent(sent,'ml',normalizer)))\n",
+    "    #                     print()   \n",
+    "                        paraid+=1"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "**Extracting data from crawled HTML files**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "lang='mr'\n",
+    "posts_dir='directory_containing_crawled_html_pages'\n",
+    "content_fname='content_fname2.csv'\n",
+    "article_mapping_fname='article_mapping_fname'\n",
+    "get_article_contents=get_article_contents_mr_loksatta\n",
+    "narticles=-1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "write_corpus(\n",
+    "             extract_all_content(posts_dir, lang, article_extract_fn=get_article_contents,narticles=narticles),\n",
+    "             content_fname,\n",
+    "             article_mapping_fname\n",
+    "            )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Aggregating all crawled data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### aggregating, de-duplicating and shuffling all the data \n",
+    "dedup_shuffle_and_print_txt([ 'content_fname1.csv', 'content_fname2.csv'  ],  'output_fname.txt' )\n",
+    "### extract dataset statistics\n",
+    "dataset_stats('output_fname.txt')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.7"
+  },
+  "toc": {
+   "base_numbering": 1,
+   "nav_menu": {
+    "height": "703px",
+    "width": "326px"
+   },
+   "number_sections": true,
+   "sideBar": true,
+   "skip_h1_title": false,
+   "title_cell": "Table of Contents",
+   "title_sidebar": "Contents",
+   "toc_cell": false,
+   "toc_position": {},
+   "toc_section_display": true,
+   "toc_window_display": false
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

indic_nlp_library/docs/Makefile

@@ -0,0 +1,153 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+PAPER         =
+BUILDDIR      = _build
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
+
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  singlehtml to make a single large HTML file"
+	@echo "  pickle     to make pickle files"
+	@echo "  json       to make JSON files"
+	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  qthelp     to make HTML files and a qthelp project"
+	@echo "  devhelp    to make HTML files and a Devhelp project"
+	@echo "  epub       to make an epub"
+	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
+	@echo "  text       to make text files"
+	@echo "  man        to make manual pages"
+	@echo "  texinfo    to make Texinfo files"
+	@echo "  info       to make Texinfo files and run them through makeinfo"
+	@echo "  gettext    to make PO message catalogs"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+
+clean:
+	-rm -rf $(BUILDDIR)/*
+
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	@echo
+	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+	@echo
+	@echo "Build finished; now you can process the pickle files."
+
+json:
+	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+	@echo
+	@echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+	@echo
+	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
+	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/IndicNLPLibrary.qhcp"
+	@echo "To view the help file:"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/IndicNLPLibrary.qhc"
+
+devhelp:
+	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+	@echo
+	@echo "Build finished."
+	@echo "To view the help file:"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/IndicNLPLibrary"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/IndicNLPLibrary"
+	@echo "# devhelp"
+
+epub:
+	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+	@echo
+	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+latex:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Run \`make' in that directory to run these through (pdf)latex" \
+	      "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through pdflatex..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+	@echo
+	@echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+man:
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+	@echo
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo
+	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Run \`make' in that directory to run these through makeinfo" \
+	      "(use \`make info' here to do that automatically)."
+
+info:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo "Running Texinfo files through makeinfo..."
+	make -C $(BUILDDIR)/texinfo info
+	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+gettext:
+	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+	@echo
+	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."

indic_nlp_library/docs/cmd.rst

@@ -0,0 +1,8 @@
+Commandline
+===========
+
+.. argparse::
+   :module: indicnlp.cli.cliparser
+   :func: get_parser
+   :prog: cliparser.py
+

indic_nlp_library/docs/code.rst

@@ -0,0 +1,5 @@
+Auto Generated Documentation
+============================
+
+.. automodule:: indicnlp.langinfo indicnlp.common
+   :members: 

indic_nlp_library/docs/conf.py

@@ -0,0 +1,242 @@
+# -*- coding: utf-8 -*-
+#
+# Indic NLP Library documentation build configuration file, created by
+# sphinx-quickstart on Tue Nov  3 01:50:37 2015.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys, os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+sys.path.insert(0, os.path.abspath('..'))
+
+# -- General configuration -----------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.mathjax', 'sphinx.ext.viewcode', 'sphinx.ext.napoleon', 'sphinxarg.ext']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = 'Indic NLP Library'
+copyright = '2015, Anoop Kunchukuttan'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '0.2'
+# The full version, including alpha/beta/rc tags.
+release = '0.2'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ['_build']
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'sphinx_rtd_theme'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'IndicNLPLibrarydoc'
+
+
+# -- Options for LaTeX output --------------------------------------------------
+
+latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',
+
+# The font size ('10pt', '11pt' or '12pt').
+#'pointsize': '10pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, documentclass [howto/manual]).
+latex_documents = [
+  ('index', 'IndicNLPLibrary.tex', 'Indic NLP Library Documentation',
+   'Anoop Kunchukuttan', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output --------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    ('index', 'indicnlplibrary', 'Indic NLP Library Documentation',
+     ['Anoop Kunchukuttan'], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# -- Options for Texinfo output ------------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+  ('index', 'IndicNLPLibrary', 'Indic NLP Library Documentation',
+   'Anoop Kunchukuttan', 'IndicNLPLibrary', 'NLP library for Indian languages',
+   'NLP'),
+]
+
+# Documents to append as an appendix to all manuals.
+#texinfo_appendices = []
+
+# If false, no module index is generated.
+#texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#texinfo_show_urls = 'footnote'

indic_nlp_library/docs/index.rst

@@ -0,0 +1,22 @@
+.. Indic NLP Library documentation master file, created by
+   sphinx-quickstart on Tue Nov  3 01:50:37 2015.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+:github_url: https://github.com/anoopkunchukuttan/indic_nlp_library
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Packages
+
+  indicnlp
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Commandline
+
+  cmd
+
+
+
+

indic_nlp_library/docs/indicnlp.MD

@@ -0,0 +1,122 @@
+# Indic NLP Library
+##  A unified approach to NLP for Indian languages
+
+### Anoop Kunchukuttan (`[email protected]`)
+
+The goal of the Indic NLP Library is to build Python based libraries for common text processing and Natural Language Processing in Indian languages. Indian languages share a lot of similarity in terms of script, phonology, language syntax, etc. and this library is an attempt to provide a general solution to very commonly required toolsets for Indian language text.
+
+The library provides the following functionalities:
+
+- Text Normalization
+- Script Information
+- Word Tokenization and Detokenization
+- Sentence Splitting 
+- Word Segmentation
+- Syllabification
+- Script Conversion
+- Romanization
+- Indicization
+- Transliteration
+- Translation
+
+The data resources required by the Indic NLP Library are hosted in a different repository. These resources are required for some modules. You can download from the [Indic NLP Resources](https://github.com/anoopkunchukuttan/indic_nlp_resources) project. 
+
+**If you are interested in Indian language NLP resources, you should check the [Indic NLP Catalog](https://github.com/anoopkunchukuttan/indic_nlp_library) for pointers.**
+
+## Pre-requisites
+
+- Python 3.x 
+   - (For Python 2.x version check the tag `PYTHON_2.7_FINAL_JAN_2019`. Not actively supporting Python 2.x anymore, but will try to maintain as much compatibility as possible)
+- [Indic NLP Resources](https://github.com/anoopkunchukuttan/indic_nlp_resources)
+- Other dependencies are listed in setup.py
+
+
+## Configuration
+
+- Installation from pip:
+
+    `pip install indic-nlp-library`
+
+- If you want to use the project from the github repo, add the project to the Python Path: 
+
+    - Clone this repository
+    - Install dependencies: `pip install -r requirements.txt`
+    - Run: `export PYTHONPATH=$PYTHONPATH:<project base directory>`
+
+- In either case, export the path to the _Indic NLP Resources_ directory
+
+    Run: `export INDIC_RESOURCES_PATH=<path to Indic NLP resources>` 
+
+## Usage 
+
+You can use the Python API to access all the features of the library. Many of the most common operations are also accessible via a unified commandline API. 
+
+### Getting Started
+
+Check [this IPython Notebook](http://nbviewer.ipython.org/url/anoopkunchukuttan.github.io/indic_nlp_library/doc/indic_nlp_examples.ipynb) for examples to use the Python API.
+  - You can find the Python 2.x Notebook [here](http://nbviewer.ipython.org/url/anoopkunchukuttan.github.io/indic_nlp_library/doc/indic_nlp_examples_2_7.ipynb) 
+
+### Documentation
+
+You can find detailed documentation  [HERE](https://indic-nlp-library.readthedocs.io/en/latest)
+
+This documents the Python API as well as the commandline reference. 
+
+## Citing
+
+If you use this library, please include the following citation: 
+
+```
+@unpublished{kunchukuttan2020indicnlp,
+author = "Anoop Kunchukuttan",
+title = "The IndicNLP Library",
+year = "2020",
+}
+```
+You can find the document [HERE](docs/indicnlp.pdf)
+
+## Website
+
+`http://anoopkunchukuttan.github.io/indic_nlp_library`
+
+## Author
+Anoop Kunchukuttan ([[email protected]]([email protected]))
+
+## Version: 0.7
+
+## Revision Log
+
+0.7 : 02 Apr 2020:
+
+    - Unified commandline 
+    - Improved documentation
+    - Added setup.py
+
+0.6 : 16 Dec 2019:
+
+    - New romanizer and indicizer
+    - Script Unifiers
+    - Improved script normalizers
+    - Added contrib directory for sample uses
+    - changed to MIT license 
+
+0.5 : 03 Jun 2019: 
+
+    - Improved word tokenizer to handle dates and numbers. 
+    - Added sentence splitter that can handle common prefixes/honorofics and uses some heuristics.
+    - Added detokenizer
+    - Added acronym transliterator that can convert English acronyms to Brahmi-derived scripts
+
+0.4 : 28 Jan 2019: Ported to Python 3, and lots of feature additions since last release; primarily around script information, script similarity and syllabification.
+
+0.3 : 21 Oct 2014: Supports morph-analysis between Indian languages
+
+0.2 : 13 Jun 2014: Supports transliteration between Indian languages and tokenization of Indian languages 
+
+0.1 : 12 Mar 2014: Initial version. Supports text normalization.
+
+## LICENSE
+
+Indic NLP Library is released under the MIT license
+
+

indic_nlp_library/docs/indicnlp.cli.rst

@@ -0,0 +1,11 @@
+cli Package
+=============
+
+:mod:`cliparser` Module
+--------------------------------
+
+.. automodule:: indicnlp.cli.cliparser
+    :members:
+    :undoc-members:
+    :show-inheritance:
+

indic_nlp_library/docs/indicnlp.morph.rst

@@ -0,0 +1,11 @@
+morph Package
+=============
+
+:mod:`unsupervised_morph` Module
+--------------------------------
+
+.. automodule:: indicnlp.morph.unsupervised_morph
+    :members:
+    :undoc-members:
+    :show-inheritance:
+

indic_nlp_library/docs/indicnlp.normalize.rst

@@ -0,0 +1,15 @@
+normalize Package
+=================
+
+:mod:`indic_normalize` Module
+-----------------------------
+
+.. automodule:: indicnlp.normalize.indic_normalize
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+.. autoclass:: indicnlp.normalize.indic_normalize.
+    :members:
+    :undoc-members:
+    :show-inheritance:

indic_nlp_library/docs/indicnlp.rst

@@ -0,0 +1,47 @@
+indicnlp Package
+================
+
+:mod:`common` Module
+--------------------
+
+.. automodule:: indicnlp.common
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`langinfo` Module
+----------------------
+
+.. automodule:: indicnlp.langinfo
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`loader` Module
+--------------------
+
+.. automodule:: indicnlp.loader
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+
+    indicnlp.cli
+    indicnlp.morph
+    indicnlp.normalize
+    indicnlp.script
+    indicnlp.syllable
+    indicnlp.tokenize
+    indicnlp.transliterate
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+

indic_nlp_library/docs/indicnlp.script.rst

@@ -0,0 +1,26 @@
+script Package
+==============
+
+:mod:`indic_scripts` Module
+---------------------------
+
+.. automodule:: indicnlp.script.indic_scripts
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`english_script` Module
+---------------------------
+
+.. automodule:: indicnlp.script.english_script
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`phonetic_sim` Module
+---------------------------
+
+.. automodule:: indicnlp.script.phonetic_sim
+    :members: 
+    :undoc-members:
+    :show-inheritance:

indic_nlp_library/docs/indicnlp.syllable.rst

@@ -0,0 +1,11 @@
+syllable Package
+==============
+
+:mod:`syllabifier` Module
+---------------------------
+
+.. automodule:: indicnlp.syllable.syllabifier
+    :members:
+    :undoc-members:
+    :show-inheritance:
+

indic_nlp_library/docs/indicnlp.tokenize.rst

@@ -0,0 +1,26 @@
+tokenize Package
+================
+
+:mod:`indic_tokenize` Module
+----------------------------
+
+.. automodule:: indicnlp.tokenize.indic_tokenize
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`indic_detokenize` Module
+------------------------------
+
+.. automodule:: indicnlp.tokenize.indic_detokenize
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`sentence_tokenize` Module
+----------------------------
+
+.. automodule:: indicnlp.tokenize.sentence_tokenize
+    :members:
+    :undoc-members:
+    :show-inheritance:

indic_nlp_library/docs/indicnlp.transliterate.rst

@@ -0,0 +1,34 @@
+transliterate Package
+=====================
+
+:mod:`sinhala_transliterator` Module
+------------------------------------
+
+.. automodule:: indicnlp.transliterate.sinhala_transliterator
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`unicode_transliterate` Module
+-----------------------------------
+
+.. automodule:: indicnlp.transliterate.unicode_transliterate
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`acronym_transliterator` Module
+-----------------------------------
+
+.. automodule:: indicnlp.transliterate.acronym_transliterator
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`script_unifier` Module
+-----------------------------------
+
+.. automodule:: indicnlp.transliterate.script_unifier
+    :members:
+    :undoc-members:
+    :show-inheritance:

indic_nlp_library/docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

indic_nlp_library/docs/modules.rst

@@ -0,0 +1,7 @@
+indicnlp
+===
+
+.. toctree::
+   :maxdepth: 4
+
+   indicnlp

indic_nlp_library/indicnlp/__init__.py

@@ -0,0 +1,10 @@
+import os
+import sys
+
+try:
+    from .version import __version__  # noqa
+except ImportError:
+    version_txt = os.path.join(os.path.dirname(__file__), "version.txt")
+    with open(version_txt) as f:
+        __version__ = f.read().strip()
+

indic_nlp_library/indicnlp/cli/cliparser.py

@@ -0,0 +1,266 @@
+import argparse 
+import sys
+
+from indicnlp import loader
+from indicnlp.tokenize import indic_tokenize
+from indicnlp.tokenize import indic_detokenize
+from indicnlp.normalize import indic_normalize
+from indicnlp.morph import unsupervised_morph
+from indicnlp.tokenize import sentence_tokenize
+from indicnlp.syllable import  syllabifier
+from indicnlp.transliterate import unicode_transliterate
+from indicnlp.transliterate import script_unifier
+
+DEFAULT_ENCODING='utf-8'
+
+def run_detokenize(args):
+    for line in args.infile:
+        args.outfile.write(indic_detokenize.trivial_detokenize(line,args.lang))
+
+def run_tokenize(args):
+    for line in args.infile:
+        args.outfile.write(' '.join(
+            indic_tokenize.trivial_tokenize(line,args.lang)))
+
+def run_sentence_split(args):            
+    text=' '.join([  l.replace('\n','').replace('\r','') for l in args.infile])
+    outlines=sentence_tokenize.sentence_split(text,args.lang)
+    for line in outlines:
+        args.outfile.write(line+'\n')
+
+def run_normalize(args):
+
+    # TODO: add more options to cli
+    remove_nuktas=False
+    normalize_nasals='do_nothing'
+
+    # create normalizer
+    factory=indic_normalize.IndicNormalizerFactory()
+    normalizer=factory.get_normalizer(args.lang,
+            remove_nuktas=remove_nuktas,
+            nasals_mode=normalize_nasals)
+
+    # DO normalization 
+    for line in args.infile:
+        normalized_line=normalizer.normalize(line)
+        args.outfile.write(normalized_line)
+
+def run_morph(args):
+
+    add_marker=False
+    analyzer=unsupervised_morph.UnsupervisedMorphAnalyzer(args.lang,add_marker)
+    for line in args.infile:
+        morph_tokens=analyzer.morph_analyze_document(line.strip().split(' '))
+        args.outfile.write(' '.join(morph_tokens) + '\n')
+
+def run_syllabify(args):
+    for line in args.infile:
+        new_line = ' '.join(
+                        [ ' '.join(syllabifier.orthographic_syllabify(w,args.lang)) 
+                                for w in line.strip().split(' ')  ]
+                    )
+        args.outfile.write(new_line+'\n')
+
+def run_wc(args):
+    # if args.l==False and args.w==False and args.c==False:
+    #     args.l, args.w, args.c= True, True, True 
+    
+    nl=0
+    nw=0
+    nc=0
+
+    for line in args.infile:
+        nl+=1
+        nw+=len(line.strip(' ').split(' '))
+        nc+=len(line)
+
+    print('{} {} {}'.format(nl,nw,nc))
+
+def run_indic2roman(args):
+    for line in args.infile:
+        transliterated_line=unicode_transliterate.ItransTransliterator.to_itrans(
+            line,args.lang)
+        args.outfile.write(transliterated_line)      
+
+def run_roman2indic(args):
+    for line in args.infile:
+        transliterated_line=unicode_transliterate.ItransTransliterator.from_itrans(
+            line,args.lang)
+        args.outfile.write(transliterated_line)    
+
+def run_script_unify(args):
+
+    unifier=None
+
+    if args.mode=='aggressive':
+        unifier=script_unifier.AggressiveScriptUnifier(nasals_mode='to_anusvaara_relaxed', common_lang=args.common_lang)
+
+    elif args.mode=='basic':
+        unifier=script_unifier.BasicScriptUnifier(nasals_mode='do_nothing',
+                                common_lang=args.common_lang)
+
+    elif args.mode=='naive':
+        unifier=script_unifier.NaiveScriptUnifier(common_lang=args.common_lang)
+
+    assert(unifier is not None)
+
+    for line in args.infile:
+        transliterated_line=unifier.transform(line,args.lang)
+        args.outfile.write(transliterated_line)     
+
+def run_script_convert(args):
+    for line in args.infile:
+        transliterated_line=unicode_transliterate.UnicodeIndicTransliterator.transliterate(
+            line,args.srclang,args.tgtlang)
+        args.outfile.write(transliterated_line)    
+
+def add_common_monolingual_args(task_parser):
+    task_parser.add_argument('infile', 
+                type=argparse.FileType('r',encoding=DEFAULT_ENCODING),
+                nargs='?',
+                default=sys.stdin,
+                help='Input File path',
+            )
+    task_parser.add_argument('outfile', 
+                type=argparse.FileType('w',encoding=DEFAULT_ENCODING),
+                nargs='?',
+                default=sys.stdout,                
+                help='Output File path',
+            )
+    task_parser.add_argument('-l', '--lang', 
+                help='Language',
+            )
+
+def add_common_bilingual_args(task_parser):
+    task_parser.add_argument('infile', 
+                type=argparse.FileType('r',encoding=DEFAULT_ENCODING),
+                nargs='?',
+                default=sys.stdin,
+                help='Input File path',
+            )
+    task_parser.add_argument('outfile', 
+                type=argparse.FileType('w',encoding=DEFAULT_ENCODING),
+                nargs='?',
+                default=sys.stdout,                
+                help='Output File path',
+            )
+    task_parser.add_argument('-s', '--srclang', 
+                help='Source Language',
+            )
+
+    task_parser.add_argument('-t', '--tgtlang', 
+                help='Target Language',
+            )
+
+def add_tokenize_parser(subparsers):
+    task_parser=subparsers.add_parser('tokenize', 
+                    help='tokenizer help')
+    add_common_monolingual_args(task_parser)
+    task_parser.set_defaults(func=run_tokenize)
+
+def add_detokenize_parser(subparsers):
+    task_parser=subparsers.add_parser('detokenize', 
+                    help='de-tokenizer help')
+    add_common_monolingual_args(task_parser)
+    task_parser.set_defaults(func=run_detokenize)
+
+def add_sentence_split_parser(subparsers):
+    task_parser=subparsers.add_parser('sentence_split', help='sentence split help')
+    add_common_monolingual_args(task_parser)
+    task_parser.set_defaults(func=run_sentence_split)
+
+def add_normalize_parser(subparsers):
+    task_parser=subparsers.add_parser('normalize', help='normalizer help')
+    add_common_monolingual_args(task_parser)
+    task_parser.set_defaults(func=run_normalize)
+
+def add_morph_parser(subparsers):
+    task_parser=subparsers.add_parser('morph', help='morph help')
+    add_common_monolingual_args(task_parser)
+    task_parser.set_defaults(func=run_morph)
+
+def add_syllabify_parser(subparsers):
+    task_parser=subparsers.add_parser('syllabify', help='syllabify help')
+    add_common_monolingual_args(task_parser)
+    task_parser.set_defaults(func=run_syllabify)
+
+def add_wc_parser(subparsers):
+    task_parser=subparsers.add_parser('wc', help='wc help')
+
+    task_parser.add_argument('infile', 
+                type=argparse.FileType('r',encoding=DEFAULT_ENCODING),
+                nargs='?',
+                default=sys.stdin,
+                help='Input File path',
+            )
+    # task_parser.add_argument('-l', action='store_true')
+    # task_parser.add_argument('-w', action='store_true')
+    # task_parser.add_argument('-c', action='store_true')
+    # task_parser.set_defaults(l=False)
+    # task_parser.set_defaults(w=False)
+    # task_parser.set_defaults(c=False)
+
+    task_parser.set_defaults(func=run_wc)    
+
+def add_indic2roman_parser(subparsers):
+    task_parser=subparsers.add_parser('indic2roman', help='indic2roman help')
+    add_common_monolingual_args(task_parser)
+    task_parser.set_defaults(func=run_indic2roman)
+
+def add_roman2indic_parser(subparsers):
+    task_parser=subparsers.add_parser('roman2indic', help='roman2indic help')
+    add_common_monolingual_args(task_parser)
+    task_parser.set_defaults(func=run_indic2roman)
+
+def add_script_unify_parser(subparsers):
+    task_parser=subparsers.add_parser('script_unify', help='script_unify help')
+    add_common_monolingual_args(task_parser)
+    task_parser.add_argument('-m','--mode', 
+                default='basic',               
+                choices=['naive', 'basic', 'aggressive'] ,
+                help='Script unification mode',
+            )   
+    task_parser.add_argument('-c','--common_lang', 
+                default='hi',                
+                help='Common language in which all languages are represented',
+            )  
+
+    task_parser.set_defaults(func=run_script_unify)    
+
+def add_script_convert_parser(subparsers):
+    task_parser=subparsers.add_parser('script_convert', help='script convert help')
+    add_common_bilingual_args(task_parser)
+    task_parser.set_defaults(func=run_script_convert)
+
+def get_parser():
+    parser = argparse.ArgumentParser(prog='indicnlp')
+    subparsers = parser.add_subparsers(help='Invoke each operation with one of the subcommands', dest='subcommand')
+
+    add_tokenize_parser(subparsers)
+    add_detokenize_parser(subparsers)
+    add_sentence_split_parser(subparsers)
+    add_normalize_parser(subparsers)
+
+    add_morph_parser(subparsers)
+    add_syllabify_parser(subparsers)
+
+    add_wc_parser(subparsers)
+
+    add_indic2roman_parser(subparsers)
+    add_roman2indic_parser(subparsers)
+    add_script_unify_parser(subparsers)
+
+    add_script_convert_parser(subparsers)
+
+    return parser
+
+def main():
+    parser=get_parser()
+    args=parser.parse_args()
+    # print(args)
+    args.func(args)
+
+if __name__ == '__main__':
+    loader.load()
+    main()
+

indic_nlp_library/indicnlp/common.py

@@ -0,0 +1,58 @@
+# 
+#  Copyright (c) 2013-present, Anoop Kunchukuttan
+#  All rights reserved.
+#  
+#  This source code is licensed under the MIT license found in the
+#  LICENSE file in the root directory of this source tree.
+# 
+
+import os
+
+"""
+Path to the Indic NLP Resources directory
+"""
+INDIC_RESOURCES_PATH=''
+
+def init():
+    """
+    Initialize the module. The following actions are performed:
+
+    - Checks of INDIC_RESOURCES_PATH variable is set. If not, checks if it can beb initialized from 
+        INDIC_RESOURCES_PATH environment variable. If that fails, an exception is raised
+    """
+    global INDIC_RESOURCES_PATH 
+    try: 
+        if INDIC_RESOURCES_PATH=='':
+            INDIC_RESOURCES_PATH=os.environ['INDIC_RESOURCES_PATH']
+    except Exception as e: 
+        raise IndicNlpException('INDIC_RESOURCES_PATH not set')
+
+    if INDIC_RESOURCES_PATH=='': 
+        raise IndicNlpException('INDIC_RESOURCES_PATH not set')
+
+
+
+def get_resources_path(): 
+    """
+        Get the path to the Indic NLP Resources directory
+    """
+    return INDIC_RESOURCES_PATH
+
+def set_resources_path(resources_path): 
+    """
+        Set the path to the Indic NLP Resources directory
+    """
+    global INDIC_RESOURCES_PATH 
+    INDIC_RESOURCES_PATH=resources_path
+
+class IndicNlpException(Exception):
+    """
+        Exceptions thrown by Indic NLP Library components are instances of this class.  
+        'msg' attribute contains exception details.
+    """
+    def __init__(self, msg):
+        self.msg = msg 
+
+    def __str__(self):
+        return repr(self.msg)
+

indic_nlp_library/indicnlp/langinfo.py

@@ -0,0 +1,488 @@
+# 
+#  Copyright (c) 2013-present, Anoop Kunchukuttan
+#  All rights reserved.
+#  
+#  This source code is licensed under the MIT license found in the
+#  LICENSE file in the root directory of this source tree.
+# 
+
+## language codes 
+LC_TA='ta'
+
+SCRIPT_RANGES={
+                 'pa':[0x0a00,0x0a7f] ,  
+                 'gu':[0x0a80,0x0aff] ,  
+                 'or':[0x0b00,0x0b7f] ,  
+                 'ta':[0x0b80,0x0bff] ,  
+                 'te':[0x0c00,0x0c7f] ,  
+                 'kn':[0x0c80,0x0cff] ,  
+                 'ml':[0x0d00,0x0d7f] ,  
+                 'si':[0x0d80,0x0dff] ,  
+                 'hi':[0x0900,0x097f] ,  
+                 'mr':[0x0900,0x097f] ,   
+                 'kK':[0x0900,0x097f] ,   
+                 'sa':[0x0900,0x097f] ,   
+                 'ne':[0x0900,0x097f] ,   
+                 'sd':[0x0900,0x097f] ,  
+                 'bn':[0x0980,0x09ff] ,  
+                 'as':[0x0980,0x09ff] ,  
+              }
+
+DRAVIDIAN_LANGUAGES=['ta', 'te', 'kn', 'ml',]
+IE_LANGUAGES=['hi', 'mr', 'kK', 'sa', 'ne', 'sd', 'bn', 'as', 'pa', 'gu', 'or', 'si', ]
+DANDA_DELIM_LANGUAGES=['as','bn','hi','ne','or','pa','sa','sd']
+
+URDU_RANGES=[
+                [0x0600,0x06ff], 
+                [0x0750,0x077f], 
+                [0xfb50,0xfdff], 
+                [0xfe70,0xfeff], 
+            ]
+
+COORDINATED_RANGE_START_INCLUSIVE=0
+COORDINATED_RANGE_END_INCLUSIVE=0x6f
+
+NUMERIC_OFFSET_START=0x66
+NUMERIC_OFFSET_END=0x6f
+
+HALANTA_OFFSET=0x4d
+AUM_OFFSET=0x50
+NUKTA_OFFSET=0x3c
+
+RUPEE_SIGN=0x20b9
+
+DANDA=0x0964
+DOUBLE_DANDA=0x0965
+
+#TODO: add missing fricatives and approximants
+VELAR_RANGE=[0x15,0x19]
+PALATAL_RANGE=[0x1a,0x1e]
+RETROFLEX_RANGE=[0x1f,0x23]
+DENTAL_RANGE=[0x24,0x29]
+LABIAL_RANGE=[0x2a,0x2e]
+
+# verify
+VOICED_LIST=[0x17,0x18,0x1c,0x1d,0x21,0x22,0x26,0x27,0x2c,0x2d]
+UNVOICED_LIST=[0x15,0x16,0x1a,0x1b,0x1f,0x20,0x24,0x25,0x2a,0x2b] #TODO: add sibilants/sonorants
+ASPIRATED_LIST=[0x16,0x18,0x1b,0x1d,0x20,0x22,0x25,0x27,0x2b,0x2d]
+UNASPIRATED_LIST=[0x15,0x17,0x1a,0x1c,0x1f,0x21,0x24,0x26,0x2a,0x2c]
+NASAL_LIST=[0x19,0x1e,0x23,0x28,0x29,0x2d]
+FRICATIVE_LIST=[0x36,0x37,0x38]
+APPROXIMANT_LIST=[0x2f,0x30,0x31,0x32,0x33,0x34,0x35]
+
+#TODO: ha has to be properly categorized 
+
+def is_danda_delim(lang):
+    """
+    Returns True if danda/double danda is a possible delimiter for the language
+    """
+    return lang in DANDA_DELIM_LANGUAGES
+
+def get_offset(c,lang): 
+    """
+    Applicable to Brahmi derived Indic scripts 
+    """
+    return ord(c)-SCRIPT_RANGES[lang][0]
+
+def offset_to_char(c,lang): 
+    """
+    Applicable to Brahmi derived Indic scripts 
+    """
+    return chr(c+SCRIPT_RANGES[lang][0])
+
+def in_coordinated_range(c_offset): 
+    """
+    Applicable to Brahmi derived Indic scripts 
+    """
+    return  (c_offset>=COORDINATED_RANGE_START_INCLUSIVE and c_offset<=COORDINATED_RANGE_END_INCLUSIVE) 
+       
+def is_indiclang_char(c,lang): 
+    """
+    Applicable to Brahmi derived Indic scripts 
+    """
+    o=get_offset(c,lang)
+    return (o>=0 and o<=0x7f) or ord(c)==DANDA or ord(c)==DOUBLE_DANDA
+
+# def is_vowel(c,lang): 
+#     """
+#     Is the character a vowel
+#     """
+#     o=get_offset(c,lang)
+#     return (o>=0x04 and o<=0x14) 
+
+# def is_vowel_sign(c,lang): 
+#     """
+#     Is the character a vowel sign (maatraa)
+#     """
+#     o=get_offset(c,lang)
+#     return (o>=0x3e and o<=0x4c) 
+
+# def is_halanta(c,lang): 
+#     """
+#     Is the character the halanta character
+#     """
+#     o=get_offset(c,lang)
+#     return (o==HALANTA_OFFSET) 
+
+# def is_nukta(c,lang): 
+#     """
+#     Is the character the halanta character
+#     """
+#     o=get_offset(c,lang)
+#     return (o==NUKTA_OFFSET) 
+
+# def is_aum(c,lang): 
+#     """
+#     Is the character a vowel sign (maatraa)
+#     """
+#     o=get_offset(c,lang)
+#     return (o==AUM_OFFSET)
+
+# def is_consonant(c,lang): 
+#     """
+#     Is the character a consonant
+#     """
+#     o=get_offset(c,lang)
+#     return (o>=0x15 and o<=0x39) 
+
+# def is_velar(c,lang): 
+#     """
+#     Is the character a velar
+#     """
+#     o=get_offset(c,lang)
+#     return (o>=VELAR_RANGE[0] and o<=VELAR_RANGE[1]) 
+
+# def is_palatal(c,lang): 
+#     """
+#     Is the character a palatal
+#     """
+#     o=get_offset(c,lang)
+#     return (o>=PALATAL_RANGE[0] and o<=PALATAL_RANGE[1]) 
+
+# def is_retroflex(c,lang): 
+#     """
+#     Is the character a retroflex
+#     """
+#     o=get_offset(c,lang)
+#     return (o>=RETROFLEX_RANGE[0] and o<=RETROFLEX_RANGE[1]) 
+
+# def is_dental(c,lang): 
+#     """
+#     Is the character a dental
+#     """
+#     o=get_offset(c,lang)
+#     return (o>=DENTAL_RANGE[0] and o<=DENTAL_RANGE[1]) 
+
+# def is_labial(c,lang): 
+#     """
+#     Is the character a labial
+#     """
+#     o=get_offset(c,lang)
+#     return (o>=LABIAL_RANGE[0] and o<=LABIAL_RANGE[1]) 
+
+# def is_voiced(c,lang): 
+#     """
+#     Is the character a voiced consonant
+#     """
+#     o=get_offset(c,lang)
+#     return o in VOICED_LIST
+
+# def is_unvoiced(c,lang): 
+#     """
+#     Is the character a unvoiced consonant
+#     """
+#     o=get_offset(c,lang)
+#     return o in UNVOICED_LIST
+
+# def is_aspirated(c,lang): 
+#     """
+#     Is the character a aspirated consonant
+#     """
+#     o=get_offset(c,lang)
+#     return o in ASPIRATED_LIST
+
+# def is_unaspirated(c,lang): 
+#     """
+#     Is the character a unaspirated consonant
+#     """
+#     o=get_offset(c,lang)
+#     return o in UNASPIRATED_LIST
+
+# def is_nasal(c,lang): 
+#     """
+#     Is the character a nasal consonant
+#     """
+#     o=get_offset(c,lang)
+#     return o in NASAL_LIST
+
+# def is_fricative(c,lang): 
+#     """
+#     Is the character a fricative consonant
+#     """
+#     o=get_offset(c,lang)
+#     return o in FRICATIVE_LIST
+
+# def is_approximant(c,lang): 
+#     """
+#     Is the character an approximant consonant
+#     """
+#     o=get_offset(c,lang)
+#     return o in APPROXIMANT_LIST
+
+# def is_number(c,lang): 
+#     """
+#     Is the character a number
+#     """
+#     o=get_offset(c,lang)
+#     return (o>=0x66 and o<=0x6f) 
+
+
+def is_vowel(c,lang): 
+    """
+    Is the character a vowel
+    """
+    o=get_offset(c,lang)
+    return (o>=0x04 and o<=0x14) 
+
+def is_vowel_sign(c,lang): 
+    """
+    Is the character a vowel sign (maatraa)
+    """
+    o=get_offset(c,lang)
+    return (o>=0x3e and o<=0x4c) 
+
+def is_halanta(c,lang): 
+    """
+    Is the character the halanta character
+    """
+    o=get_offset(c,lang)
+    return (o==HALANTA_OFFSET) 
+
+def is_nukta(c,lang): 
+    """
+    Is the character the halanta character
+    """
+    o=get_offset(c,lang)
+    return (o==NUKTA_OFFSET) 
+
+def is_aum(c,lang): 
+    """
+    Is the character a vowel sign (maatraa)
+    """
+    o=get_offset(c,lang)
+    return (o==AUM_OFFSET)
+
+def is_consonant(c,lang): 
+    """
+    Is the character a consonant
+    """
+    o=get_offset(c,lang)
+    return (o>=0x15 and o<=0x39) 
+
+def is_velar(c,lang): 
+    """
+    Is the character a velar
+    """
+    o=get_offset(c,lang)
+    return (o>=VELAR_RANGE[0] and o<=VELAR_RANGE[1]) 
+
+def is_palatal(c,lang): 
+    """
+    Is the character a palatal
+    """
+    o=get_offset(c,lang)
+    return (o>=PALATAL_RANGE[0] and o<=PALATAL_RANGE[1]) 
+
+def is_retroflex(c,lang): 
+    """
+    Is the character a retroflex
+    """
+    o=get_offset(c,lang)
+    return (o>=RETROFLEX_RANGE[0] and o<=RETROFLEX_RANGE[1]) 
+
+def is_dental(c,lang): 
+    """
+    Is the character a dental
+    """
+    o=get_offset(c,lang)
+    return (o>=DENTAL_RANGE[0] and o<=DENTAL_RANGE[1]) 
+
+def is_labial(c,lang): 
+    """
+    Is the character a labial
+    """
+    o=get_offset(c,lang)
+    return (o>=LABIAL_RANGE[0] and o<=LABIAL_RANGE[1]) 
+
+def is_voiced(c,lang): 
+    """
+    Is the character a voiced consonant
+    """
+    o=get_offset(c,lang)
+    return o in VOICED_LIST
+
+def is_unvoiced(c,lang): 
+    """
+    Is the character a unvoiced consonant
+    """
+    o=get_offset(c,lang)
+    return o in UNVOICED_LIST
+
+def is_aspirated(c,lang): 
+    """
+    Is the character a aspirated consonant
+    """
+    o=get_offset(c,lang)
+    return o in ASPIRATED_LIST
+
+def is_unaspirated(c,lang): 
+    """
+    Is the character a unaspirated consonant
+    """
+    o=get_offset(c,lang)
+    return o in UNASPIRATED_LIST
+
+def is_nasal(c,lang): 
+    """
+    Is the character a nasal consonant
+    """
+    o=get_offset(c,lang)
+    return o in NASAL_LIST
+
+def is_fricative(c,lang): 
+    """
+    Is the character a fricative consonant
+    """
+    o=get_offset(c,lang)
+    return o in FRICATIVE_LIST
+
+def is_approximant(c,lang): 
+    """
+    Is the character an approximant consonant
+    """
+    o=get_offset(c,lang)
+    return o in APPROXIMANT_LIST
+
+def is_number(c,lang): 
+    """
+    Is the character a number
+    """
+    o=get_offset(c,lang)
+    return (o>=0x66 and o<=0x6f) 
+
+
+##################################################
+
+def is_vowel_offset(c_offset):
+    """
+    Is the offset a vowel
+    """
+    return (c_offset>=0x04 and c_offset<=0x14) 
+
+def is_vowel_sign_offset(c_offset):
+    """
+    Is the offset a vowel sign (maatraa)
+    """
+    return (c_offset>=0x3e and c_offset<=0x4c) 
+
+def is_halanta_offset(c_offset):
+    """
+    Is the offset the halanta offset
+    """
+    return (c_offset==HALANTA_OFFSET) 
+
+def is_nukta_offset(c_offset):
+    """
+    Is the offset the halanta offset
+    """
+    return (c_offset==NUKTA_OFFSET) 
+
+def is_aum_offset(c_offset):
+    """
+    Is the offset a vowel sign (maatraa)
+    """
+    return (c_offset==AUM_OFFSET)
+
+def is_consonant_offset(c_offset):
+    """
+    Is the offset a consonant
+    """
+    return (c_offset>=0x15 and c_offset<=0x39) 
+
+def is_velar_offset(c_offset):
+    """
+    Is the offset a velar
+    """
+    return (c_offset>=VELAR_RANGE[0] and c_offset<=VELAR_RANGE[1]) 
+
+def is_palatal_offset(c_offset):
+    """
+    Is the offset a palatal
+    """
+    return (c_offset>=PALATAL_RANGE[0] and c_offset<=PALATAL_RANGE[1]) 
+
+def is_retroflex_offset(c_offset):
+    """
+    Is the offset a retroflex
+    """
+    return (c_offset>=RETROFLEX_RANGE[0] and c_offset<=RETROFLEX_RANGE[1]) 
+
+def is_dental_offset(c_offset):
+    """
+    Is the offset a dental
+    """
+    return (c_offset>=DENTAL_RANGE[0] and c_offset<=DENTAL_RANGE[1]) 
+
+def is_labial_offset(c_offset):
+    """
+    Is the offset a labial
+    """
+    return (c_offset>=LABIAL_RANGE[0] and c_offset<=LABIAL_RANGE[1]) 
+
+def is_voiced_offset(c_offset):
+    """
+    Is the offset a voiced consonant
+    """
+    return c_offset in VOICED_LIST
+
+def is_unvoiced_offset(c_offset):
+    """
+    Is the offset a unvoiced consonant
+    """
+    return c_offset in UNVOICED_LIST
+
+def is_aspirated_offset(c_offset):
+    """
+    Is the offset a aspirated consonant
+    """
+    return c_offset in ASPIRATED_LIST
+
+def is_unaspirated_offset(c_offset):
+    """
+    Is the offset a unaspirated consonant
+    """
+    return c_offset in UNASPIRATED_LIST
+
+def is_nasal_offset(c_offset):
+    """
+    Is the offset a nasal consonant
+    """
+    return c_offset in NASAL_LIST
+
+def is_fricative_offset(c_offset):
+    """
+    Is the offset a fricative consonant
+    """
+    return c_offset in FRICATIVE_LIST
+
+def is_approximant_offset(c_offset):
+    """
+    Is the offset an approximant consonant
+    """
+    return c_offset in APPROXIMANT_LIST
+
+def is_number_offset(c_offset):
+    """
+    Is the offset a number
+    """
+    return (c_offset>=0x66 and c_offset<=0x6f) 

indic_nlp_library/indicnlp/loader.py

@@ -0,0 +1,35 @@
+# 
+#  Copyright (c) 2013-present, Anoop Kunchukuttan
+#  All rights reserved.
+#  
+#  This source code is licensed under the MIT license found in the
+#  LICENSE file in the root directory of this source tree.
+# 
+
+from indicnlp import common
+from indicnlp.script import indic_scripts
+from indicnlp.script import english_script
+from indicnlp.transliterate import unicode_transliterate
+
+def load():
+    """
+        Initializes the Indic NLP library. Clients should call this method before using the library. 
+
+        Any module requiring initialization should have a init() method, to which a call must be made from this method 
+    """
+
+    ### Order of intialization may matter 
+
+    # Common has to be loaded first to get access to resources 
+    common.init()
+
+    ## Initialization of Indic scripts module 
+    indic_scripts.init()
+
+    ## Initialization of English scripts module 
+    english_script.init()
+
+    ## Initialization of unicode_transliterate module 
+    unicode_transliterate.init()
+
+

indic_nlp_library/indicnlp/morph/unsupervised_morph.py

@@ -0,0 +1,142 @@
+# 
+#  Copyright (c) 2013-present, Anoop Kunchukuttan
+#  All rights reserved.
+#  
+#  This source code is licensed under the MIT license found in the
+#  LICENSE file in the root directory of this source tree.
+# 
+
+import codecs, sys, itertools,re,os
+import morfessor 
+
+from functools import lru_cache
+
+from indicnlp import langinfo
+from indicnlp import common
+from indicnlp.tokenize import indic_tokenize
+
+# Unsupervised Morphological Analyser for Indian languages. 
+#
+# @author Anoop Kunchukuttan 
+#
+
+class MorphAnalyzerI(object):
+    """
+     Interface for Morph Analyzer
+    """
+
+    def morph_analyze(word):
+        pass 
+
+    def morph_analyze_document(tokens):
+        pass 
+
+class UnsupervisedMorphAnalyzer(MorphAnalyzerI): 
+    """
+    Unsupervised Morphological analyser built using Morfessor 2.0
+    """
+
+    def __init__(self,lang,add_marker=False): 
+        self.lang=lang
+        self.add_marker=add_marker
+
+        io = morfessor.MorfessorIO()
+        self._morfessor_model=io.read_any_model(os.path.join(common.INDIC_RESOURCES_PATH,'morph','morfessor','{}.model'.format(lang)))        
+
+        self._script_range_pat=r'^[{}-{}]+$'.format(chr(langinfo.SCRIPT_RANGES[lang][0]),chr(langinfo.SCRIPT_RANGES[lang][1]))
+        self._script_check_re=re.compile(self._script_range_pat)
+
+    def _contains_number(self,text):
+        if self.lang in langinfo.SCRIPT_RANGES: 
+            for c in text: 
+                offset=ord(c)-langinfo.SCRIPT_RANGES[self.lang][0]
+                if offset >=langinfo.NUMERIC_OFFSET_START and offset <= langinfo.NUMERIC_OFFSET_END:
+                    return True  
+        return False     
+
+    def _morphanalysis_needed(self,word):
+        return self._script_check_re.match(word) and not self._contains_number(word)
+
+    @lru_cache(maxsize=16384)
+    def morph_analyze(self,word):
+        """
+        Morphanalyzes a single word and returns a list of component morphemes
+
+        @param word: string input word 
+        """
+        m_list=[]
+        if self._morphanalysis_needed(word): 
+            val=self._morfessor_model.viterbi_segment(word)
+            m_list=val[0]
+            if self.add_marker:
+                m_list= [ '{}_S_'.format(m) if i>0 else '{}_R_'.format(m)  for i,m in enumerate(m_list)]
+        else:
+            if self.add_marker:
+                word='{}_E_'.format(word)
+            m_list=[word]
+        return m_list 
+
+        ### Older implementation
+        #val=self._morfessor_model.viterbi_segment(word)
+        #m_list=val[0]
+        #if self.add_marker:
+        #    m_list= [ u'{}_S_'.format(m) if i>0 else u'{}_R_'.format(m)  for i,m in enumerate(m_list)]
+        #return m_list
+    
+
+    def morph_analyze_document(self,tokens):
+        """
+        Morphanalyzes a document, represented as a list of tokens
+        Each word  is morphanalyzed and result is a list of morphemes constituting the document 
+
+        @param tokens: string sequence of words 
+
+        @return list of segments in the document after morph analysis 
+        """
+
+        out_tokens=[]
+        for token in tokens: 
+            morphs=self.morph_analyze(token)
+            out_tokens.extend(morphs)
+        return out_tokens    
+
+        #### Older implementation
+        #out_tokens=[]
+        #for token in tokens: 
+        #    if self._morphanalysis_needed(token): 
+        #        morphs=self.morph_analyze(token)
+        #        out_tokens.extend(morphs)
+        #    else:
+        #        if self.add_marker:
+        #            token=u'{}_E_'.format(token)
+        #        out_tokens.append(token)
+        #return out_tokens    
+
+
+if __name__ == '__main__': 
+
+    if len(sys.argv)<4:
+        print("Usage: python unsupervised_morph.py <infile> <outfile> <language> <indic_resources_path> [<add_marker>]")
+        sys.exit(1)
+
+    language=sys.argv[3]
+    common.INDIC_RESOURCES_PATH=sys.argv[4]
+
+    add_marker=False
+
+    if len(sys.argv)==6:
+        add_marker= True if sys.argv[5] == 'True' else False 
+
+    print('Loading morph analyser for ' + language) 
+    analyzer=UnsupervisedMorphAnalyzer(language,add_marker)
+    print('Loaded morph analyser for ' + language) 
+
+    with codecs.open(sys.argv[1],'r','utf-8') as ifile:
+        with codecs.open(sys.argv[2],'w','utf-8') as ofile:
+            for line in ifile.readlines():
+                line=line.strip()
+                tokens=indic_tokenize.trivial_tokenize(line)
+                morph_tokens=analyzer.morph_analyze_document(tokens)
+                ofile.write(' '.join(morph_tokens))
+                ofile.write('\n')
+

indic_nlp_library/indicnlp/normalize/indic_normalize.py

@@ -0,0 +1,984 @@
+# -*- coding: utf-8 -*-
+
+# 
+#  Copyright (c) 2013-present, Anoop Kunchukuttan
+#  All rights reserved.
+#  
+#  This source code is licensed under the MIT license found in the
+#  LICENSE file in the root directory of this source tree.
+# 
+
+#Program for normalization of text written in Unicode. This is mainly geared towards Indic scripts 
+#
+# @author Anoop Kunchukuttan 
+#
+
+import sys, codecs, string, itertools, re
+from indicnlp import langinfo
+
+
+class NormalizerI(object):
+    """
+    The normalizer classes do the following: 
+
+    * Some characters have multiple Unicode codepoints. The normalizer chooses a single standard representation
+    * Some control characters are deleted
+    * While typing using the Latin keyboard, certain typical mistakes occur which are corrected by the module
+
+    Base class for normalizer. Performs some common normalization, which includes: 
+
+    * Byte order mark, word joiner, etc. removal 
+    * ZERO_WIDTH_NON_JOINER and ZERO_WIDTH_JOINER removal 
+    * ZERO_WIDTH_SPACE and NO_BREAK_SPACE replaced by spaces 
+
+    Script specific normalizers should derive from this class and override the normalize() method. 
+    They can call the super class 'normalize() method to avail of the common normalization 
+
+    """
+
+    BYTE_ORDER_MARK='\uFEFF'
+    BYTE_ORDER_MARK_2='\uFFFE'
+    WORD_JOINER='\u2060'
+    SOFT_HYPHEN='\u00AD'
+
+    ZERO_WIDTH_SPACE='\u200B'
+    NO_BREAK_SPACE='\u00A0'
+
+    ZERO_WIDTH_NON_JOINER='\u200C'
+    ZERO_WIDTH_JOINER='\u200D'
+
+    def _normalize_punctuations(self, text):
+        """
+        Normalize punctuations. 
+        Applied many of the punctuation normalizations that are part of MosesNormalizer 
+        from sacremoses
+        """
+        text=text.replace(NormalizerI.BYTE_ORDER_MARK,'')
+        text=text.replace('„', r'"')
+        text=text.replace('“', r'"')
+        text=text.replace('”', r'"')
+        text=text.replace('–', r'-')
+        text=text.replace('—', r' - ')
+        text=text.replace('´', r"'")
+        text=text.replace('‘', r"'")
+        text=text.replace('‚', r"'")
+        text=text.replace('’', r"'")
+        text=text.replace("''", r'"')
+        text=text.replace('´´', r'"')
+        text=text.replace('…', r'...')
+
+        return text
+
+    def normalize(self,text):
+        pass 
+
+
+class BaseNormalizer(NormalizerI):
+
+    def __init__(self,lang,
+            remove_nuktas=False,
+            nasals_mode='do_nothing',
+            do_normalize_chandras=False,
+            do_normalize_vowel_ending=False):
+
+        self.lang=lang
+        self.remove_nuktas=remove_nuktas
+        self.nasals_mode=nasals_mode
+        self.do_normalize_chandras=do_normalize_chandras
+        self.do_normalize_vowel_ending=do_normalize_vowel_ending
+
+        self._init_normalize_chandras()
+        self._init_normalize_nasals()
+        self._init_normalize_vowel_ending()
+        #self._init_visarga_correction()
+        
+    def _init_normalize_vowel_ending(self):
+
+        if self.lang in langinfo.IE_LANGUAGES:
+            self.fn_vowel_ending=self._normalize_word_vowel_ending_ie
+        elif self.lang in langinfo.DRAVIDIAN_LANGUAGES:
+            self.fn_vowel_ending=self._normalize_word_vowel_ending_dravidian
+        else:
+            self.fn_vowel_ending=lambda x: x
+
+    def _init_normalize_chandras(self):
+
+        substitution_offsets =\
+            [
+                [0x0d , 0x0f], # chandra e, independent
+                [0x11 , 0x13], # chandra o, independent
+                [0x45 , 0x47], # chandra e , 0xde],pendent
+                [0x49 , 0x4b], # chandra o , 0xde],pendent
+                # [0x72 , 0x0f], # mr: chandra e, independent
+
+                [0x00 , 0x02], # chandrabindu
+                [0x01 , 0x02], # chandrabindu
+            ]
+
+        self.chandra_substitutions =  [ 
+                (langinfo.offset_to_char(x[0],self.lang), langinfo.offset_to_char(x[1],self.lang)) 
+                    for x in substitution_offsets ]
+
+    def _normalize_chandras(self,text):
+        for match, repl in self.chandra_substitutions:
+            text=text.replace(match,repl)
+        return text
+
+    def _init_to_anusvaara_strict(self):
+        """
+        `r1_nasal=re.compile(r'\\u0919\\u094D([\\u0915-\\u0918])')`
+        """
+    
+        pat_signatures=\
+            [
+                 [0x19,0x15,0x18],
+                 [0x1e,0x1a,0x1d],            
+                 [0x23,0x1f,0x22],                        
+                 [0x28,0x24,0x27],        
+                 [0x29,0x24,0x27],                    
+                 [0x2e,0x2a,0x2d],                    
+            ]    
+        
+        halant_offset=0x4d
+        anusvaara_offset=0x02
+        
+        pats=[]
+        
+        for pat_signature in pat_signatures:
+            pat=re.compile(r'{nasal}{halant}([{start_r}-{end_r}])'.format(
+                nasal=langinfo.offset_to_char(pat_signature[0],self.lang),
+                halant=langinfo.offset_to_char(halant_offset,self.lang),
+                start_r=langinfo.offset_to_char(pat_signature[1],self.lang),
+                end_r=langinfo.offset_to_char(pat_signature[2],self.lang),
+            ))
+            pats.append(pat)
+        
+        repl_string='{anusvaara}\\1'.format(anusvaara=langinfo.offset_to_char(anusvaara_offset,self.lang))
+
+        self.pats_repls=(pats,repl_string)
+    
+    def _to_anusvaara_strict(self,text):
+        
+        pats, repl_string = self.pats_repls
+        for pat in pats:
+            text=pat.sub(repl_string,text)
+            
+        return text
+
+    def _init_to_anusvaara_relaxed(self):
+        """
+        `r1_nasal=re.compile(r'\\u0919\\u094D([\\u0915-\\u0918])')`
+        """
+            
+        nasals_list=[0x19,0x1e,0x23,0x28,0x29,0x2e]    
+        nasals_list_str=','.join([langinfo.offset_to_char(x,self.lang) for x in nasals_list])
+        
+        halant_offset=0x4d    
+        anusvaara_offset=0x02    
+        
+        pat=re.compile(r'[{nasals_list_str}]{halant}'.format(
+                nasals_list_str=nasals_list_str,
+                halant=langinfo.offset_to_char(halant_offset,self.lang),
+            ))
+        
+        repl_string='{anusvaara}'.format(anusvaara=langinfo.offset_to_char(anusvaara_offset,self.lang))
+
+        self.pats_repls = (pat,repl_string)
+    
+    def _to_anusvaara_relaxed(self,text):
+        pat, repl_string = self.pats_repls
+        return pat.sub(repl_string,text)
+    
+
+    def _init_to_nasal_consonants(self):
+        """
+        `r1_nasal=re.compile(r'\\u0919\\u094D([\\u0915-\\u0918])')`
+        """
+    
+        pat_signatures=\
+            [
+                 [0x19,0x15,0x18],
+                 [0x1e,0x1a,0x1d],            
+                 [0x23,0x1f,0x22],                        
+                 [0x28,0x24,0x27],        
+                 [0x29,0x24,0x27],                    
+                 [0x2e,0x2a,0x2d],                    
+            ]    
+        
+        halant_offset=0x4d
+        anusvaara_offset=0x02 
+        
+        pats=[]
+        repl_strings=[]
+        
+        for pat_signature in pat_signatures:
+            pat=re.compile(r'{anusvaara}([{start_r}-{end_r}])'.format(
+                anusvaara=langinfo.offset_to_char(anusvaara_offset,self.lang),
+                start_r=langinfo.offset_to_char(pat_signature[1],self.lang),
+                end_r=langinfo.offset_to_char(pat_signature[2],self.lang),
+            ))
+            pats.append(pat)
+            repl_string='{nasal}{halant}\\1'.format(
+                nasal=langinfo.offset_to_char(pat_signature[0],self.lang),
+                halant=langinfo.offset_to_char(halant_offset,self.lang),
+                )
+            repl_strings.append(repl_string)
+    
+        self.pats_repls=list(zip(pats,repl_strings))
+
+    def _to_nasal_consonants(self,text):
+    
+        for pat, repl in self.pats_repls:
+            text=pat.sub(repl,text)
+            
+        return text
+
+    def _init_normalize_nasals(self):
+
+        if self.nasals_mode == 'to_anusvaara_strict':
+            self._init_to_anusvaara_strict()
+        elif self.nasals_mode == 'to_anusvaara_relaxed':
+            self._init_to_anusvaara_relaxed()
+        elif self.nasals_mode == 'to_nasal_consonants':
+            self._init_to_nasal_consonants()
+
+    def _normalize_nasals(self,text): 
+        if self.nasals_mode == 'to_anusvaara_strict':
+            return self._to_anusvaara_strict(text)
+        elif self.nasals_mode == 'to_anusvaara_relaxed':
+            return self._to_anusvaara_relaxed(text)
+        elif self.nasals_mode == 'to_nasal_consonants':
+            return self._to_nasal_consonants(text)
+        else:
+            return text
+
+    
+    def _normalize_word_vowel_ending_dravidian(self,word):
+        """
+        for Dravidian
+        - consonant ending: add 'a' ki maatra
+        - halant ending: no change
+        - 'a' ki maatra: no change
+        """
+        if len(word)>0 and langinfo.is_consonant(word[-1],self.lang):
+            return word+langinfo.offset_to_char(0x3e,self.lang)
+        else:
+            return word
+
+    def _normalize_word_vowel_ending_ie(self,word):
+        """
+        for IE
+        - consonant ending: add halant
+        - halant ending: no change
+        - 'a' ki maatra: no change
+        """
+        if len(word)>0 and langinfo.is_consonant(word[-1],self.lang):
+            return word+langinfo.offset_to_char(langinfo.HALANTA_OFFSET,self.lang)
+        else:
+            return word 
+
+    def _normalize_vowel_ending(self,text):
+        return ' '.join([ self.fn_vowel_ending(w) for w in text.split(' ') ])
+
+    def normalize(self,text):
+        """
+        Method to be implemented for normalization for each script 
+        """
+        text=text.replace(NormalizerI.BYTE_ORDER_MARK,'')
+        text=text.replace(NormalizerI.BYTE_ORDER_MARK_2,'')
+        text=text.replace(NormalizerI.WORD_JOINER,'')
+        text=text.replace(NormalizerI.SOFT_HYPHEN,'')
+
+        text=text.replace(NormalizerI.ZERO_WIDTH_SPACE,' ') # ??
+        text=text.replace(NormalizerI.NO_BREAK_SPACE,' ')
+
+        text=text.replace(NormalizerI.ZERO_WIDTH_NON_JOINER, '')
+        text=text.replace(NormalizerI.ZERO_WIDTH_JOINER,'')
+        
+        text=self._normalize_punctuations(text)
+
+        if self.do_normalize_chandras:
+            text=self._normalize_chandras(text)
+        text=self._normalize_nasals(text)
+        if self.do_normalize_vowel_ending:
+            text=self._normalize_vowel_ending(text)
+        
+        return text
+
+
+    def get_char_stats(self,text):    
+        print(len(re.findall(NormalizerI.BYTE_ORDER_MARK,text)))
+        print(len(re.findall(NormalizerI.BYTE_ORDER_MARK_2,text)))
+        print(len(re.findall(NormalizerI.WORD_JOINER,text)))
+        print(len(re.findall(NormalizerI.SOFT_HYPHEN,text)))
+
+        print(len(re.findall(NormalizerI.ZERO_WIDTH_SPACE,text) ))
+        print(len(re.findall(NormalizerI.NO_BREAK_SPACE,text)))
+
+        print(len(re.findall(NormalizerI.ZERO_WIDTH_NON_JOINER,text)))
+        print(len(re.findall(NormalizerI.ZERO_WIDTH_JOINER,text)))
+
+        #for mobj in re.finditer(NormalizerI.ZERO_WIDTH_NON_JOINER,text):
+        #    print text[mobj.start()-10:mobj.end()+10].replace('\n', ' ').replace(NormalizerI.ZERO_WIDTH_NON_JOINER,'').encode('utf-8')
+        #print hex(ord(text[mobj.end():mobj.end()+1]))
+
+    def correct_visarga(self,text,visarga_char,char_range):
+        text=re.sub(r'([\u0900-\u097f]):','\\1\u0903',text)
+        
+
+
+class DevanagariNormalizer(BaseNormalizer): 
+    """
+    Normalizer for the Devanagari script. In addition to basic normalization by the super class, 
+
+    * Replaces the composite characters containing nuktas by their decomposed form 
+    * replace pipe character '|' by poorna virama character
+    * replace colon ':' by visarga if the colon follows a charcter in this script 
+    
+    """
+
+    NUKTA='\u093C' 
+
+    def __init__(self,lang='hi',remove_nuktas=False,nasals_mode='do_nothing',
+            do_normalize_chandras=False,do_normalize_vowel_ending=False):
+        super(DevanagariNormalizer,self).__init__(lang,remove_nuktas,nasals_mode,do_normalize_chandras,do_normalize_vowel_ending)
+
+    def normalize(self,text): 
+
+        # common normalization for Indic scripts 
+        text=super(DevanagariNormalizer,self).normalize(text)
+
+        # chandra a replacement for Marathi
+        text=text.replace('\u0972','\u090f')
+
+        # decomposing Nukta based composite characters
+        text=text.replace('\u0929','\u0928'+DevanagariNormalizer.NUKTA)
+        text=text.replace('\u0931','\u0930'+DevanagariNormalizer.NUKTA)
+        text=text.replace('\u0934','\u0933'+DevanagariNormalizer.NUKTA)
+        text=text.replace('\u0958','\u0915'+DevanagariNormalizer.NUKTA)
+        text=text.replace('\u0959','\u0916'+DevanagariNormalizer.NUKTA)
+        text=text.replace('\u095A','\u0917'+DevanagariNormalizer.NUKTA)
+        text=text.replace('\u095B','\u091C'+DevanagariNormalizer.NUKTA)
+        text=text.replace('\u095C','\u0921'+DevanagariNormalizer.NUKTA)
+        text=text.replace('\u095D','\u0922'+DevanagariNormalizer.NUKTA)
+        text=text.replace('\u095E','\u092B'+DevanagariNormalizer.NUKTA)
+        text=text.replace('\u095F','\u092F'+DevanagariNormalizer.NUKTA)
+
+        if self.remove_nuktas:
+            text=text.replace(DevanagariNormalizer.NUKTA,'')
+
+        # replace pipe character for poorna virama 
+        text=text.replace('\u007c','\u0964')
+
+        # correct visarga 
+        text=re.sub(r'([\u0900-\u097f]):','\\1\u0903',text)
+
+        return text
+
+    def get_char_stats(self,text):
+        super(DevanagariNormalizer,self).get_char_stats(text)
+
+        print((len(re.findall('\u0929',text))))
+        print((len(re.findall('\u0931',text))))
+        print((len(re.findall('\u0934',text))))
+        print((len(re.findall('\u0958',text))))
+        print((len(re.findall('\u0959',text))))
+        print((len(re.findall('\u095A',text))))
+        print((len(re.findall('\u095B',text))))
+        print((len(re.findall('\u095C',text))))
+        print((len(re.findall('\u095D',text))))
+        print((len(re.findall('\u095E',text))))
+        print((len(re.findall('\u095F',text))))
+
+        #print(len(re.findall(u'\u0928'+DevanagariNormalizer.NUKTA,text)))
+        #print(len(re.findall(u'\u0930'+DevanagariNormalizer.NUKTA,text)))
+        #print(len(re.findall(u'\u0933'+DevanagariNormalizer.NUKTA,text)))
+        #print(len(re.findall(u'\u0915'+DevanagariNormalizer.NUKTA,text)))
+        #print(len(re.findall(u'\u0916'+DevanagariNormalizer.NUKTA,text)))
+        #print(len(re.findall(u'\u0917'+DevanagariNormalizer.NUKTA,text)))
+        #print(len(re.findall(u'\u091C'+DevanagariNormalizer.NUKTA,text)))
+        #print(len(re.findall(u'\u0921'+DevanagariNormalizer.NUKTA,text)))
+        #print(len(re.findall(u'\u0922'+DevanagariNormalizer.NUKTA,text)))
+        #print(len(re.findall(u'\u092B'+DevanagariNormalizer.NUKTA,text)))
+        #print(len(re.findall(u'\u092F'+DevanagariNormalizer.NUKTA,text)))
+
+class GurmukhiNormalizer(BaseNormalizer): 
+    """
+    Normalizer for the Gurmukhi script. In addition to basic normalization by the super class, 
+
+    * Replaces the composite characters containing nuktas by their decomposed form 
+    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama 
+    * replace pipe character '|' by poorna virama character
+    * replace colon ':' by visarga if the colon follows a charcter in this script 
+    """
+
+    NUKTA='\u0A3C' 
+
+    VOWEL_NORM_MAPS={
+        ## http://www.unicode.org/versions/Unicode12.1.0/ch12.pdf
+        ## Table 12-16
+        '\u0a05\u0a3e': '\u0a06',
+        '\u0a72\u0a3f': '\u0a07',
+        '\u0a72\u0a40': '\u0a08',
+        '\u0a73\u0a41': '\u0a09',
+        '\u0a73\u0a42': '\u0a0a',
+        '\u0a72\u0a47': '\u0a0f',
+        '\u0a05\u0a48': '\u0a10',
+        '\u0a73\u0a4b': '\u0a13',
+        '\u0a05\u0a4c': '\u0a14',            
+    }
+
+    def __init__(self,lang='pa',remove_nuktas=False,nasals_mode='do_nothing',do_normalize_chandras=False,
+                do_normalize_vowel_ending=False,
+                do_canonicalize_addak=False, 
+                do_canonicalize_tippi=False, 
+                do_replace_vowel_bases=False):
+        super(GurmukhiNormalizer,self).__init__(lang,remove_nuktas,nasals_mode,do_normalize_chandras,do_normalize_vowel_ending)
+        self.do_canonicalize_addak=do_canonicalize_addak
+        self.do_canonicalize_tippi=do_canonicalize_tippi
+        self.do_replace_vowel_bases=do_replace_vowel_bases
+
+
+    def _normalize_vowels(self,text):
+        """
+
+        """
+
+        ## standard vowel replacements as per suggestions in 
+        ## http://www.unicode.org/versions/Unicode12.1.0/ch12.pdf
+        ## Table 12-16
+
+        for k,v in GurmukhiNormalizer.VOWEL_NORM_MAPS.items():
+            text=text.replace(k,v)
+        
+        ## the above mappings should account for majority of the variantions, 
+        ## Rest are handled via this generic rule which looks at the diacritic 
+        ## following the 2 special characters 
+        ## TBD: don't see evidence for this in Wikipedia corpus
+
+        ## If these special characters occur without any diacritic, replace them with closet
+        ## equivalent vowels
+        if self.do_replace_vowel_bases:
+            text=text.replace('\u0a72','\u0a07')
+            text=text.replace('\u0a73','\u0a09')
+
+        return text
+
+
+    def normalize(self,text): 
+
+        # Addak
+        if self.do_canonicalize_addak:
+            ## replace addak+consonant with consonat+halant+consonant
+            text=re.sub(r'\u0a71(.)','\\1\u0a4d\\1',text)
+            
+        # Tippi 
+        if self.do_canonicalize_tippi:
+            text=text.replace('\u0a70','\u0a02') 
+
+        # Vowels: Gurumuki has multiple ways of representing independent vowels due
+        # to the characters 'iri' and 'ura'. 
+        text=self._normalize_vowels(text)
+
+        # common normalization for Indic scripts 
+        text=super(GurmukhiNormalizer,self).normalize(text)
+
+        # decomposing Nukta based composite characters
+        text=text.replace('\u0a33','\u0a32'+GurmukhiNormalizer.NUKTA)
+        text=text.replace('\u0a36','\u0a38'+GurmukhiNormalizer.NUKTA)
+        text=text.replace('\u0a59','\u0a16'+GurmukhiNormalizer.NUKTA)
+        text=text.replace('\u0a5a','\u0a17'+GurmukhiNormalizer.NUKTA)
+        text=text.replace('\u0a5b','\u0a1c'+GurmukhiNormalizer.NUKTA)
+        text=text.replace('\u0a5e','\u0a2b'+GurmukhiNormalizer.NUKTA)
+
+        if self.remove_nuktas:
+            text=text.replace(GurmukhiNormalizer.NUKTA,'')
+
+        # replace the poorna virama codes specific to script 
+        # with generic Indic script codes
+        text=text.replace('\u0a64','\u0964')
+        text=text.replace('\u0a65','\u0965')
+
+        ## replace pipe character for poorna virama 
+        text=text.replace('\u007c','\u0964')
+
+        # correct visarge 
+        text=re.sub(r'([\u0a00-\u0a7f]):','\\1\u0a03',text)
+
+        return text
+
+
+class GujaratiNormalizer(BaseNormalizer): 
+    """
+    Normalizer for the Gujarati script. In addition to basic normalization by the super class, 
+
+    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama 
+    * replace colon ':' by visarga if the colon follows a charcter in this script 
+    """
+
+    NUKTA='\u0ABC' 
+
+    def __init__(self,lang='gu',remove_nuktas=False,nasals_mode='do_nothing',do_normalize_chandras=False,
+                    do_normalize_vowel_ending=False):
+        super(GujaratiNormalizer,self).__init__(lang,remove_nuktas,nasals_mode,do_normalize_chandras,do_normalize_vowel_ending)
+
+    def normalize(self,text): 
+
+        # common normalization for Indic scripts 
+        text=super(GujaratiNormalizer,self).normalize(text)
+
+        # decomposing Nukta based composite characters
+        if self.remove_nuktas:
+            text=text.replace(GujaratiNormalizer.NUKTA,'')
+
+
+        # replace the poorna virama codes specific to script 
+        # with generic Indic script codes
+        text=text.replace('\u0ae4','\u0964')
+        text=text.replace('\u0ae5','\u0965')
+
+        # correct visarge 
+        text=re.sub(r'([\u0a80-\u0aff]):','\\1\u0a83',text)
+
+        return text
+
+
+class OriyaNormalizer(BaseNormalizer): 
+    """
+    Normalizer for the Oriya script. In addition to basic normalization by the super class, 
+
+    * Replaces the composite characters containing nuktas by their decomposed form 
+    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama 
+    * Canonicalize two part dependent vowels
+    * Replace 'va' with 'ba'
+    * replace pipe character '|' by poorna virama character
+    * replace colon ':' by visarga if the colon follows a charcter in this script 
+    """
+
+    NUKTA='\u0B3C' 
+
+    VOWEL_NORM_MAPS={
+        ## See Table 12-22 in http://www.unicode.org/versions/Unicode12.1.0/ch12.pdf
+        '\u0b05\u0b3e': '\u0b06',
+        '\u0b0f\u0b57': '\u0b10',
+        '\u0b13\u0b57': '\u0b14',
+    }
+
+
+    def __init__(self,lang='or',remove_nuktas=False,nasals_mode='do_nothing',do_normalize_chandras=False,
+                do_normalize_vowel_ending=False,
+                do_remap_wa=False):
+        super(OriyaNormalizer,self).__init__(lang,remove_nuktas,nasals_mode,do_normalize_chandras,do_normalize_vowel_ending)
+        self.do_remap_wa=do_remap_wa
+
+    def normalize(self,text): 
+
+        # common normalization for Indic scripts 
+        text=super(OriyaNormalizer,self).normalize(text)
+
+        ## standard vowel replacements as per suggestions in Unicode documents
+        for k,v in OriyaNormalizer.VOWEL_NORM_MAPS.items():
+            text=text.replace(k,v)
+
+        # decomposing Nukta based composite characters
+        text=text.replace('\u0b5c','\u0b21'+OriyaNormalizer.NUKTA)
+        text=text.replace('\u0b5d','\u0b22'+OriyaNormalizer.NUKTA)
+
+        if self.remove_nuktas:
+            text=text.replace(OriyaNormalizer.NUKTA,'')
+
+        # replace the poorna virama codes specific to script 
+        # with generic Indic script codes
+        text=text.replace('\u0b64','\u0964')
+        text=text.replace('\u0b65','\u0965')
+
+        # replace pipe character for poorna virama 
+        text=text.replace('\u0b7c','\u0964')
+
+        # replace wa with ba 
+        if self.do_remap_wa:
+            text=text.replace('\u0b71','\u0b2c')
+
+        # replace va with ba 
+        # NOTE: documentation (chapter on Indic scripts) and codepoint chart seem contradictory 
+        # (this applied to wa to ba rule also above)
+        text=text.replace('\u0b35','\u0b2c')
+
+        # AI dependent vowel sign 
+        text=text.replace('\u0b47\u0b56','\u0b58')
+
+        # two part dependent vowels
+        text=text.replace('\u0b47\u0b3e','\u0b4b')
+        text=text.replace('\u0b47\u0b57','\u0b4c')
+
+
+        # additional consonant - not clear how to handle this
+        # ignore
+
+        # correct visarge 
+        text=re.sub(r'([\u0b00-\u0b7f]):','\\1\u0b03',text)
+
+        return text
+
+
+class BengaliNormalizer(BaseNormalizer): 
+    """
+    Normalizer for the Bengali script. In addition to basic normalization by the super class, 
+
+    * Replaces the composite characters containing nuktas by their decomposed form 
+    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama 
+    * Canonicalize two part dependent vowels
+    * replace pipe character '|' by poorna virama character
+    * replace colon ':' by visarga if the colon follows a charcter in this script 
+
+    """
+
+    NUKTA='\u09BC' 
+
+    def __init__(self,lang='bn',remove_nuktas=False,nasals_mode='do_nothing',do_normalize_chandras=False,
+                    do_normalize_vowel_ending=False,
+                    do_remap_assamese_chars=False):
+        super(BengaliNormalizer,self).__init__(lang,remove_nuktas,nasals_mode,do_normalize_chandras,do_normalize_vowel_ending)
+        self.do_remap_assamese_chars=do_remap_assamese_chars
+
+    def normalize(self,text):
+
+        # common normalization for Indic scripts 
+        text=super(BengaliNormalizer,self).normalize(text)
+
+        # decomposing Nukta based composite characters
+        text=text.replace('\u09dc','\u09a1'+BengaliNormalizer.NUKTA)
+        text=text.replace('\u09dd','\u09a2'+BengaliNormalizer.NUKTA)
+        text=text.replace('\u09df','\u09af'+BengaliNormalizer.NUKTA)
+
+        if self.remove_nuktas:
+            text=text.replace(BengaliNormalizer.NUKTA,'')
+
+        if self.do_remap_assamese_chars and self.lang=='as':
+            text=text.replace('\u09f0','\u09b0')  #  'ra' character
+            text=text.replace('\u09f1','\u09ac')  #  'va' character 
+
+        # replace the poorna virama codes specific to script 
+        # with generic Indic script codes
+        text=text.replace('\u09e4','\u0964')
+        text=text.replace('\u09e5','\u0965')
+
+        # replace pipe character for poorna virama 
+        text=text.replace('\u007c','\u0964')
+        # replace bengali currency numerator four for poorna virama  (it looks similar and is used as a substitute)
+        text=text.replace('\u09f7','\u0964')
+
+        # two part dependent vowels
+        text=text.replace('\u09c7\u09be','\u09cb')
+        text=text.replace('\u09c7\u09d7','\u09cc')
+
+        # correct visarge 
+        text=re.sub(r'([\u0980-\u09ff]):','\\1\u0983',text)
+
+        return text
+
+
+class TamilNormalizer(BaseNormalizer): 
+    """
+    Normalizer for the Tamil script. In addition to basic normalization by the super class, 
+
+    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama 
+    * canonicalize two-part dependent vowel signs
+    * replace colon ':' by visarga if the colon follows a charcter in this script 
+    """
+
+    def __init__(self,lang='ta',remove_nuktas=False,nasals_mode='do_nothing',
+            do_normalize_chandras=False,do_normalize_vowel_ending=False):
+        super(TamilNormalizer,self).__init__(lang,remove_nuktas,nasals_mode,do_normalize_chandras,do_normalize_vowel_ending)
+
+    def normalize(self,text): 
+
+        # common normalization for Indic scripts 
+        text=super(TamilNormalizer,self).normalize(text)
+
+        # replace the poorna virama codes specific to script 
+        # with generic Indic script codes
+        text=text.replace('\u0be4','\u0964')
+        text=text.replace('\u0be5','\u0965')
+
+        # two part dependent vowels
+        text=text.replace('\u0b92\u0bd7','\u0b94')
+        text=text.replace('\u0bc6\u0bbe','\u0bca')
+        text=text.replace('\u0bc7\u0bbe','\u0bcb')
+        text=text.replace('\u0bc6\u0bd7','\u0bcc')
+
+        # correct visarge 
+        text=re.sub(r'([\u0b80-\u0bff]):','\\1\u0b83',text)
+
+        return text
+
+
+class TeluguNormalizer(BaseNormalizer): 
+    """
+    Normalizer for the Teluguscript. In addition to basic normalization by the super class, 
+
+    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama 
+    * canonicalize two-part dependent vowel signs
+    * replace colon ':' by visarga if the colon follows a charcter in this script 
+    """
+
+    def __init__(self,lang='te',remove_nuktas=False,nasals_mode='do_nothing',
+                do_normalize_chandras=False,do_normalize_vowel_ending=False):
+        super(TeluguNormalizer,self).__init__(lang,remove_nuktas,nasals_mode,do_normalize_chandras,do_normalize_vowel_ending)
+
+    def normalize(self,text): 
+
+        # common normalization for Indic scripts 
+        text=super(TeluguNormalizer,self).normalize(text)
+
+        # replace the poorna virama codes specific to script 
+        # with generic Indic script codes
+        text=text.replace('\u0c64','\u0964')
+        text=text.replace('\u0c65','\u0965')
+
+        # dependent vowels
+        text=text.replace('\u0c46\u0c56','\u0c48')
+
+        # correct visarge 
+        text=re.sub(r'([\u0c00-\u0c7f]):','\\1\u0c03',text)
+
+        return text
+
+    def get_char_stats(self,text):
+        pass 
+
+class KannadaNormalizer(BaseNormalizer): 
+    """
+    Normalizer for the Kannada script. In addition to basic normalization by the super class, 
+
+    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama 
+    * canonicalize two-part dependent vowel signs
+    * replace colon ':' by visarga if the colon follows a charcter in this script 
+    """
+
+    def __init__(self,lang='kn',remove_nuktas=False,nasals_mode='do_nothing',
+            do_normalize_chandras=False,do_normalize_vowel_ending=False):
+        super(KannadaNormalizer,self).__init__(lang,remove_nuktas,nasals_mode,do_normalize_chandras,do_normalize_vowel_ending)
+
+
+    def normalize(self,text): 
+
+        # common normalization for Indic scripts 
+        text=super(KannadaNormalizer,self).normalize(text)
+
+        # replace the poorna virama codes specific to script 
+        # with generic Indic script codes
+        text=text.replace('\u0ce4','\u0964')
+        text=text.replace('\u0ce5','\u0965')
+
+        # dependent vowels
+        text=text.replace('\u0cbf\u0cd5','\u0cc0')
+        text=text.replace('\u0cc6\u0cd5','\u0cc7')
+        text=text.replace('\u0cc6\u0cd6','\u0cc8')
+        text=text.replace('\u0cc6\u0cc2','\u0cca')
+        text=text.replace('\u0cca\u0cd5','\u0ccb')
+
+        # correct visarge 
+        text=re.sub(r'([\u0c80-\u0cff]):','\\1\u0c83',text)
+
+        return text
+
+
+class MalayalamNormalizer(BaseNormalizer): 
+    """
+    Normalizer for the Malayalam script. In addition to basic normalization by the super class, 
+
+    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama 
+    * canonicalize two-part dependent vowel signs
+    * Change from old encoding of chillus (till Unicode 5.0) to new encoding
+    * replace colon ':' by visarga if the colon follows a charcter in this script 
+    """
+
+    CHILLU_CHAR_MAP= {
+                    '\u0d7a': '\u0d23', 
+                    '\u0d7b': '\u0d28',
+                    '\u0d7c': '\u0d30',
+                    '\u0d7d': '\u0d32',
+                    '\u0d7e': '\u0d33',
+                    '\u0d7f': '\u0d15',
+                 }
+
+    def _canonicalize_chillus(self,text):
+        for chillu, char in MalayalamNormalizer.CHILLU_CHAR_MAP.items(): 
+            text=text.replace(chillu,'{}\u0d4d'.format(char)) 
+        return text
+
+    def _correct_geminated_T(self,text):
+        return text.replace('\u0d31\u0d4d\u0d31','\u0d1f\u0d4d\u0d1f')
+
+    def __init__(self,lang='ml',remove_nuktas=False,nasals_mode='do_nothing',do_normalize_chandras=False,
+                do_normalize_vowel_ending=False,
+                do_canonicalize_chillus=False, do_correct_geminated_T=False):
+        super(MalayalamNormalizer,self).__init__(lang,remove_nuktas,nasals_mode,do_normalize_chandras,do_normalize_vowel_ending)
+        self.do_canonicalize_chillus=do_canonicalize_chillus
+        self.do_correct_geminated_T=do_correct_geminated_T
+
+    def normalize(self,text): 
+
+        # Change from old encoding of chillus (till Unicode 5.0) to new encoding
+        text=text.replace('\u0d23\u0d4d\u200d','\u0d7a')
+        text=text.replace('\u0d28\u0d4d\u200d','\u0d7b')
+        text=text.replace('\u0d30\u0d4d\u200d','\u0d7c')
+        text=text.replace('\u0d32\u0d4d\u200d','\u0d7d')
+        text=text.replace('\u0d33\u0d4d\u200d','\u0d7e')
+        text=text.replace('\u0d15\u0d4d\u200d','\u0d7f')
+
+        # Normalize chillus
+        if self.do_canonicalize_chillus:
+            text=self._canonicalize_chillus(text)
+
+        # common normalization for Indic scripts 
+        text=super(MalayalamNormalizer,self).normalize(text)
+
+        # replace the poorna virama codes specific to script 
+        # with generic Indic script codes
+        text=text.replace('\u0d64','\u0964')
+        text=text.replace('\u0d65','\u0965')
+
+        # dependent vowels
+        text=text.replace('\u0d46\u0d3e','\u0d4a')
+        text=text.replace('\u0d47\u0d3e','\u0d4b')
+
+        # au forms
+        text=text.replace('\u0d46\u0d57','\u0d4c')
+        text=text.replace('\u0d57','\u0d4c')
+
+        # correct geminated T
+        if self.do_correct_geminated_T:
+            text=self._correct_geminated_T(text)
+
+        # correct visarga 
+        text=re.sub(r'([\u0d00-\u0d7f]):','\\1\u0d03',text)
+
+        return text
+
+class UrduNormalizer(NormalizerI):
+    '''Uses UrduHack library.
+    https://docs.urduhack.com/en/stable/_modules/urduhack/normalization/character.html#normalize
+    '''
+
+    def __init__(self, lang, remove_nuktas=True):
+        self.lang = lang
+        self.remove_nuktas = remove_nuktas
+    
+        from urduhack.normalization import (
+            remove_diacritics,
+            normalize_characters,
+            normalize_combine_characters
+        ) # TODO: Use only required normalizers
+        from urduhack.preprocessing import (
+            normalize_whitespace,
+            digits_space,
+            all_punctuations_space,
+            english_characters_space
+        )
+
+    def normalize(self, text):
+        text = self._normalize_punctuations(text)
+        text = UrduNormalizer.normalize_whitespace(text)
+        if self.remove_nuktas:
+            text = UrduNormalizer.remove_diacritics(text)
+        text = UrduNormalizer.normalize_characters(text)
+        text = UrduNormalizer.normalize_combine_characters(text)
+        text = UrduNormalizer.digits_space(text)
+        text = UrduNormalizer.all_punctuations_space(text)
+        text = UrduNormalizer.english_characters_space(text)
+        return text
+
+
+class IndicNormalizerFactory(object):
+    """
+    Factory class to create language specific normalizers. 
+
+    """
+
+    def get_normalizer(self,language,**kwargs):
+        """
+            Call the get_normalizer function to get the language specific normalizer
+
+            Paramters: 
+            |language: language code
+            |remove_nuktas: boolean, should the normalizer remove nukta characters 
+        """
+        normalizer=None
+        if language in ['hi','mr','sa','kK','ne','sd']:
+            normalizer=DevanagariNormalizer(lang=language, **kwargs)
+        elif language in ['ur']:
+            normalizer = UrduNormalizer(lang=language, **kwargs)
+        elif language in ['pa']:
+            normalizer=GurmukhiNormalizer(lang=language, **kwargs)
+        elif language in ['gu']:
+            normalizer=GujaratiNormalizer(lang=language, **kwargs)
+        elif language in ['bn']:
+            normalizer=BengaliNormalizer(lang=language, **kwargs)
+        elif language in ['as']:
+            normalizer=BengaliNormalizer(lang=language, **kwargs)
+        elif language in ['or']:
+            normalizer=OriyaNormalizer(lang=language, **kwargs)
+        elif language in ['ml']:
+            normalizer=MalayalamNormalizer(lang=language, **kwargs)
+        elif language in ['kn']:
+            normalizer=KannadaNormalizer(lang=language, **kwargs)
+        elif language in ['ta']:
+            normalizer=TamilNormalizer(lang=language, **kwargs)
+        elif language in ['te']:
+            normalizer=TeluguNormalizer(lang=language, **kwargs)
+        else:    
+            normalizer=BaseNormalizer(lang=language, **kwargs)
+
+        return normalizer    
+
+    def is_language_supported(self,language): 
+        """
+        Is the language supported?
+        """
+        if language in ['hi','mr','sa','kK','ne','sd',
+                        'ur',
+                        'pa',
+                        'gu',
+                        'bn','as',
+                        'or',
+                        'ml',
+                        'kn',
+                        'ta',
+                        'te']:
+            return True
+        else:
+            return False
+
+
+if __name__ == '__main__': 
+
+    if len(sys.argv)<4:
+        print("Usage: python normalize.py <infile> <outfile> <language> [<replace_nukta(True,False)>] [<normalize_nasals(do_nothing|to_anusvaara_strict|to_anusvaara_relaxed|to_nasal_consonants)>]") 
+        sys.exit(1)
+
+    language=sys.argv[3]
+    remove_nuktas=False
+    normalize_nasals='do_nothing'
+    if len(sys.argv)>=5:
+        remove_nuktas=bool(sys.argv[4])
+    if len(sys.argv)>=6:
+        normalize_nasals=sys.argv[5]
+
+    # create normalizer
+    factory=IndicNormalizerFactory()
+    normalizer=factory.get_normalizer(language,remove_nuktas=remove_nuktas,nasals_mode=normalize_nasals)
+
+    # DO normalization 
+    with codecs.open(sys.argv[1],'r','utf-8') as ifile:
+        with codecs.open(sys.argv[2],'w','utf-8') as ofile:
+            for line in ifile.readlines():
+                normalized_line=normalizer.normalize(line)
+                ofile.write(normalized_line)
+   
+    ## gather status about normalization 
+    #with codecs.open(sys.argv[1],'r','utf-8') as ifile:
+    #    normalizer=DevanagariNormalizer()
+    #    text=string.join(ifile.readlines(),sep='')
+    #    normalizer.get_char_stats(text)

indic_nlp_library/indicnlp/script/english_script.py

@@ -0,0 +1,154 @@
+# 
+#  Copyright (c) 2013-present, Anoop Kunchukuttan
+#  All rights reserved.
+#  
+#  This source code is licensed under the MIT license found in the
+#  LICENSE file in the root directory of this source tree.
+# 
+
+import pandas as pd
+import numpy as np
+
+from indicnlp import common
+from indicnlp.common import IndicNlpException
+
+
+#### Maps from ARPABET to Internal Id
+ARPABET_ID_MAP={}
+ID_ARPABET_MAP={}
+
+
+###
+# Phonetic Information about script characters 
+###
+
+""" Phonetic data for English """
+ENGLISH_PHONETIC_DATA=None
+
+""" Phonetic vector for English"""
+ENGLISH_PHONETIC_VECTORS=None
+
+""" Length of phonetic vector """
+PHONETIC_VECTOR_LENGTH=38
+
+""" Start offset for the phonetic feature vector in the phonetic data vector """
+PHONETIC_VECTOR_START_OFFSET=6
+
+## PHONETIC PROPERTIES in order in which they occur in the vector 
+## This list must be in sync with the keys in the PV_PROP_RANGES dictionary 
+PV_PROP=['basic_type',
+    'vowel_length',
+    'vowel_strength',
+    'vowel_status',
+    'consonant_type',
+    'articulation_place',
+    'aspiration',
+    'voicing',
+    'nasalization',
+    'vowel_horizontal',
+    'vowel_vertical',
+    'vowel_roundness',
+    ]
+
+### 
+# Bit vector ranges for various properties 
+###
+
+PV_PROP_RANGES={
+        'basic_type': [0,6],
+        'vowel_length': [6,8],
+        'vowel_strength': [8,11],
+        'vowel_status': [11,13],
+        'consonant_type': [13,18],
+        'articulation_place': [18,23],
+        'aspiration': [23,25],
+        'voicing': [25,27],
+        'nasalization': [27,29],
+        'vowel_horizontal': [29,32],
+        'vowel_vertical': [32,36],
+        'vowel_roundness': [36,38],
+        }
+
+
+####
+# Indexes into the Phonetic Vector 
+####
+PVIDX_BT_VOWEL=0
+PVIDX_BT_CONSONANT=1
+PVIDX_BT_NUKTA=2
+PVIDX_BT_HALANT=3
+PVIDX_BT_ANUSVAAR=4
+PVIDX_BT_MISC=5
+PVIDX_BT_S=PVIDX_BT_VOWEL 
+PVIDX_BT_E=PVIDX_BT_MISC+1
+
+PVIDX_VSTAT_DEP=12
+
+####
+SCRIPT_RANGE_START=0x0D00
+## TBD
+SCRIPT_RANGE_END=0x0D2E
+
+
+def init():
+    """
+    To be called by library loader, do not call it in your program 
+    """
+
+    global ENGLISH_PHONETIC_DATA, ENGLISH_PHONETIC_VECTORS, PHONETIC_VECTOR_LENGTH, PHONETIC_VECTOR_START_OFFSET
+
+    ENGLISH_PHONETIC_DATA=pd.read_csv(common.get_resources_path()+'/script/english_script_phonetic_data.csv',encoding='utf-8')    
+
+    ENGLISH_PHONETIC_VECTORS=ENGLISH_PHONETIC_DATA.iloc[:,PHONETIC_VECTOR_START_OFFSET:].values
+
+    PHONETIC_VECTOR_LENGTH=ENGLISH_PHONETIC_VECTORS.shape[1]
+
+    ### Load mapping from ARPABET representation of phoneme to internal ID
+    global ARPABET_ID_MAP, ID_ARPABET_MAP
+
+    with open(common.get_resources_path()+'/script/english_arpabet_list.csv','r',encoding='utf-8') as infile: 
+        for ph_id, name in enumerate(iter(infile)): 
+            name=name.strip()
+            ARPABET_ID_MAP[name]=ph_id
+            ID_ARPABET_MAP[ph_id]=name
+
+
+def phoneme_to_offset(ph): 
+    return ARPABET_ID_MAP[ph]
+
+def offset_to_phoneme(ph_id): 
+    return ID_ARPABET_MAP[ph_id]
+
+def phoneme_to_enc(ph): 
+    return chr(SCRIPT_RANGE_START+phoneme_to_offset(ph))
+
+def enc_to_phoneme(ph): 
+    return offset_to_phoneme(enc_to_offset(ph))
+
+def enc_to_offset(c): 
+    return ord(c)-SCRIPT_RANGE_START
+
+def in_range(offset): 
+    return offset>=SCRIPT_RANGE_START and offset<SCRIPT_RANGE_END 
+
+def get_phonetic_info(lang): 
+    return (ENGLISH_PHONETIC_DATA, ENGLISH_PHONETIC_VECTORS)
+
+def invalid_vector():
+    ##  TODO: check if np datatype is correct?
+    return np.array([0]*PHONETIC_VECTOR_LENGTH)
+
+def get_phonetic_feature_vector(p,lang):
+
+    offset=enc_to_offset(p) 
+
+    if not in_range(offset): 
+        return invalid_vector()
+
+    phonetic_data, phonetic_vectors= get_phonetic_info(lang)
+
+    if phonetic_data.iloc[offset]['Valid Vector Representation']==0: 
+        return invalid_vector()
+
+    return phonetic_vectors[offset]
+

indic_nlp_library/indicnlp/script/indic_scripts.py

@@ -0,0 +1,301 @@
+# 
+#  Copyright (c) 2013-present, Anoop Kunchukuttan
+#  All rights reserved.
+#  
+#  This source code is licensed under the MIT license found in the
+#  LICENSE file in the root directory of this source tree.
+# 
+
+import pandas as pd
+import numpy as np
+import os
+
+from indicnlp import common
+from indicnlp.common import IndicNlpException
+from indicnlp import langinfo as li
+
+###
+# Phonetic Information about script characters 
+###
+
+""" Phonetic data about all languages except Tamil """
+ALL_PHONETIC_DATA=None
+
+""" Phonetic data for Tamil """
+TAMIL_PHONETIC_DATA=None
+
+""" Phonetic vector for all languages except Tamil """
+ALL_PHONETIC_VECTORS=None
+
+""" Phonetic vector for Tamil """
+TAMIL_PHONETIC_VECTORS=None
+
+""" Length of phonetic vector """
+PHONETIC_VECTOR_LENGTH=38
+
+""" Start offset for the phonetic feature vector in the phonetic data vector """
+PHONETIC_VECTOR_START_OFFSET=6
+
+## PHONETIC PROPERTIES in order in which they occur in the vector 
+## This list must be in sync with the keys in the PV_PROP_RANGES dictionary 
+PV_PROP=['basic_type',
+    'vowel_length',
+    'vowel_strength',
+    'vowel_status',
+    'consonant_type',
+    'articulation_place',
+    'aspiration',
+    'voicing',
+    'nasalization',
+    'vowel_horizontal',
+    'vowel_vertical',
+    'vowel_roundness',
+    ]
+
+### 
+# Bit vector ranges for various properties 
+###
+
+PV_PROP_RANGES={
+        'basic_type': [0,6],
+        'vowel_length': [6,8],
+        'vowel_strength': [8,11],
+        'vowel_status': [11,13],
+        'consonant_type': [13,18],
+        'articulation_place': [18,23],
+        'aspiration': [23,25],
+        'voicing': [25,27],
+        'nasalization': [27,29],
+        'vowel_horizontal': [29,32],
+        'vowel_vertical': [32,36],
+        'vowel_roundness': [36,38],
+        }
+
+
+####
+# Indexes into the Phonetic Vector 
+####
+PVIDX_BT_VOWEL=0
+PVIDX_BT_CONSONANT=1
+PVIDX_BT_NUKTA=2
+PVIDX_BT_HALANT=3
+PVIDX_BT_ANUSVAAR=4
+PVIDX_BT_MISC=5
+PVIDX_BT_S=PVIDX_BT_VOWEL 
+PVIDX_BT_E=PVIDX_BT_MISC+1
+
+PVIDX_VSTAT_DEP=12
+
+#####
+# Unicode information about characters 
+#####
+
+SCRIPT_OFFSET_START=0
+SCRIPT_OFFSET_RANGE=0x80
+
+def init():
+    """
+    To be called by library loader, do not call it in your program 
+    """
+
+    global ALL_PHONETIC_DATA, ALL_PHONETIC_VECTORS, TAMIL_PHONETIC_DATA, TAMIL_PHONETIC_VECTORS, PHONETIC_VECTOR_LENGTH, PHONETIC_VECTOR_START_OFFSET
+
+    ALL_PHONETIC_DATA=pd.read_csv(os.path.join(common.get_resources_path(),'script','all_script_phonetic_data.csv'),encoding='utf-8')    
+    TAMIL_PHONETIC_DATA=pd.read_csv(os.path.join(common.get_resources_path(),'script','tamil_script_phonetic_data.csv'),encoding='utf-8')    
+
+    ALL_PHONETIC_VECTORS= ALL_PHONETIC_DATA.iloc[:,PHONETIC_VECTOR_START_OFFSET:].values
+    TAMIL_PHONETIC_VECTORS=TAMIL_PHONETIC_DATA.iloc[:,PHONETIC_VECTOR_START_OFFSET:].values
+
+    PHONETIC_VECTOR_LENGTH=ALL_PHONETIC_VECTORS.shape[1]
+
+def is_supported_language(lang): 
+    return lang in list(li.SCRIPT_RANGES.keys())
+
+def get_offset(c,lang): 
+    if not is_supported_language(lang): 
+        raise IndicNlpException('Language {}  not supported'.format(lang))
+    return ord(c)-li.SCRIPT_RANGES[lang][0]
+
+def offset_to_char(off,lang): 
+    """
+    Applicable to Brahmi derived Indic scripts 
+    """
+    if not is_supported_language(lang): 
+        raise IndicNlpException('Language {}  not supported'.format(lang))
+    return chr(off+li.SCRIPT_RANGES[lang][0])
+
+def is_indiclang_char(c,lang): 
+    """
+    Applicable to Brahmi derived Indic scripts 
+    Note that DANDA and DOUBLE_DANDA have the same Unicode codepoint for all Indic scripts 
+    """
+    if not is_supported_language(lang): 
+        raise IndicNlpException('Language {}  not supported'.format(lang))
+    o=get_offset(c,lang)
+    return (o>=SCRIPT_OFFSET_START and o<SCRIPT_OFFSET_RANGE) \
+            or ord(c)==li.DANDA or ord(c)==li.DOUBLE_DANDA
+
+def in_coordinated_range_offset(c_offset): 
+    """ 
+    Applicable to Brahmi derived Indic scripts 
+    """
+    return  (c_offset>=li.COORDINATED_RANGE_START_INCLUSIVE and c_offset<=li.COORDINATED_RANGE_END_INCLUSIVE) 
+
+def in_coordinated_range(c,lang):
+    if not is_supported_language(lang): 
+        raise IndicNlpException('Language {}  not supported'.format(lang))
+    return in_coordinated_range_offset(get_offset(c,lang))
+
+def get_phonetic_info(lang): 
+    if not is_supported_language(lang): 
+        raise IndicNlpException('Language {}  not supported'.format(lang))
+    phonetic_data= ALL_PHONETIC_DATA if lang!=li.LC_TA else TAMIL_PHONETIC_DATA
+    phonetic_vectors= ALL_PHONETIC_VECTORS if lang!=li.LC_TA else TAMIL_PHONETIC_VECTORS
+   
+    return (phonetic_data, phonetic_vectors)
+
+def invalid_vector():
+    ##  TODO: check if np datatype is correct?
+    return np.array([0]*PHONETIC_VECTOR_LENGTH)
+
+def get_phonetic_feature_vector(c,lang):
+
+    offset=get_offset(c,lang) 
+
+    if not in_coordinated_range_offset(offset): 
+        return invalid_vector()
+
+    phonetic_data, phonetic_vectors= get_phonetic_info(lang)
+
+    if phonetic_data.iloc[offset]['Valid Vector Representation']==0: 
+        return invalid_vector()
+
+    return phonetic_vectors[offset]
+
+def get_phonetic_feature_vector_offset(offset,lang):
+
+    if not in_coordinated_range_offset(offset): 
+        return invalid_vector()
+
+    phonetic_data, phonetic_vectors= get_phonetic_info(lang)
+
+    if phonetic_data.iloc[offset]['Valid Vector Representation']==0: 
+        return invalid_vector()
+
+    return phonetic_vectors[offset]
+
+### Unary operations on vectors 
+def is_valid(v): 
+    return np.sum(v)>0
+
+def is_vowel(v): 
+    return v[PVIDX_BT_VOWEL]==1 
+
+def is_consonant(v): 
+    return  v[PVIDX_BT_CONSONANT]==1 
+
+def is_halant(v): 
+    return  v[PVIDX_BT_HALANT]==1 
+
+def is_nukta(v): 
+    return  v[PVIDX_BT_NUKTA]==1 
+
+def is_anusvaar(v): 
+    return  v[PVIDX_BT_ANUSVAAR]==1 
+
+def is_misc(v): 
+    return v[PVIDX_BT_MISC]==1 
+
+def is_dependent_vowel(v): 
+    return is_vowel(v) and v[PVIDX_VSTAT_DEP]==1
+
+def is_plosive(v): 
+    return is_consonant(v) and get_property_vector(v,'consonant_type')[0]==1
+
+### Binary operations on phonetic vectors
+
+def or_vectors(v1,v2): 
+    return np.array([ 1 if (b1+b2)>=1 else 0 for b1,b2 in zip(v1,v2) ])
+
+def xor_vectors(v1,v2): 
+    return np.array([ 1 if b1!=b2 else 0 for b1,b2 in zip(v1,v2) ])
+
+### Getting properties from phonetic vectors 
+
+def get_property_vector(v,prop_name): 
+    return v[PV_PROP_RANGES[prop_name][0]:PV_PROP_RANGES[prop_name][1]]
+
+def get_property_value(v,prop_name): 
+    factor_bits=get_property_vector(v,prop_name).tolist()
+    
+    v=0
+    c=1
+    for b in factor_bits[::-1]: 
+        v+=(c*b)
+        c=c*2.0
+
+    return int(v)
+
+def lcsr_indic(srcw,tgtw,slang,tlang):
+    """
+    compute the Longest Common Subsequence Ratio (LCSR) between two strings at the character level.
+    This works for Indic scripts by mapping both languages to a common script
+
+    srcw: source language string
+    tgtw: source language string
+    slang: source language 
+    tlang: target language 
+    """    
+    score_mat=np.zeros((len(srcw)+1,len(tgtw)+1))
+
+    for si,sc in enumerate(srcw,1): 
+        for ti,tc in enumerate(tgtw,1): 
+            so=get_offset(sc,slang)
+            to=get_offset(tc,tlang)
+
+            if in_coordinated_range_offset(so) and in_coordinated_range_offset(to) and so==to: 
+                score_mat[si,ti]=score_mat[si-1,ti-1]+1.0
+            elif not (in_coordinated_range_offset(so) or in_coordinated_range_offset(to)) and sc==tc: 
+                score_mat[si,ti]=score_mat[si-1,ti-1]+1.0
+            else: 
+                score_mat[si,ti]= max(
+                    score_mat[si,ti-1],
+                    score_mat[si-1,ti])
+
+    return (score_mat[-1,-1]/float(max(len(srcw),len(tgtw))),float(len(srcw)),float(len(tgtw)))
+
+def lcsr_any(srcw,tgtw):
+    """
+    LCSR computation if both languages have the same script
+    """
+    score_mat=np.zeros((len(srcw)+1,len(tgtw)+1))
+
+    for si,sc in enumerate(srcw,1): 
+        for ti,tc in enumerate(tgtw,1): 
+
+            if sc==tc: 
+                score_mat[si,ti]=score_mat[si-1,ti-1]+1.0
+            else: 
+                score_mat[si,ti]= max(
+                    score_mat[si,ti-1],
+                    score_mat[si-1,ti])
+
+    return (score_mat[-1,-1]/float(max(len(srcw),len(tgtw))),float(len(srcw)),float(len(tgtw)))
+
+def lcsr(srcw,tgtw,slang,tlang):
+    """
+    compute the Longest Common Subsequence Ratio (LCSR) between two strings at the character level.
+
+    srcw: source language string
+    tgtw: source language string
+    slang: source language 
+    tlang: target language 
+    """
+
+    if slang==tlang or not is_supported_language(slang) or not is_supported_language(tlang):
+        return lcsr_any(srcw,tgtw,slang,tlang)
+    else:  
+        return lcsr_indic(srcw,tgtw)
+
+
+

indic_nlp_library/indicnlp/script/phonetic_sim.py

@@ -0,0 +1,59 @@
+# 
+#  Copyright (c) 2013-present, Anoop Kunchukuttan
+#  All rights reserved.
+#  
+#  This source code is licensed under the MIT license found in the
+#  LICENSE file in the root directory of this source tree.
+# 
+
+from indicnlp import loader
+from indicnlp import langinfo
+from indicnlp.script.indic_scripts import * 
+import numpy as np
+import gzip
+import pandas as pd
+import sys 
+
+def equal(v1,v2): 
+    return 0.0 if  np.sum( xor_vectors(v1, v2)) > 0  else 1.0
+
+def dice(v1,v2):
+    dotprod=2*float(np.dot( v1, v2.T ))
+    return dotprod/float(len(v1)+len(v2))
+
+def jaccard(v1,v2):
+    dotprod=float(np.dot( v1, v2.T ))
+    return dotprod/float(len(v1)+len(v2)-dotprod)
+
+def cosine(v1,v2):
+    dotprod=float(np.dot( v1, v2.T ))
+    norm1=float(np.dot( v1, v1.T ))
+    norm2=float(np.dot( v2, v2.T ))
+    return ((dotprod)/(np.sqrt(norm1*norm2)+0.00001))
+
+def dotprod(v1,v2): 
+    return float(np.dot( v1, v2.T ))
+
+def sim1(v1,v2,base=5.0): 
+    return np.power(base,dotprod(v1,v2)) 
+
+def softmax(v1,v2): 
+    return sim1(v1,v2,np.e)
+
+def create_similarity_matrix(sim_func,slang,tlang,normalize=True):
+
+    dim=langinfo.COORDINATED_RANGE_END_INCLUSIVE-langinfo.COORDINATED_RANGE_START_INCLUSIVE+1    
+    sim_mat=np.zeros((dim,dim))    
+
+    for offset1 in range(langinfo.COORDINATED_RANGE_START_INCLUSIVE, langinfo.COORDINATED_RANGE_END_INCLUSIVE+1): 
+        v1=get_phonetic_feature_vector(offset_to_char(offset1,slang),slang)
+        for offset2 in range(langinfo.COORDINATED_RANGE_START_INCLUSIVE, langinfo.COORDINATED_RANGE_END_INCLUSIVE+1): 
+            v2=get_phonetic_feature_vector(offset_to_char(offset2,tlang),tlang)
+            sim_mat[offset1,offset2]=sim_func(v1,v2)
+
+    if normalize: 
+        sums=np.sum(sim_mat, axis=1)
+        sim_mat=(sim_mat.transpose()/sums).transpose()
+
+    return sim_mat
+

indic_nlp_library/indicnlp/syllable/syllabifier.py

@@ -0,0 +1,302 @@
+# 
+#  Copyright (c) 2013-present, Anoop Kunchukuttan
+#  All rights reserved.
+#  
+#  This source code is licensed under the MIT license found in the
+#  LICENSE file in the root directory of this source tree.
+# 
+
+import codecs, sys
+from indicnlp.script import indic_scripts as si 
+import re
+
+chillu_char_map= {
+                    '\u0d7a': '\u0d23', 
+                    '\u0d7b': '\u0d28',
+                    '\u0d7c': '\u0d30',
+                    '\u0d7d': '\u0d32',
+                    '\u0d7e': '\u0d33',
+                    '\u0d7f': '\u0d15',
+                 }
+
+char_chillu_map= {}
+for k,v in chillu_char_map.items():
+    char_chillu_map[v]=k
+
+def normalize_malayalam(word): 
+
+    word_mask=re.sub(r'[0-9]','0',word)
+
+    # instead of chillu characters, use consonant+halant 
+    for chillu,char in chillu_char_map.items(): 
+        word=word.replace(chillu,'{}\u0d4d'.format(char)) 
+        word_mask=word_mask.replace(chillu,'41')
+
+    word_mask=re.sub(r'[^0-9]','0',word_mask)
+
+    return word, word_mask
+
+def denormalize_malayalam(word, word_mask):
+
+    word=list(word)
+    word_mask=list(word_mask)
+
+    ## pattern 4
+    idx=0
+    while idx>=0: 
+        try: 
+            idx=word_mask.index('4',idx)
+            word[idx:idx+2]=char_chillu_map[word[idx]]
+            word_mask[idx:idx+2]='0'
+            start=idx            
+        except ValueError as e: 
+            break
+
+    return ''.join(word)
+
+def normalize_punjabi(word): 
+    word_mask=re.sub(r'[0-9]','0',word)
+
+    ## replace tippi with anusvaar
+    word=word.replace('\u0a70','\u0a02')
+    word_mask=word_mask.replace('\u0a70','2')
+
+    ## replace addak+consonant with consonat+halant+consonant
+    word=re.sub(r'\u0a71(.)','\\1\u0a4d\\1',word)
+    word_mask=re.sub(r'\u0a71(.)','311',word_mask)
+
+    word_mask=re.sub(r'[^0-9]','0',word_mask)
+
+    return word, word_mask
+
+def denormalize_punjabi(word, word_mask):
+    
+    word=list(word)
+    word_mask=list(word_mask)
+
+    ## pattern 2
+    idx=0
+    while idx>=0: 
+        try: 
+            idx=word_mask.index('2',idx)
+            word[idx]='\u0a70'
+            word_mask[idx]='0'
+            start=idx            
+        except ValueError as e: 
+            break
+
+    ## pattern 3
+    idx=0
+    while idx>=0: 
+        try: 
+            idx=word_mask.index('3',idx)
+            word[idx:idx+3]='\u0a71{}'.format(word[idx])
+            word_mask[idx:idx+3]='00'
+            start=idx            
+        except ValueError as e: 
+            break
+
+    return ''.join(word)
+
+def char_backoff(syllables_list,vocab):
+    syllables_final=[]    
+
+    if vocab is None:
+        syllables_final=syllables_list
+    else:
+        for s in syllables_list:
+            if s in vocab: 
+                syllables_final.append(s)
+            else: 
+                for x in s:
+                    syllables_final.append(x)                    
+
+    return syllables_final        
+
+
+def orthographic_syllabify_improved(word,lang,vocab=None): 
+
+    word_mask=['0']*len(word)
+
+    if lang=='ml':
+        word, word_mask = normalize_malayalam(word)
+        word=word
+    elif lang=='pa':
+        word, word_mask = normalize_punjabi(word)
+
+    p_vectors=[si.get_phonetic_feature_vector(c,lang) for c in word]
+
+    syllables=[]
+    syllables_mask=[]
+
+    for i in range(len(word)): 
+        v=p_vectors[i]
+
+        syllables.append(word[i])
+        syllables_mask.append(word_mask[i])
+
+        ### simplified syllabification 
+        #if i+1<len(word) and \
+        #        (not si.is_valid(p_vectors[i+1]) or si.is_misc(p_vectors[i+1])):
+        #    syllables.append(u' ')
+        #    syllables_mask.append(u'0')
+
+        #elif not si.is_valid(v) or si.is_misc(v) or si.is_vowel(v):
+        #    syllables.append(u' ')
+        #    syllables_mask.append(u'0')
+
+        #elif i+1<len(word) and \
+        #     (si.is_consonant(v) or si.is_nukta(v)) and \
+        #     (si.is_consonant(p_vectors[i+1]) or si.is_anusvaar(p_vectors[i+1])):
+        #    syllables.append(u' ')
+        #    syllables_mask.append(u'0')
+
+        #### better syllabification 
+        if i+1<len(word) and (not si.is_valid(p_vectors[i+1]) or si.is_misc(p_vectors[i+1])):
+            syllables.append(' ')
+            syllables_mask.append('0')
+
+        elif not si.is_valid(v) or si.is_misc(v) :
+            syllables.append(' ')
+            syllables_mask.append('0')
+
+        elif si.is_vowel(v):
+
+            anu_nonplos= ( i+2<len(word) and \
+                           si.is_anusvaar(p_vectors[i+1]) and \
+                           not si.is_plosive(p_vectors[i+2])\
+                         )
+            
+            anu_eow= ( i+2==len(word) and \
+                           si.is_anusvaar(p_vectors[i+1]) )
+
+            if not(anu_nonplos or anu_eow):              
+                syllables.append(' ')
+                syllables_mask.append('0')
+
+        elif i+1<len(word) and \
+                (si.is_consonant(v) or si.is_nukta(v)): 
+            if si.is_consonant(p_vectors[i+1]): 
+                syllables.append(' ')
+                syllables_mask.append('0')
+            elif si.is_vowel(p_vectors[i+1]) and \
+                    not si.is_dependent_vowel(p_vectors[i+1]): 
+                syllables.append(' ')
+                syllables_mask.append('0')
+            elif si.is_anusvaar(p_vectors[i+1]):
+                anu_nonplos= ( i+2<len(word) and \
+                               not si.is_plosive(p_vectors[i+2])\
+                             )
+                
+                anu_eow= i+2==len(word) 
+
+                if not(anu_nonplos or anu_eow):              
+                    syllables.append(' ')
+                    syllables_mask.append('0')
+
+    syllables_mask=''.join(syllables_mask)
+    syllables=''.join(syllables)
+
+    #assert len(syllables_mask) == len(syllables)
+    #assert syllables_mask.find('01') == -1
+    if syllables_mask.find('01') >= 0: 
+        print('Warning')
+
+    if lang=='ml': 
+        syllables = denormalize_malayalam(syllables,syllables_mask)
+    elif lang=='pa': 
+        syllables = denormalize_punjabi(syllables,syllables_mask)
+
+    syllables_list = syllables.strip().split(' ')  
+    return(char_backoff(syllables_list,vocab))    
+
+def orthographic_syllabify(word,lang,vocab=None): 
+
+    p_vectors=[si.get_phonetic_feature_vector(c,lang) for c in word]
+
+    syllables=[]
+
+    for i in range(len(word)): 
+        v=p_vectors[i]
+
+        syllables.append(word[i])
+
+        ### simplified syllabification 
+        #if i+1<len(word) and \
+        #        (not si.is_valid(p_vectors[i+1]) or si.is_misc(p_vectors[i+1])):
+        #    syllables.append(u' ')
+
+        #elif not si.is_valid(v) or si.is_misc(v) or si.is_vowel(v):
+        #    syllables.append(u' ')
+
+        #elif i+1<len(word) and \
+        #     (si.is_consonant(v) or si.is_nukta(v)) and \
+        #     (si.is_consonant(p_vectors[i+1]) or si.is_anusvaar(p_vectors[i+1])):
+        #    syllables.append(u' ')
+
+        #### better syllabification 
+        if i+1<len(word) and (not si.is_valid(p_vectors[i+1]) or si.is_misc(p_vectors[i+1])):
+            syllables.append(' ')
+
+        elif not si.is_valid(v) or si.is_misc(v) :
+            syllables.append(' ')
+
+        elif si.is_vowel(v):
+
+            anu_nonplos= ( i+2<len(word) and \
+                           si.is_anusvaar(p_vectors[i+1]) and \
+                           not si.is_plosive(p_vectors[i+2])\
+                         )
+            
+            anu_eow= ( i+2==len(word) and \
+                           si.is_anusvaar(p_vectors[i+1]) )
+
+            if not(anu_nonplos or anu_eow):              
+                syllables.append(' ')
+
+        elif i+1<len(word) and \
+                (si.is_consonant(v) or si.is_nukta(v)): 
+            if si.is_consonant(p_vectors[i+1]): 
+                syllables.append(' ')
+            elif si.is_vowel(p_vectors[i+1]) and \
+                    not si.is_dependent_vowel(p_vectors[i+1]): 
+                syllables.append(' ')
+            elif si.is_anusvaar(p_vectors[i+1]):
+                anu_nonplos= ( i+2<len(word) and \
+                               not si.is_plosive(p_vectors[i+2])\
+                             )
+                
+                anu_eow= i+2==len(word) 
+
+                if not(anu_nonplos or anu_eow):              
+                    syllables.append(' ')
+
+    syllables_list = ''.join(syllables).strip().split(' ') 
+    return(char_backoff(syllables_list,vocab))
+
+def orthographic_simple_syllabify(word,lang,vocab=None): 
+
+    p_vectors=[si.get_phonetic_feature_vector(c,lang) for c in word]
+
+    syllables=[]
+
+    for i in range(len(word)): 
+        v=p_vectors[i]
+
+        syllables.append(word[i])
+
+        ## simplified syllabification 
+        if i+1<len(word) and \
+                (not si.is_valid(p_vectors[i+1]) or si.is_misc(p_vectors[i+1])):
+            syllables.append(' ')
+
+        elif not si.is_valid(v) or si.is_misc(v) or si.is_vowel(v):
+            syllables.append(' ')
+
+        elif i+1<len(word) and \
+             (si.is_consonant(v) or si.is_nukta(v)) and \
+             (si.is_consonant(p_vectors[i+1]) or si.is_anusvaar(p_vectors[i+1])):
+            syllables.append(' ')
+
+    syllables_list = ''.join(syllables).strip().split(' ') 
+    return(char_backoff(syllables_list,vocab))