Get all text paths

src/train_tokenizer.ipynb

@@ -20,6 +20,41 @@
     "# load_dataset(\"balochiml/balochi-language-data\", data_dir=\"data\", cache_dir=\"../data\")"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "4294"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import os\n",
+    "\n",
+    "def get_txt_file_paths(directory):\n",
+    "    txt_file_paths = []\n",
+    "    for root, dirs, files in os.walk(directory):\n",
+    "        for file in files:\n",
+    "            if file.endswith(\".txt\"):\n",
+    "                file_path = os.path.join(root, file)\n",
+    "                txt_file_paths.append(file_path)\n",
+    "    return txt_file_paths\n",
+    "\n",
+    "# Replace \"directory_path\" with the actual path of the directory you want to search\n",
+    "directory_path = \"../data/raw_text\"\n",
+    "txt_paths = get_txt_file_paths(directory_path)\n",
+    "\n",
+    "len(txt_paths)\n"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,