tokenizer.json

{"class_name": "Tokenizer", "config": {"num_words": null, "filters": "!\"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n", "lower": true, "split": " ", "char_level": true, "oov_token": null, "document_count": 107, "word_counts": "{\"a\": 2, \"b\": 2, \"c\": 2, \"d\": 2, \"e\": 2, \"f\": 2, \"g\": 2, \"h\": 2, \"i\": 2, \"j\": 2, \"k\": 2, \"l\": 2, \"m\": 2, \"n\": 2, \"o\": 2, \"p\": 2, \"q\": 2, \"r\": 2, \"s\": 2, \"t\": 2, \"u\": 2, \"v\": 2, \"w\": 2, \"x\": 2, \"y\": 2, \"z\": 2, \"1\": 1, \"2\": 1, \"3\": 1, \"4\": 1, \"5\": 1, \"6\": 1, \"7\": 1, \"8\": 1, \"9\": 1, \"0\": 1, \" \": 1, \".\": 1, \",\": 1, \"!\": 1, \"?\": 1, \"@\": 1, \"#\": 1, \"$\": 1, \"%\": 1, \"^\": 1, \"&\": 1, \"*\": 1, \"(\": 1, \")\": 1, \"_\": 1, \"-\": 1, \"+\": 1, \"=\": 1, \"{\": 1, \"}\": 1, \"[\": 1, \"]\": 1, \"|\": 1, \";\": 1, \":\": 1, \"'\": 1, \"\\\"\": 1, \"\\u20ac\": 1, \"\\u00a3\": 1, \"\\u00a2\": 1, \"\\u00a5\": 1, \"\\u00a9\": 1, \"\\u00ae\": 1, \"\\u2122\": 1, \"\\u00a7\": 1, \"\\u2022\": 1, \"\\u2713\": 1, \"\\u25b2\": 1, \"\\u25bc\": 1, \"\\u25c6\": 1, \"\\u2606\": 1, \"\\u2665\": 1, \"\\u2600\": 1, \"\\ufe0f\": 2, \"\\ud83c\\udf19\": 1, \"\\u26a1\": 1}", "word_docs": "{\"a\": 2, \"b\": 2, \"c\": 2, \"d\": 2, \"e\": 2, \"f\": 2, \"g\": 2, \"h\": 2, \"i\": 2, \"j\": 2, \"k\": 2, \"l\": 2, \"m\": 2, \"n\": 2, \"o\": 2, \"p\": 2, \"q\": 2, \"r\": 2, \"s\": 2, \"t\": 2, \"u\": 2, \"v\": 2, \"w\": 2, \"x\": 2, \"y\": 2, \"z\": 2, \"1\": 1, \"2\": 1, \"3\": 1, \"4\": 1, \"5\": 1, \"6\": 1, \"7\": 1, \"8\": 1, \"9\": 1, \"0\": 1, \" \": 1, \".\": 1, \",\": 1, \"!\": 1, \"?\": 1, \"@\": 1, \"#\": 1, \"$\": 1, \"%\": 1, \"^\": 1, \"&\": 1, \"*\": 1, \"(\": 1, \")\": 1, \"_\": 1, \"-\": 1, \"+\": 1, \"=\": 1, \"{\": 1, \"}\": 1, \"[\": 1, \"]\": 1, \"|\": 1, \";\": 1, \":\": 1, \"'\": 1, \"\\\"\": 1, \"\\u20ac\": 1, \"\\u00a3\": 1, \"\\u00a2\": 1, \"\\u00a5\": 1, \"\\u00a9\": 1, \"\\u00ae\": 1, \"\\u2122\": 1, \"\\u00a7\": 1, \"\\u2022\": 1, \"\\u2713\": 1, \"\\u25b2\": 1, \"\\u25bc\": 1, \"\\u25c6\": 1, \"\\u2606\": 1, \"\\u2665\": 1, \"\\ufe0f\": 2, \"\\u2600\": 1, \"\\ud83c\\udf19\": 1, \"\\u26a1\": 1}", "index_docs": "{\"1\": 2, \"2\": 2, \"3\": 2, \"4\": 2, \"5\": 2, \"6\": 2, \"7\": 2, \"8\": 2, \"9\": 2, \"10\": 2, \"11\": 2, \"12\": 2, \"13\": 2, \"14\": 2, \"15\": 2, \"16\": 2, \"17\": 2, \"18\": 2, \"19\": 2, \"20\": 2, \"21\": 2, \"22\": 2, \"23\": 2, \"24\": 2, \"25\": 2, \"26\": 2, \"28\": 1, \"29\": 1, \"30\": 1, \"31\": 1, \"32\": 1, \"33\": 1, \"34\": 1, \"35\": 1, \"36\": 1, \"37\": 1, \"38\": 1, \"39\": 1, \"40\": 1, \"41\": 1, \"42\": 1, \"43\": 1, \"44\": 1, \"45\": 1, \"46\": 1, \"47\": 1, \"48\": 1, \"49\": 1, \"50\": 1, \"51\": 1, \"52\": 1, \"53\": 1, \"54\": 1, \"55\": 1, \"56\": 1, \"57\": 1, \"58\": 1, \"59\": 1, \"60\": 1, \"61\": 1, \"62\": 1, \"63\": 1, \"64\": 1, \"65\": 1, \"66\": 1, \"67\": 1, \"68\": 1, \"69\": 1, \"70\": 1, \"71\": 1, \"72\": 1, \"73\": 1, \"74\": 1, \"75\": 1, \"76\": 1, \"77\": 1, \"78\": 1, \"79\": 1, \"27\": 2, \"80\": 1, \"81\": 1, \"82\": 1}", "index_word": "{\"1\": \"a\", \"2\": \"b\", \"3\": \"c\", \"4\": \"d\", \"5\": \"e\", \"6\": \"f\", \"7\": \"g\", \"8\": \"h\", \"9\": \"i\", \"10\": \"j\", \"11\": \"k\", \"12\": \"l\", \"13\": \"m\", \"14\": \"n\", \"15\": \"o\", \"16\": \"p\", \"17\": \"q\", \"18\": \"r\", \"19\": \"s\", \"20\": \"t\", \"21\": \"u\", \"22\": \"v\", \"23\": \"w\", \"24\": \"x\", \"25\": \"y\", \"26\": \"z\", \"27\": \"\\ufe0f\", \"28\": \"1\", \"29\": \"2\", \"30\": \"3\", \"31\": \"4\", \"32\": \"5\", \"33\": \"6\", \"34\": \"7\", \"35\": \"8\", \"36\": \"9\", \"37\": \"0\", \"38\": \" \", \"39\": \".\", \"40\": \",\", \"41\": \"!\", \"42\": \"?\", \"43\": \"@\", \"44\": \"#\", \"45\": \"$\", \"46\": \"%\", \"47\": \"^\", \"48\": \"&\", \"49\": \"*\", \"50\": \"(\", \"51\": \")\", \"52\": \"_\", \"53\": \"-\", \"54\": \"+\", \"55\": \"=\", \"56\": \"{\", \"57\": \"}\", \"58\": \"[\", \"59\": \"]\", \"60\": \"|\", \"61\": \";\", \"62\": \":\", \"63\": \"'\", \"64\": \"\\\"\", \"65\": \"\\u20ac\", \"66\": \"\\u00a3\", \"67\": \"\\u00a2\", \"68\": \"\\u00a5\", \"69\": \"\\u00a9\", \"70\": \"\\u00ae\", \"71\": \"\\u2122\", \"72\": \"\\u00a7\", \"73\": \"\\u2022\", \"74\": \"\\u2713\", \"75\": \"\\u25b2\", \"76\": \"\\u25bc\", \"77\": \"\\u25c6\", \"78\": \"\\u2606\", \"79\": \"\\u2665\", \"80\": \"\\u2600\", \"81\": \"\\ud83c\\udf19\", \"82\": \"\\u26a1\"}", "word_index": "{\"a\": 1, \"b\": 2, \"c\": 3, \"d\": 4, \"e\": 5, \"f\": 6, \"g\": 7, \"h\": 8, \"i\": 9, \"j\": 10, \"k\": 11, \"l\": 12, \"m\": 13, \"n\": 14, \"o\": 15, \"p\": 16, \"q\": 17, \"r\": 18, \"s\": 19, \"t\": 20, \"u\": 21, \"v\": 22, \"w\": 23, \"x\": 24, \"y\": 25, \"z\": 26, \"\\ufe0f\": 27, \"1\": 28, \"2\": 29, \"3\": 30, \"4\": 31, \"5\": 32, \"6\": 33, \"7\": 34, \"8\": 35, \"9\": 36, \"0\": 37, \" \": 38, \".\": 39, \",\": 40, \"!\": 41, \"?\": 42, \"@\": 43, \"#\": 44, \"$\": 45, \"%\": 46, \"^\": 47, \"&\": 48, \"*\": 49, \"(\": 50, \")\": 51, \"_\": 52, \"-\": 53, \"+\": 54, \"=\": 55, \"{\": 56, \"}\": 57, \"[\": 58, \"]\": 59, \"|\": 60, \";\": 61, \":\": 62, \"'\": 63, \"\\\"\": 64, \"\\u20ac\": 65, \"\\u00a3\": 66, \"\\u00a2\": 67, \"\\u00a5\": 68, \"\\u00a9\": 69, \"\\u00ae\": 70, \"\\u2122\": 71, \"\\u00a7\": 72, \"\\u2022\": 73, \"\\u2713\": 74, \"\\u25b2\": 75, \"\\u25bc\": 76, \"\\u25c6\": 77, \"\\u2606\": 78, \"\\u2665\": 79, \"\\u2600\": 80, \"\\ud83c\\udf19\": 81, \"\\u26a1\": 82}"}}