erwt-year-st / added_tokens.json
Kaspar Beelen
add tokenizer
cd57bf1
{
"[1801]": 30579,
"[1802]": 30591,
"[1803]": 30577,
"[1804]": 30589,
"[1805]": 30596,
"[1806]": 30547,
"[1807]": 30532,
"[1808]": 30555,
"[1809]": 30530,
"[1810]": 30593,
"[1811]": 30584,
"[1812]": 30581,
"[1813]": 30553,
"[1814]": 30573,
"[1815]": 30536,
"[1816]": 30568,
"[1817]": 30587,
"[1818]": 30570,
"[1819]": 30586,
"[1820]": 30578,
"[1821]": 30597,
"[1822]": 30557,
"[1823]": 30561,
"[1824]": 30566,
"[1825]": 30569,
"[1826]": 30595,
"[1827]": 30580,
"[1828]": 30594,
"[1829]": 30582,
"[1830]": 30583,
"[1831]": 30534,
"[1832]": 30588,
"[1833]": 30590,
"[1834]": 30539,
"[1835]": 30565,
"[1836]": 30567,
"[1837]": 30549,
"[1838]": 30585,
"[1839]": 30592,
"[1840]": 30562,
"[1841]": 30541,
"[1842]": 30575,
"[1843]": 30598,
"[1844]": 30552,
"[1845]": 30554,
"[1846]": 30544,
"[1847]": 30558,
"[1848]": 30533,
"[1849]": 30531,
"[1850]": 30543,
"[1851]": 30559,
"[1852]": 30550,
"[1853]": 30551,
"[1854]": 30556,
"[1855]": 30542,
"[1856]": 30548,
"[1857]": 30563,
"[1858]": 30571,
"[1859]": 30529,
"[1860]": 30564,
"[1861]": 30538,
"[1862]": 30537,
"[1863]": 30546,
"[1864]": 30572,
"[1865]": 30535,
"[1866]": 30545,
"[1867]": 30560,
"[1868]": 30540,
"[1869]": 30576,
"[1870]": 30574,
"[1871]": 30599,
"[LOC]": 30603,
"[MET]": 30600,
"[POL]": 30602,
"[YEAR]": 30601,
"[con]": 30523,
"[lib]": 30522,
"[liverpool]": 30528,
"[london]": 30527,
"[neutr]": 30526,
"[none]": 30524,
"[rad]": 30525
}