grammarBERT / added_tokens.json
Nbeau's picture
Upload tokenizer
0fa71a2 verified
{
"AnnAssign": 50274,
"Assert": 50278,
"Assign": 50272,
"AsyncFor": 50275,
"AsyncFunctionDef": 50270,
"AsyncWith": 50276,
"AugAssign": 50273,
"AugLoad": 50302,
"AugStore": 50303,
"Await": 50293,
"BinOp": 50284,
"BitAnd": 50312,
"BitOr": 50310,
"BitXor": 50311,
"BoolOp": 50282,
"ClassDef": 50271,
"Constant": 50298,
"Dict": 50288,
"DictComp": 50291,
"Eq": 50317,
"ExceptHandler": 50325,
"Expr": 50281,
"Expression": 50266,
"ExtSlice": 50305,
"FloorDiv": 50313,
"FormattedValue": 50296,
"FunctionDef": 50269,
"FunctionType": 50267,
"GeneratorExp": 50292,
"Gt": 50320,
"GtE": 50321,
"IfExp": 50287,
"ImportFrom": 50279,
"Interactive": 50265,
"Invert": 50314,
"IsNot": 50322,
"JoinedStr": 50297,
"LShift": 50308,
"Lambda": 50286,
"ListComp": 50289,
"LtE": 50319,
"MatMult": 50306,
"NamedExpr": 50283,
"Nonlocal": 50280,
"NotEq": 50318,
"NotIn": 50323,
"Pow": 50307,
"RShift": 50309,
"Raise": 50277,
"Reduce": 50330,
"Reduce_primitif": 50331,
"SetComp": 50290,
"Slice": 50304,
"Starred": 50300,
"Subscript": 50299,
"Suite": 50268,
"Tuple": 50301,
"TypeIgnore": 50329,
"UAdd": 50315,
"USub": 50316,
"UnaryOp": 50285,
"Yield": 50294,
"YieldFrom": 50295,
"arguments": 50326,
"comprehension": 50324,
"keyword": 50327,
"withitem": 50328
}