| {"bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "sep_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "pad_token": {"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "cls_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true}, "additional_special_tokens": ["string_literal", "heredoc_content", "class_variable", "simple_symbol", "shorthand_property_identifier", "constant", "comment", "statement_identifier", "null_literal", "\"", "boolean_type", "regex_pattern", "ERROR", "integer", "raw_string_literal", "boolean", "number", "php_tag", "extends", "identifier", "instance_variable", "property_identifier", "package_identifier", "none", "label_name", "character_literal", "heredoc_beginning", "separators", "string_content", "text", "string_fragment", "regex_flags", "hash_key_symbol", "namespace", "operator", "global_variable", "string", "decimal_integer_literal", "void_type", "field_identifier", "int_literal", "rune_literal", "class", "decimal_floating_point_literal", "keyword", "escape_sequence", "shorthand_property_identifier_pattern", "heredoc_end", "float_literal", "name", "type_identifier", "hex_integer_literal"]} |