Yntec commited on
Commit
bd2750c
1 Parent(s): 525a01b

Update tokenizer/tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer/tokenizer_config.json +4 -2
tokenizer/tokenizer_config.json CHANGED
@@ -7,7 +7,7 @@
7
  "normalized": true,
8
  "rstrip": false,
9
  "single_word": false,
10
- "special": true
11
  },
12
  "49407": {
13
  "content": "<|endoftext|>",
@@ -15,9 +15,10 @@
15
  "normalized": false,
16
  "rstrip": false,
17
  "single_word": false,
18
- "special": true
19
  }
20
  },
 
21
  "bos_token": "<|startoftext|>",
22
  "clean_up_tokenization_spaces": true,
23
  "do_lower_case": true,
@@ -26,5 +27,6 @@
26
  "model_max_length": 77,
27
  "pad_token": "<|endoftext|>",
28
  "tokenizer_class": "CLIPTokenizer",
 
29
  "unk_token": "<|endoftext|>"
30
  }
 
7
  "normalized": true,
8
  "rstrip": false,
9
  "single_word": false,
10
+ "special": false
11
  },
12
  "49407": {
13
  "content": "<|endoftext|>",
 
15
  "normalized": false,
16
  "rstrip": false,
17
  "single_word": false,
18
+ "special": false
19
  }
20
  },
21
+ "additional_special_tokens": [],
22
  "bos_token": "<|startoftext|>",
23
  "clean_up_tokenization_spaces": true,
24
  "do_lower_case": true,
 
27
  "model_max_length": 77,
28
  "pad_token": "<|endoftext|>",
29
  "tokenizer_class": "CLIPTokenizer",
30
+ "tokenizer_file": "/home/user/.cache/huggingface/hub/models--openai--clip-vit-large-patch14/snapshots/32bd64288804d66eefd0ccbe215aa642df71cc41/tokenizer.json",
31
  "unk_token": "<|endoftext|>"
32
  }