openai
/
clip-vit-base-patch16
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add tokenizer.json

This commit is contained in:
SaulLu 2022-03-03 10:52:49 +01:00
parent 10fcb38b50
commit a564c9d054
2 changed files with 98427 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98393
tokenizer.json Normal file
View File

File diff suppressed because it is too large Load Diff

35
tokenizer_config.json
View File

@ -1 +1,34 @@
{"errors": "replace", "unk_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "bos_token": {"content": "<|startoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "pad_token": "<|endoftext|>", "add_prefix_space": false, "do_lower_case": true, "name_or_path": "openai/clip-vit-base-patch32", "model_max_length": 77, "tokenizer_class": "CLIPTokenizer"}
{
"unk_token": {
"content": "<|endoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true,
"__type": "AddedToken"
},
"bos_token": {
"content": "<|startoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true,
"__type": "AddedToken"
},
"eos_token": {
"content": "<|endoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true,
"__type": "AddedToken"
},
"pad_token": "<|endoftext|>",
"errors": "replace",
"add_prefix_space": false,
"do_lower_case": true,
"name_or_path": "openai/clip-vit-base-patch16",
"model_max_length": 77,
"special_tokens_map_file": "./special_tokens_map.json",
"tokenizer_class": "CLIPTokenizer"
}