xhyi
/
layoutlmv3_docvqa_t11c5000
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add tokenizer

This commit is contained in:
xhyi 2022-07-22 11:52:54 -07:00
parent 519f37fd49
commit a0efbb32b5
No known key found for this signature in database
GPG Key ID: E6BB1BF2F869ECC9
5 changed files with 150495 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50001
merges.txt Normal file
View File

File diff suppressed because it is too large Load Diff

51
special_tokens_map.json Normal file
View File

@ -0,0 +1,51 @@
{
"bos_token": {
"content": "<s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"cls_token": {
"content": "<s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "</s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"mask_token": {
"content": "<mask>",
"lstrip": true,
"normalized": true,
"rstrip": false,
"single_word": false
},
"pad_token": {
"content": "<pad>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"sep_token": {
"content": "</s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"unk_token": {
"content": "<unk>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}

100355
tokenizer.json Normal file
View File

File diff suppressed because it is too large Load Diff

87
tokenizer_config.json Normal file
View File

@ -0,0 +1,87 @@
{
"add_prefix_space": true,
"apply_ocr": true,
"bos_token": {
"__type": "AddedToken",
"content": "<s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"cls_token": {
"__type": "AddedToken",
"content": "<s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"cls_token_box": [
0,
0,
0,
0
],
"eos_token": {
"__type": "AddedToken",
"content": "</s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"errors": "replace",
"mask_token": {
"__type": "AddedToken",
"content": "<mask>",
"lstrip": true,
"normalized": true,
"rstrip": false,
"single_word": false
},
"model_max_length": 512,
"name_or_path": "/home/xdrie/Downloads/layoutlmv3_docvqa_t11c5000/",
"only_label_first_subword": true,
"pad_token": {
"__type": "AddedToken",
"content": "<pad>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"pad_token_box": [
0,
0,
0,
0
],
"pad_token_label": -100,
"processor_class": "LayoutLMv3Processor",
"sep_token": {
"__type": "AddedToken",
"content": "</s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"sep_token_box": [
0,
0,
0,
0
],
"special_tokens_map_file": null,
"tokenizer_class": "LayoutLMv3Tokenizer",
"trim_offsets": true,
"unk_token": {
"__type": "AddedToken",
"content": "<unk>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}

1
vocab.json Normal file
View File

File diff suppressed because one or more lines are too long