Upload tokenizer
This commit is contained in:
parent
cc0dc2035d
commit
0ce71a9bbd
|
@ -122,6 +122,7 @@
|
||||||
"rstrip": false,
|
"rstrip": false,
|
||||||
"single_word": false
|
"single_word": false
|
||||||
},
|
},
|
||||||
|
"pad_token": "<|endoftext|>",
|
||||||
"unk_token": {
|
"unk_token": {
|
||||||
"content": "",
|
"content": "",
|
||||||
"lstrip": false,
|
"lstrip": false,
|
||||||
|
|
Loading…
Reference in New Issue