Upload tokenizer
This commit is contained in:
parent
f11df2a199
commit
c703e332b3
|
@ -109,5 +109,6 @@
|
|||
],
|
||||
"bos_token": "<|endoftext|>",
|
||||
"eos_token": "<|endoftext|>",
|
||||
"pad_token": "<|endoftext|>",
|
||||
"unk_token": "<|endoftext|>"
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue