Upload tokenizer

2022-10-05 16:27:05 +00:00 · 2022-10-05 16:27:05 +00:00 · 0ce71a9bbd
parent cc0dc2035d
commit 0ce71a9bbd
1 changed files with 1 additions and 0 deletions
--- a/special_tokens_map.json
+++ b/special_tokens_map.json
@ -122,6 +122,7 @@
    "rstrip": false,
    "single_word": false
  },
+  "pad_token": "<|endoftext|>",
  "unk_token": {
    "content": "",
    "lstrip": false,