diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..32b043b --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1 @@ +{"do_lower_case": true, "max_len": 512, "bos_token": "", "eos_token": "", "unk_token": "", "sep_token": "", "pad_token": "", "cls_token": "", "mask_token": ""} \ No newline at end of file