diff --git a/tokenizer_config.json b/tokenizer_config.json index 9f2cec8..97d4216 100644 --- a/tokenizer_config.json +++ b/tokenizer_config.json @@ -1 +1 @@ -{"unk_token": "", "bos_token": "", "eos_token": "", "pad_token": "", "do_lower_case": false} \ No newline at end of file +{"unk_token": "", "bos_token": "", "eos_token": "", "pad_token": "", "do_lower_case": false, "return_attention_mask": true, "do_normalize": true} \ No newline at end of file