diff --git a/tokenizer_config.json b/tokenizer_config.json index a233160..978a15a 100644 --- a/tokenizer_config.json +++ b/tokenizer_config.json @@ -1 +1 @@ -{"unk_token": "", "bos_token": "", "eos_token": "", "pad_token": "", "do_lower_case": false, "return_attention_mask": false, "do_normalize": false} \ No newline at end of file +{"unk_token": "", "bos_token": "", "eos_token": "", "pad_token": "", "do_lower_case": false, "return_attention_mask": false, "do_normalize": true} \ No newline at end of file