diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..533b68c --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,5 @@ +{ + "do_lower_case": false, + "word_tokenizer_type": "mecab", + "subword_tokenizer_type": "wordpiece" +}