Update tokenizer_config.json

2020-06-07 12:26:07 +00:00 · 2020-06-07 12:26:07 +00:00 · 30931391d3
parent f9b8d28251
commit 30931391d3
1 changed files with 5 additions and 0 deletions
--- a/tokenizer_config.json
+++ b/tokenizer_config.json
@ -0,0 +1,5 @@
+{
+    "do_lower_case": false,
+    "word_tokenizer_type": "mecab",
+    "subword_tokenizer_type": "wordpiece"
+}