[config.json] tokenizer_class: BertJapaneseTokenizer

2020-11-19 14:16:34 +01:00 · 2020-11-19 14:16:34 +01:00 · 855e787bb3
parent 30931391d3
commit 855e787bb3
1 changed files with 1 additions and 0 deletions
--- a/config.json
+++ b/config.json
@ -14,6 +14,7 @@
  "num_attention_heads": 12,
  "num_hidden_layers": 12,
  "pad_token_id": 0,
+  "tokenizer_class": "BertJapaneseTokenizer",
  "type_vocab_size": 2,
  "vocab_size": 32000
 }