From d7fd482af9cd2039abe9970a49bbefa090583fc2 Mon Sep 17 00:00:00 2001
From: Seethal Elias <seethal.elias@aot-technologies.com>
Date: Wed, 13 Apr 2022 18:49:43 +0000
Subject: [PATCH] Upload tokenizer_config.json

---
 tokenizer_config.json | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 tokenizer_config.json

diff --git a/tokenizer_config.json b/tokenizer_config.json
new file mode 100644
index 0000000..4036a93
--- /dev/null
+++ b/tokenizer_config.json
@@ -0,0 +1 @@
+{"do_lower_case": true, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "model_max_length": 512, "special_tokens_map_file": null, "name_or_path": "distilbert-base-uncased", "tokenizer_class": "DistilBertTokenizer"}
\ No newline at end of file