From 36b7f2d0ad518fe921850cf00aebb52b9d93cce9 Mon Sep 17 00:00:00 2001
From: duzx16 <zx-du20@mails.tsinghua.edu.cn>
Date: Thu, 6 Apr 2023 22:43:35 +0800
Subject: [PATCH] Add gmask token id

---
 tokenization_chatglm.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tokenization_chatglm.py b/tokenization_chatglm.py
index 02aa8cd..39aaa19 100644
--- a/tokenization_chatglm.py
+++ b/tokenization_chatglm.py
@@ -200,6 +200,12 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
 
         """ Initialisation """
 
+    @property
+    def gmask_token_id(self) -> Optional[int]:
+        if self.gmask_token is None:
+            return None
+        return self.convert_tokens_to_ids(self.gmask_token)
+
     @property
     def eop_token_id(self) -> Optional[int]:
         """