diff --git a/tokenization_chatglm.py b/tokenization_chatglm.py index 02aa8cd..39aaa19 100644 --- a/tokenization_chatglm.py +++ b/tokenization_chatglm.py @@ -200,6 +200,12 @@ class ChatGLMTokenizer(PreTrainedTokenizer): """ Initialisation """ + @property + def gmask_token_id(self) -> Optional[int]: + if self.gmask_token is None: + return None + return self.convert_tokens_to_ids(self.gmask_token) + @property def eop_token_id(self) -> Optional[int]: """