aicloud
/
chatglm-6b
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add gmask token id

This commit is contained in:
duzx16 2023-04-06 22:43:35 +08:00
parent 63ce1bac4a
commit 36b7f2d0ad
1 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
tokenization_chatglm.py
View File

@ -200,6 +200,12 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
""" Initialisation """ """ Initialisation """
@property
def gmask_token_id(self) -> Optional[int]:
if self.gmask_token is None:
return None
return self.convert_tokens_to_ids(self.gmask_token)
@property @property
def eop_token_id(self) -> Optional[int]: def eop_token_id(self) -> Optional[int]:
""" """