Add gmask token id

This commit is contained in:
duzx16 2023-04-06 22:43:35 +08:00
parent 63ce1bac4a
commit 36b7f2d0ad
1 changed files with 6 additions and 0 deletions

View File

@ -200,6 +200,12 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
""" Initialisation """
@property
def gmask_token_id(self) -> Optional[int]:
if self.gmask_token is None:
return None
return self.convert_tokens_to_ids(self.gmask_token)
@property
def eop_token_id(self) -> Optional[int]:
"""