Add gmask token id
This commit is contained in:
parent
63ce1bac4a
commit
36b7f2d0ad
|
@ -200,6 +200,12 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
|
|||
|
||||
""" Initialisation """
|
||||
|
||||
@property
|
||||
def gmask_token_id(self) -> Optional[int]:
|
||||
if self.gmask_token is None:
|
||||
return None
|
||||
return self.convert_tokens_to_ids(self.gmask_token)
|
||||
|
||||
@property
|
||||
def eop_token_id(self) -> Optional[int]:
|
||||
"""
|
||||
|
|
Loading…
Reference in New Issue