Support single integer or empty list as input to decode (#7)

- Update tokenization_chatglm.py (dffe870a7ef1558ebbc6f3dfdf46491cdb2b3e31)


Co-authored-by: Yichao 'Peak' Ji <peakji@users.noreply.huggingface.co>
This commit is contained in:
Zhengxiao Du 2023-04-04 09:48:07 +00:00 committed by system
parent f82b180d8d
commit fdb7a601d8
1 changed files with 4 additions and 0 deletions

View File

@ -261,6 +261,10 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
spaces_between_special_tokens: bool = True,
**kwargs
) -> str:
if not isinstance(token_ids, list):
token_ids = [token_ids]
if len(token_ids) == 0:
return ""
if isinstance(token_ids[0], list):
tokens = []
for single_token_ids in token_ids: