Support single integer or empty list as input to decode (#7)
- Update tokenization_chatglm.py (dffe870a7ef1558ebbc6f3dfdf46491cdb2b3e31) Co-authored-by: Yichao 'Peak' Ji <peakji@users.noreply.huggingface.co>
This commit is contained in:
parent
f82b180d8d
commit
fdb7a601d8
|
@ -261,6 +261,10 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
|
||||||
spaces_between_special_tokens: bool = True,
|
spaces_between_special_tokens: bool = True,
|
||||||
**kwargs
|
**kwargs
|
||||||
) -> str:
|
) -> str:
|
||||||
|
if not isinstance(token_ids, list):
|
||||||
|
token_ids = [token_ids]
|
||||||
|
if len(token_ids) == 0:
|
||||||
|
return ""
|
||||||
if isinstance(token_ids[0], list):
|
if isinstance(token_ids[0], list):
|
||||||
tokens = []
|
tokens = []
|
||||||
for single_token_ids in token_ids:
|
for single_token_ids in token_ids:
|
||||||
|
|
Loading…
Reference in New Issue