change hebrew code from iw (deprecated) to he (#6)

- change hebrew code from iw (deprecated) to he (2f637251702bb9799886af508a2863d3584bbb1a)


Co-authored-by: Altryne <altryne@users.noreply.huggingface.co>
This commit is contained in:
Arthur Zucker 2023-01-31 20:11:57 +00:00 committed by system
parent 38b4c8a3e4
commit 42189dea11
5 changed files with 5 additions and 5 deletions

View File

@ -20,7 +20,7 @@ language:
- hi
- fi
- vi
- iw
- he
- uk
- el
- ms

View File

@ -29,6 +29,7 @@
"<|gu|>": 50333,
"<|haw|>": 50352,
"<|ha|>": 50354,
"<|he|>": 50279,
"<|hi|>": 50276,
"<|hr|>": 50291,
"<|ht|>": 50339,
@ -37,7 +38,6 @@
"<|id|>": 50275,
"<|is|>": 50311,
"<|it|>": 50274,
"<|iw|>": 50279,
"<|ja|>": 50266,
"<|jw|>": 50356,
"<|ka|>": 50329,

View File

@ -49,6 +49,7 @@
"<|gu|>": 50333,
"<|haw|>": 50352,
"<|ha|>": 50354,
"<|he|>": 50279,
"<|hi|>": 50276,
"<|hr|>": 50291,
"<|ht|>": 50339,
@ -57,7 +58,6 @@
"<|id|>": 50275,
"<|is|>": 50311,
"<|it|>": 50274,
"<|iw|>": 50279,
"<|ja|>": 50266,
"<|jw|>": 50356,
"<|ka|>": 50329,

View File

@ -22,7 +22,7 @@
"<|hi|>",
"<|fi|>",
"<|vi|>",
"<|iw|>",
"<|he|>",
"<|uk|>",
"<|el|>",
"<|ms|>",

View File

@ -203,7 +203,7 @@
},
{
"id": 50279,
"content": "<|iw|>",
"content": "<|he|>",
"single_word": false,
"lstrip": false,
"rstrip": false,