change hebrew code from iw (deprecated) to he (#6)

- change hebrew code from iw (deprecated) to he (2f637251702bb9799886af508a2863d3584bbb1a)


Co-authored-by: Altryne <altryne@users.noreply.huggingface.co>
This commit is contained in:
Arthur Zucker 2023-01-31 20:11:57 +00:00 committed by system
parent 38b4c8a3e4
commit 42189dea11
5 changed files with 5 additions and 5 deletions

View File

@ -20,7 +20,7 @@ language:
- hi - hi
- fi - fi
- vi - vi
- iw - he
- uk - uk
- el - el
- ms - ms

View File

@ -29,6 +29,7 @@
"<|gu|>": 50333, "<|gu|>": 50333,
"<|haw|>": 50352, "<|haw|>": 50352,
"<|ha|>": 50354, "<|ha|>": 50354,
"<|he|>": 50279,
"<|hi|>": 50276, "<|hi|>": 50276,
"<|hr|>": 50291, "<|hr|>": 50291,
"<|ht|>": 50339, "<|ht|>": 50339,
@ -37,7 +38,6 @@
"<|id|>": 50275, "<|id|>": 50275,
"<|is|>": 50311, "<|is|>": 50311,
"<|it|>": 50274, "<|it|>": 50274,
"<|iw|>": 50279,
"<|ja|>": 50266, "<|ja|>": 50266,
"<|jw|>": 50356, "<|jw|>": 50356,
"<|ka|>": 50329, "<|ka|>": 50329,

View File

@ -49,6 +49,7 @@
"<|gu|>": 50333, "<|gu|>": 50333,
"<|haw|>": 50352, "<|haw|>": 50352,
"<|ha|>": 50354, "<|ha|>": 50354,
"<|he|>": 50279,
"<|hi|>": 50276, "<|hi|>": 50276,
"<|hr|>": 50291, "<|hr|>": 50291,
"<|ht|>": 50339, "<|ht|>": 50339,
@ -57,7 +58,6 @@
"<|id|>": 50275, "<|id|>": 50275,
"<|is|>": 50311, "<|is|>": 50311,
"<|it|>": 50274, "<|it|>": 50274,
"<|iw|>": 50279,
"<|ja|>": 50266, "<|ja|>": 50266,
"<|jw|>": 50356, "<|jw|>": 50356,
"<|ka|>": 50329, "<|ka|>": 50329,

View File

@ -22,7 +22,7 @@
"<|hi|>", "<|hi|>",
"<|fi|>", "<|fi|>",
"<|vi|>", "<|vi|>",
"<|iw|>", "<|he|>",
"<|uk|>", "<|uk|>",
"<|el|>", "<|el|>",
"<|ms|>", "<|ms|>",

View File

@ -203,7 +203,7 @@
}, },
{ {
"id": 50279, "id": 50279,
"content": "<|iw|>", "content": "<|he|>",
"single_word": false, "single_word": false,
"lstrip": false, "lstrip": false,
"rstrip": false, "rstrip": false,