change hebrew code from iw (deprecated) to he (#6)
- change hebrew code from iw (deprecated) to he (2f637251702bb9799886af508a2863d3584bbb1a) Co-authored-by: Altryne <altryne@users.noreply.huggingface.co>
This commit is contained in:
parent
38b4c8a3e4
commit
42189dea11
|
@ -29,6 +29,7 @@
|
|||
"<|gu|>": 50333,
|
||||
"<|haw|>": 50352,
|
||||
"<|ha|>": 50354,
|
||||
"<|he|>": 50279,
|
||||
"<|hi|>": 50276,
|
||||
"<|hr|>": 50291,
|
||||
"<|ht|>": 50339,
|
||||
|
@ -37,7 +38,6 @@
|
|||
"<|id|>": 50275,
|
||||
"<|is|>": 50311,
|
||||
"<|it|>": 50274,
|
||||
"<|iw|>": 50279,
|
||||
"<|ja|>": 50266,
|
||||
"<|jw|>": 50356,
|
||||
"<|ka|>": 50329,
|
||||
|
|
|
@ -49,6 +49,7 @@
|
|||
"<|gu|>": 50333,
|
||||
"<|haw|>": 50352,
|
||||
"<|ha|>": 50354,
|
||||
"<|he|>": 50279,
|
||||
"<|hi|>": 50276,
|
||||
"<|hr|>": 50291,
|
||||
"<|ht|>": 50339,
|
||||
|
@ -57,7 +58,6 @@
|
|||
"<|id|>": 50275,
|
||||
"<|is|>": 50311,
|
||||
"<|it|>": 50274,
|
||||
"<|iw|>": 50279,
|
||||
"<|ja|>": 50266,
|
||||
"<|jw|>": 50356,
|
||||
"<|ka|>": 50329,
|
||||
|
|
|
@ -22,7 +22,7 @@
|
|||
"<|hi|>",
|
||||
"<|fi|>",
|
||||
"<|vi|>",
|
||||
"<|iw|>",
|
||||
"<|he|>",
|
||||
"<|uk|>",
|
||||
"<|el|>",
|
||||
"<|ms|>",
|
||||
|
|
|
@ -203,7 +203,7 @@
|
|||
},
|
||||
{
|
||||
"id": 50279,
|
||||
"content": "<|iw|>",
|
||||
"content": "<|he|>",
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
|
|
Loading…
Reference in New Issue