change hebrew code from iw (deprecated) to he (#6)
- change hebrew code from iw (deprecated) to he (2f637251702bb9799886af508a2863d3584bbb1a) Co-authored-by: Altryne <altryne@users.noreply.huggingface.co>
This commit is contained in:
parent
38b4c8a3e4
commit
42189dea11
|
@ -20,7 +20,7 @@ language:
|
||||||
- hi
|
- hi
|
||||||
- fi
|
- fi
|
||||||
- vi
|
- vi
|
||||||
- iw
|
- he
|
||||||
- uk
|
- uk
|
||||||
- el
|
- el
|
||||||
- ms
|
- ms
|
||||||
|
|
|
@ -29,6 +29,7 @@
|
||||||
"<|gu|>": 50333,
|
"<|gu|>": 50333,
|
||||||
"<|haw|>": 50352,
|
"<|haw|>": 50352,
|
||||||
"<|ha|>": 50354,
|
"<|ha|>": 50354,
|
||||||
|
"<|he|>": 50279,
|
||||||
"<|hi|>": 50276,
|
"<|hi|>": 50276,
|
||||||
"<|hr|>": 50291,
|
"<|hr|>": 50291,
|
||||||
"<|ht|>": 50339,
|
"<|ht|>": 50339,
|
||||||
|
@ -37,7 +38,6 @@
|
||||||
"<|id|>": 50275,
|
"<|id|>": 50275,
|
||||||
"<|is|>": 50311,
|
"<|is|>": 50311,
|
||||||
"<|it|>": 50274,
|
"<|it|>": 50274,
|
||||||
"<|iw|>": 50279,
|
|
||||||
"<|ja|>": 50266,
|
"<|ja|>": 50266,
|
||||||
"<|jw|>": 50356,
|
"<|jw|>": 50356,
|
||||||
"<|ka|>": 50329,
|
"<|ka|>": 50329,
|
||||||
|
|
|
@ -49,6 +49,7 @@
|
||||||
"<|gu|>": 50333,
|
"<|gu|>": 50333,
|
||||||
"<|haw|>": 50352,
|
"<|haw|>": 50352,
|
||||||
"<|ha|>": 50354,
|
"<|ha|>": 50354,
|
||||||
|
"<|he|>": 50279,
|
||||||
"<|hi|>": 50276,
|
"<|hi|>": 50276,
|
||||||
"<|hr|>": 50291,
|
"<|hr|>": 50291,
|
||||||
"<|ht|>": 50339,
|
"<|ht|>": 50339,
|
||||||
|
@ -57,7 +58,6 @@
|
||||||
"<|id|>": 50275,
|
"<|id|>": 50275,
|
||||||
"<|is|>": 50311,
|
"<|is|>": 50311,
|
||||||
"<|it|>": 50274,
|
"<|it|>": 50274,
|
||||||
"<|iw|>": 50279,
|
|
||||||
"<|ja|>": 50266,
|
"<|ja|>": 50266,
|
||||||
"<|jw|>": 50356,
|
"<|jw|>": 50356,
|
||||||
"<|ka|>": 50329,
|
"<|ka|>": 50329,
|
||||||
|
|
|
@ -22,7 +22,7 @@
|
||||||
"<|hi|>",
|
"<|hi|>",
|
||||||
"<|fi|>",
|
"<|fi|>",
|
||||||
"<|vi|>",
|
"<|vi|>",
|
||||||
"<|iw|>",
|
"<|he|>",
|
||||||
"<|uk|>",
|
"<|uk|>",
|
||||||
"<|el|>",
|
"<|el|>",
|
||||||
"<|ms|>",
|
"<|ms|>",
|
||||||
|
|
|
@ -203,7 +203,7 @@
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": 50279,
|
"id": 50279,
|
||||||
"content": "<|iw|>",
|
"content": "<|he|>",
|
||||||
"single_word": false,
|
"single_word": false,
|
||||||
"lstrip": false,
|
"lstrip": false,
|
||||||
"rstrip": false,
|
"rstrip": false,
|
||||||
|
|
Loading…
Reference in New Issue