From 9f18ffc800d8d4b31c382531bf47393dc301806b Mon Sep 17 00:00:00 2001 From: Arthur Zucker Date: Fri, 3 Feb 2023 13:28:29 +0000 Subject: [PATCH] change hebrew code from iw (deprecated) to he (#18) - change hebrew code from iw (deprecated) to he (55b872f5c860ae8edd7134293b2fd22a1357a007) Co-authored-by: Altryne --- README.md | 2 +- added_tokens.json | 2 +- generation_config.json | 2 +- special_tokens_map.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c83b9d7..555b1fe 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ language: - hi - fi - vi -- iw +- he - uk - el - ms diff --git a/added_tokens.json b/added_tokens.json index 47e9dd3..1e8cf95 100644 --- a/added_tokens.json +++ b/added_tokens.json @@ -30,6 +30,7 @@ "<|gu|>": 50333, "<|haw|>": 50352, "<|ha|>": 50354, + "<|he|>": 50279, "<|hi|>": 50276, "<|hr|>": 50291, "<|ht|>": 50339, @@ -38,7 +39,6 @@ "<|id|>": 50275, "<|is|>": 50311, "<|it|>": 50274, - "<|iw|>": 50279, "<|ja|>": 50266, "<|jw|>": 50356, "<|ka|>": 50329, diff --git a/generation_config.json b/generation_config.json index 82e2636..fc15027 100644 --- a/generation_config.json +++ b/generation_config.json @@ -49,6 +49,7 @@ "<|gu|>": 50333, "<|haw|>": 50352, "<|ha|>": 50354, + "<|he|>": 50279, "<|hi|>": 50276, "<|hr|>": 50291, "<|ht|>": 50339, @@ -57,7 +58,6 @@ "<|id|>": 50275, "<|is|>": 50311, "<|it|>": 50274, - "<|iw|>": 50279, "<|ja|>": 50266, "<|jw|>": 50356, "<|ka|>": 50329, diff --git a/special_tokens_map.json b/special_tokens_map.json index 9115b68..7977e47 100644 --- a/special_tokens_map.json +++ b/special_tokens_map.json @@ -22,7 +22,7 @@ "<|hi|>", "<|fi|>", "<|vi|>", - "<|iw|>", + "<|he|>", "<|uk|>", "<|el|>", "<|ms|>",