From 42189dea11934ae9312d470debe286451387f34f Mon Sep 17 00:00:00 2001 From: Arthur Zucker Date: Tue, 31 Jan 2023 20:11:57 +0000 Subject: [PATCH] change hebrew code from iw (deprecated) to he (#6) - change hebrew code from iw (deprecated) to he (2f637251702bb9799886af508a2863d3584bbb1a) Co-authored-by: Altryne --- README.md | 2 +- added_tokens.json | 2 +- generation_config.json | 2 +- special_tokens_map.json | 2 +- tokenizer.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ffe9377..90c4a14 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ language: - hi - fi - vi -- iw +- he - uk - el - ms diff --git a/added_tokens.json b/added_tokens.json index f84c61a..a973b01 100644 --- a/added_tokens.json +++ b/added_tokens.json @@ -29,6 +29,7 @@ "<|gu|>": 50333, "<|haw|>": 50352, "<|ha|>": 50354, + "<|he|>": 50279, "<|hi|>": 50276, "<|hr|>": 50291, "<|ht|>": 50339, @@ -37,7 +38,6 @@ "<|id|>": 50275, "<|is|>": 50311, "<|it|>": 50274, - "<|iw|>": 50279, "<|ja|>": 50266, "<|jw|>": 50356, "<|ka|>": 50329, diff --git a/generation_config.json b/generation_config.json index 82e2636..fc15027 100644 --- a/generation_config.json +++ b/generation_config.json @@ -49,6 +49,7 @@ "<|gu|>": 50333, "<|haw|>": 50352, "<|ha|>": 50354, + "<|he|>": 50279, "<|hi|>": 50276, "<|hr|>": 50291, "<|ht|>": 50339, @@ -57,7 +58,6 @@ "<|id|>": 50275, "<|is|>": 50311, "<|it|>": 50274, - "<|iw|>": 50279, "<|ja|>": 50266, "<|jw|>": 50356, "<|ka|>": 50329, diff --git a/special_tokens_map.json b/special_tokens_map.json index 5edbe0e..97b19be 100644 --- a/special_tokens_map.json +++ b/special_tokens_map.json @@ -22,7 +22,7 @@ "<|hi|>", "<|fi|>", "<|vi|>", - "<|iw|>", + "<|he|>", "<|uk|>", "<|el|>", "<|ms|>", diff --git a/tokenizer.json b/tokenizer.json index f7b9886..7818adb 100644 --- a/tokenizer.json +++ b/tokenizer.json @@ -203,7 +203,7 @@ }, { "id": 50279, - "content": "<|iw|>", + "content": "<|he|>", "single_word": false, "lstrip": false, "rstrip": false,