openai
/
whisper-tiny
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Upload tokenizer

This commit is contained in:
Arthur Zucker 2022-09-28 08:15:02 +00:00 committed by huggingface-web
parent 4c640cdbe0
commit c65924544a
3 changed files with 125 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

213
added_tokens.json
View File

@ -1,108 +1,109 @@
{ {
"<|af|>": 50326, "<|af|>": 50327,
"<|am|>": 50333, "<|am|>": 50334,
"<|ar|>": 50271, "<|ar|>": 50272,
"<|as|>": 50349, "<|as|>": 50350,
"<|az|>": 50303, "<|az|>": 50304,
"<|ba|>": 50354, "<|ba|>": 50355,
"<|be|>": 50329, "<|be|>": 50330,
"<|bg|>": 50291, "<|bg|>": 50292,
"<|bn|>": 50301, "<|bn|>": 50302,
"<|bo|>": 50346, "<|bo|>": 50347,
"<|br|>": 50308, "<|br|>": 50309,
"<|bs|>": 50314, "<|bs|>": 50315,
"<|ca|>": 50269, "<|ca|>": 50270,
"<|cs|>": 50282, "<|cs|>": 50283,
"<|cy|>": 50296, "<|cy|>": 50297,
"<|da|>": 50284, "<|da|>": 50285,
"<|de|>": 50260, "<|de|>": 50261,
"<|el|>": 50280, "<|el|>": 50281,
"<|en|>": 50258, "<|endoftext|>": 50257,
"<|es|>": 50261, "<|en|>": 50259,
"<|et|>": 50306, "<|es|>": 50262,
"<|eu|>": 50309, "<|et|>": 50307,
"<|fa|>": 50299, "<|eu|>": 50310,
"<|fi|>": 50276, "<|fa|>": 50300,
"<|fo|>": 50337, "<|fi|>": 50277,
"<|fr|>": 50264, "<|fo|>": 50338,
"<|gl|>": 50318, "<|fr|>": 50265,
"<|gu|>": 50332, "<|gl|>": 50319,
"<|haw|>": 50351, "<|gu|>": 50333,
"<|ha|>": 50353, "<|haw|>": 50352,
"<|hi|>": 50275, "<|ha|>": 50354,
"<|hr|>": 50290, "<|hi|>": 50276,
"<|ht|>": 50338, "<|hr|>": 50291,
"<|hu|>": 50285, "<|ht|>": 50339,
"<|hy|>": 50311, "<|hu|>": 50286,
"<|id|>": 50274, "<|hy|>": 50312,
"<|is|>": 50310, "<|id|>": 50275,
"<|it|>": 50273, "<|is|>": 50311,
"<|iw|>": 50278, "<|it|>": 50274,
"<|ja|>": 50265, "<|iw|>": 50279,
"<|jw|>": 50355, "<|ja|>": 50266,
"<|ka|>": 50328, "<|jw|>": 50356,
"<|kk|>": 50315, "<|ka|>": 50329,
"<|km|>": 50322, "<|kk|>": 50316,
"<|kn|>": 50305, "<|km|>": 50323,
"<|ko|>": 50263, "<|kn|>": 50306,
"<|la|>": 50293, "<|ko|>": 50264,
"<|lb|>": 50344, "<|la|>": 50294,
"<|ln|>": 50352, "<|lb|>": 50345,
"<|lo|>": 50335, "<|ln|>": 50353,
"<|lt|>": 50292, "<|lo|>": 50336,
"<|lv|>": 50300, "<|lt|>": 50293,
"<|mg|>": 50348, "<|lv|>": 50301,
"<|mi|>": 50294, "<|mg|>": 50349,
"<|mk|>": 50307, "<|mi|>": 50295,
"<|ml|>": 50295, "<|mk|>": 50308,
"<|mn|>": 50313, "<|ml|>": 50296,
"<|mr|>": 50319, "<|mn|>": 50314,
"<|ms|>": 50281, "<|mr|>": 50320,
"<|mt|>": 50342, "<|ms|>": 50282,
"<|my|>": 50345, "<|mt|>": 50343,
"<|ne|>": 50312, "<|my|>": 50346,
"<|nl|>": 50270, "<|ne|>": 50313,
"<|nn|>": 50341, "<|nl|>": 50271,
"<|nocaptions|>": 50361, "<|nn|>": 50342,
"<|notimestamps|>": 50362, "<|nocaptions|>": 50362,
"<|no|>": 50287, "<|notimestamps|>": 50363,
"<|oc|>": 50327, "<|no|>": 50288,
"<|pa|>": 50320, "<|oc|>": 50328,
"<|pl|>": 50268, "<|pa|>": 50321,
"<|ps|>": 50339, "<|pl|>": 50269,
"<|pt|>": 50266, "<|ps|>": 50340,
"<|ro|>": 50283, "<|pt|>": 50267,
"<|ru|>": 50262, "<|ro|>": 50284,
"<|sa|>": 50343, "<|ru|>": 50263,
"<|sd|>": 50331, "<|sa|>": 50344,
"<|si|>": 50321, "<|sd|>": 50332,
"<|sk|>": 50297, "<|si|>": 50322,
"<|sl|>": 50304, "<|sk|>": 50298,
"<|sn|>": 50323, "<|sl|>": 50305,
"<|so|>": 50325, "<|sn|>": 50324,
"<|sq|>": 50316, "<|so|>": 50326,
"<|sr|>": 50302, "<|sq|>": 50317,
"<|startoflm|>": 50359, "<|sr|>": 50303,
"<|startofprev|>": 50360, "<|startoflm|>": 50360,
"<|startoftranscript|>": 50257, "<|startofprev|>": 50361,
"<|su|>": 50356, "<|startoftranscript|>": 50258,
"<|sv|>": 50272, "<|su|>": 50357,
"<|sw|>": 50317, "<|sv|>": 50273,
"<|ta|>": 50286, "<|sw|>": 50318,
"<|te|>": 50298, "<|ta|>": 50287,
"<|tg|>": 50330, "<|te|>": 50299,
"<|th|>": 50288, "<|tg|>": 50331,
"<|tk|>": 50340, "<|th|>": 50289,
"<|tl|>": 50347, "<|tk|>": 50341,
"<|transcribe|>": 50358, "<|tl|>": 50348,
"<|translate|>": 50357, "<|transcribe|>": 50359,
"<|tr|>": 50267, "<|translate|>": 50358,
"<|tt|>": 50350, "<|tr|>": 50268,
"<|uk|>": 50279, "<|tt|>": 50351,
"<|ur|>": 50289, "<|uk|>": 50280,
"<|uz|>": 50336, "<|ur|>": 50290,
"<|vi|>": 50277, "<|uz|>": 50337,
"<|yi|>": 50334, "<|vi|>": 50278,
"<|yo|>": 50324, "<|yi|>": 50335,
"<|zh|>": 50259 "<|yo|>": 50325,
"<|zh|>": 50260
} }

19
special_tokens_map.json
View File

@ -1,5 +1,6 @@
{ {
"additional_special_tokens": [ "additional_special_tokens": [
"<|endoftext|>",
"<|startoftranscript|>", "<|startoftranscript|>",
"<|en|>", "<|en|>",
"<|zh|>", "<|zh|>",
@ -107,13 +108,25 @@
"<|nocaptions|>", "<|nocaptions|>",
"<|notimestamps|>" "<|notimestamps|>"
], ],
"bos_token": "<|endoftext|>", "bos_token": {
"eos_token": "<|endoftext|>",
"unk_token": {
"content": "<|endoftext|>", "content": "<|endoftext|>",
"lstrip": false, "lstrip": false,
"normalized": true, "normalized": true,
"rstrip": false, "rstrip": false,
"single_word": false "single_word": false
},
"eos_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"unk_token": {
"content": "",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
} }
} }

5
tokenizer_config.json
View File

@ -19,14 +19,13 @@
}, },
"errors": "replace", "errors": "replace",
"model_max_length": 1024, "model_max_length": 1024,
"name_or_path": "openai/whisper-tiny", "name_or_path": "whisper-multi/test_added_eot",
"pad_token": null, "pad_token": null,
"processor_class": "WhisperProcessor",
"special_tokens_map_file": null, "special_tokens_map_file": null,
"tokenizer_class": "WhisperTokenizer", "tokenizer_class": "WhisperTokenizer",
"unk_token": { "unk_token": {
"__type": "AddedToken", "__type": "AddedToken",
"content": "<|endoftext|>", "content": "",
"lstrip": false, "lstrip": false,
"normalized": true, "normalized": true,
"rstrip": false, "rstrip": false,