aicloud
/
chinese_llama_vicuna_tokenizer
generated from xuyuqing/ailab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix wrong tokenizer

This commit is contained in:
mjchen6 2023-06-08 19:02:11 +08:00
parent f7ae9c3a06
commit 25e1768c00
5 changed files with 65 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
config.json
View File

@ -1,31 +1,24 @@
{ {
"_name_or_path": "THUDM/chatglm-6b", "_name_or_path": "decapoda-research/llama-7b-hf",
"architectures": [ "architectures": [
"ChatGLMModel" "LLaMAForCausalLM"
], ],
"auto_map": { "bos_token_id": 0,
"AutoConfig": "THUDM/chatglm-6b--configuration_chatglm.ChatGLMConfig", "eos_token_id": 1,
"AutoModel": "THUDM/chatglm-6b--modeling_chatglm.ChatGLMForConditionalGeneration", "hidden_act": "silu",
"AutoModelForSeq2SeqLM": "THUDM/chatglm-6b--modeling_chatglm.ChatGLMForConditionalGeneration"
},
"bos_token_id": 130004,
"eos_token_id": 130005,
"gmask_token_id": 130001,
"hidden_size": 4096, "hidden_size": 4096,
"inner_hidden_size": 16384, "initializer_range": 0.02,
"layernorm_epsilon": 1e-05, "intermediate_size": 11008,
"mask_token_id": 130000, "max_position_embeddings": 2048,
"max_sequence_length": 2048, "max_sequence_length": 2048,
"model_type": "chatglm", "model_type": "llama",
"num_attention_heads": 32, "num_attention_heads": 32,
"num_layers": 28, "num_hidden_layers": 32,
"pad_token_id": 3, "pad_token_id": -1,
"position_encoding_2d": true, "rms_norm_eps": 1e-06,
"pre_seq_len": null, "tie_word_embeddings": false,
"prefix_projection": false,
"quantization_bit": 0,
"torch_dtype": "float16", "torch_dtype": "float16",
"transformers_version": "4.29.2", "transformers_version": "4.29.2",
"use_cache": true, "use_cache": true,
"vocab_size": 130528 "vocab_size": 32000
} }

BIN
ice_text.model
View File

Binary file not shown.

26
special_tokens_map.json
View File

@ -1,7 +1,23 @@
{ {
"bos_token": "<sop>", "bos_token": {
"eos_token": "<eop>", "content": "",
"mask_token": "[MASK]", "lstrip": false,
"pad_token": "<pad>", "normalized": true,
"unk_token": "<unk>" "rstrip": false,
"single_word": false
},
"eos_token": {
"content": "",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"unk_token": {
"content": "",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
} }

BIN
tokenizer.model Normal file
View File

Binary file not shown.

49
tokenizer_config.json
View File

@ -1,22 +1,33 @@
{ {
"auto_map": { "add_bos_token": true,
"AutoTokenizer": [ "add_eos_token": true,
"THUDM/chatglm-6b--tokenization_chatglm.ChatGLMTokenizer", "bos_token": {
null "__type": "AddedToken",
] "content": "",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}, },
"bos_token": "<sop>", "clean_up_tokenization_spaces": false,
"clean_up_tokenization_spaces": true, "eos_token": {
"do_lower_case": false, "__type": "AddedToken",
"end_token": "</s>", "content": "",
"eos_token": "<eop>", "lstrip": false,
"gmask_token": "[gMASK]", "normalized": true,
"mask_token": "[MASK]", "rstrip": false,
"model_max_length": 2048, "single_word": false
"num_image_tokens": 0, },
"pad_token": "<pad>", "model_max_length": 1000000000000000019884624838656,
"padding_side": "left", "pad_token": null,
"remove_space": false, "sp_model_kwargs": {},
"tokenizer_class": "ChatGLMTokenizer", "tokenizer_class": "LlamaTokenizer",
"unk_token": "<unk>" "unk_token": {
"__type": "AddedToken",
"content": "",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
} }