EleutherAI
/
gpt-j-6B
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update to GPT2 style

This commit is contained in:
Stella Biderman 2021-08-05 20:05:38 +00:00 committed by huggingface-web
parent 658edeb2a7
commit 220857605d
1 changed files with 4 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
config.json
View File

@ -4,30 +4,18 @@
"GPTJForCausalLM" "GPTJForCausalLM"
], ],
"attention_dropout": 0, "attention_dropout": 0,
"attention_layers": ["global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global"],
"attention_types": [
[
[
"global"
],
28
]
],
"bos_token_id": 50256, "bos_token_id": 50256,
"embed_dropout": 0, "embed_dropout": 0,
"eos_token_id": 50256, "eos_token_id": 50256,
"gradient_checkpointing": false, "gradient_checkpointing": false,
"hidden_size": 4096, "n_embd": 4096,
"initializer_range": 0.02, "initializer_range": 0.02,
"intermediate_size": null, "intermediate_size": null,
"jax": true,
"layer_norm_epsilon": 1e-05, "layer_norm_epsilon": 1e-05,
"max_position_embeddings": 2048, "n_positions": 2048,
"model_type": "gptj", "model_type": "gptj",
"num_heads": 16, "n_head": 16,
"num_layers": 28, "n_layer": 28,
"resid_dropout": 0,
"rotary": true,
"rotary_dim": 64, "rotary_dim": 64,
"summary_activation": null, "summary_activation": null,
"summary_first_dropout": 0.1, "summary_first_dropout": 0.1,