gpt-j-6B/tokenizer.json

1 line
1.3 MiB
JSON
Raw Normal View History

2021-09-22 15:24:00 +00:00
{"version":"1.0","truncation":null,"padding":null,"added_tokens":[{"id":50256,"special":true,"content":"<|endoftext|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50257,"special":false,"content":"<|extratoken_1|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50258,"special":false,"content":"<|extratoken_2|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50259,"special":false,"content":"<|extratoken_3|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50260,"special":false,"content":"<|extratoken_4|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50261,"special":false,"content":"<|extratoken_5|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50262,"special":false,"content":"<|extratoken_6|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50263,"special":false,"content":"<|extratoken_7|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50264,"special":false,"content":"<|extratoken_8|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50265,"special":false,"content":"<|extratoken_9|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50266,"special":false,"content":"<|extratoken_10|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50267,"special":false,"content":"<|extratoken_11|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50268,"special":false,"content":"<|extratoken_12|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50269,"special":false,"content":"<|extratoken_13|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50270,"special":false,"content":"<|extratoken_14|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50271,"special":false,"content":"<|extratoken_15|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50272,"special":false,"content":"<|extratoken_16|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50273,"special":false,"content":"<|extratoken_17|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50274,"special":false,"content":"<|extratoken_18|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50275,"special":false,"content":"<|extratoken_19|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50276,"special":false,"content":"<|extratoken_20|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50277,"special":false,"content":"<|extratoken_21|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50278,"special":false,"content":"<|extratoken_22|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50279,"special":false,"content":"<|extratoken_23|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50280,"special":false,"content":"<|extratoken_24|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50281,"special":false,"content":"<|extratoken_25|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50282,"special":false,"content":"<|extratoken_26|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50283,"special":false,"content":"<|extratoken_27|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50284,"special":false,"content":"<|extratoken_28|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50285,"special":false,"content":"<|extratoken_29|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50286,"special":false,"content":"<|extratoken_30|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50287,"special":false,"content":"<|extratoken_31|>","single_word":false,"lstrip":false,"rstrip":fals