crh_monocorpus-bpe-50_256 / special_tokens_map.json
transhumanist-already-exists's picture
Initial upload
52418c3 verified
{
"additional_special_tokens": [
"<PAD>",
"<UNK>",
"<CLS>",
"<SEP>",
"<MASK_TOKEN>",
"<BOS_TOKEN>",
"<EOS_TOKEN>",
"<EOP_TOKEN>",
"<|START_OF_TURN_TOKEN|>",
"<|END_OF_TURN_TOKEN|>",
"<|YES_TOKEN|>",
"<|NO_TOKEN|>",
"<|GOOD_TOKEN|>",
"<|BAD_TOKEN|>",
"<|USER_TOKEN|>",
"<|CHATBOT_TOKEN|>",
"<|SYSTEM_TOKEN|>",
"<|USER_0_TOKEN|>",
"<|USER_1_TOKEN|>",
"<|USER_2_TOKEN|>",
"<|USER_3_TOKEN|>",
"<|USER_4_TOKEN|>",
"<|USER_5_TOKEN|>",
"<|USER_6_TOKEN|>",
"<|USER_7_TOKEN|>",
"<|USER_8_TOKEN|>",
"<|USER_9_TOKEN|>",
"<|EXTRA_0_TOKEN|>",
"<|EXTRA_1_TOKEN|>",
"<|EXTRA_2_TOKEN|>",
"<|EXTRA_3_TOKEN|>",
"<|EXTRA_4_TOKEN|>",
"<|EXTRA_5_TOKEN|>",
"<|EXTRA_6_TOKEN|>",
"<|EXTRA_7_TOKEN|>",
"<|EXTRA_8_TOKEN|>",
"<|EXTRA_9_TOKEN|>"
],
"bos_token": {
"content": "<BOS_TOKEN>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "<|END_OF_TURN_TOKEN|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"pad_token": {
"content": "<PAD>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
}
}