Update tokenizer_config.json
Browse filesThis fixes the tokenizer to work with Transformers V5 otherwise you get the incorrect normalization.
See https://github.com/ml-explore/mlx-lm/issues/725 and https://github.com/huggingface/transformers/issues/43122 for relevant discussion.
- tokenizer_config.json +1 -1
tokenizer_config.json
CHANGED
|
@@ -490,6 +490,6 @@
|
|
| 490 |
"clean_up_tokenization_spaces": false,
|
| 491 |
"eos_token": "[e~[",
|
| 492 |
"model_max_length": 40960000,
|
| 493 |
-
"tokenizer_class": "
|
| 494 |
"unk_token": "]!d~["
|
| 495 |
}
|
|
|
|
| 490 |
"clean_up_tokenization_spaces": false,
|
| 491 |
"eos_token": "[e~[",
|
| 492 |
"model_max_length": 40960000,
|
| 493 |
+
"tokenizer_class": "TokenizersBackend",
|
| 494 |
"unk_token": "]!d~["
|
| 495 |
}
|