fix-vocab-size (#17)
Browse files- fix vocab size (15c1aa443e6aa5877a0aef051055cf54ea01c85c)
- tokenization_kimi.py +1 -1
tokenization_kimi.py
CHANGED
|
@@ -104,7 +104,7 @@ class TikTokenTokenizer(PreTrainedTokenizer):
|
|
| 104 |
self.special_tokens = {
|
| 105 |
special_tokens_mapping.get(i, f"<|reserved_token_{i}|>"): i
|
| 106 |
for i in range(
|
| 107 |
-
num_base_tokens, num_base_tokens + self.num_reserved_special_tokens
|
| 108 |
)
|
| 109 |
}
|
| 110 |
|
|
|
|
| 104 |
self.special_tokens = {
|
| 105 |
special_tokens_mapping.get(i, f"<|reserved_token_{i}|>"): i
|
| 106 |
for i in range(
|
| 107 |
+
num_base_tokens, num_base_tokens + self.num_reserved_special_tokens
|
| 108 |
)
|
| 109 |
}
|
| 110 |
|