bigmoyan commited on
Commit
12d9c7c
·
verified ·
1 Parent(s): 94a4053
Files changed (1) hide show
  1. tokenization_kimi.py +1 -1
tokenization_kimi.py CHANGED
@@ -104,7 +104,7 @@ class TikTokenTokenizer(PreTrainedTokenizer):
104
  self.special_tokens = {
105
  special_tokens_mapping.get(i, f"<|reserved_token_{i}|>"): i
106
  for i in range(
107
- num_base_tokens, num_base_tokens + self.num_reserved_special_tokens + 2
108
  )
109
  }
110
 
 
104
  self.special_tokens = {
105
  special_tokens_mapping.get(i, f"<|reserved_token_{i}|>"): i
106
  for i in range(
107
+ num_base_tokens, num_base_tokens + self.num_reserved_special_tokens
108
  )
109
  }
110