tokenizer fix

tokenization_codegen25.py

@@ -59,18 +59,18 @@ def tiktoken_tokenizer(base="gpt2", pad_token=None, add_special=True):
         ]
         return fim_tokens
 
-    def include_codegen2_tokens():
+    def include_additional_tokens():
         tokens = []
         tokens += [f"<dummy_{i}>" for i in range(4)]
         tokens.append("<sep>")  # 50317
         tokens.append("<eom>")  # 50318
         tokens += [f"<mask_{i}>" for i in reversed(range(1, 51199-50318+1))]
-        return tokens
+        return tokens        
 
     add_whitespaces = include_whitespace(n_min=2, n_max=32)
     add_tabs = include_tabs(n_min=2, n_max=10)
     fim_tokens = include_fim_tokens()
-    codegen2_tokens = include_codegen2_tokens()
+    additional_tokens = include_additional_tokens()
 
     tokenizer = tiktoken.get_encoding(base)
 
@@ -90,9 +90,9 @@ def tiktoken_tokenizer(base="gpt2", pad_token=None, add_special=True):
     for sp in fim_tokens:
         special_tokens[sp] = idx
         idx += 1
-    for sp in codegen2_tokens:
+    for sp in additional_tokens:
         special_tokens[sp] = idx
-        idx += 1
+        idx += 1        
 
     if pad_token and pad_token not in tokenizer._special_tokens and pad_token not in special_tokens:
         special_tokens[pad_token] = idx
@@ -115,7 +115,7 @@ def tiktoken_tokenizer(base="gpt2", pad_token=None, add_special=True):
 
 class CodeGen25Tokenizer(PreTrainedTokenizer):
     """
-    Construct a CodeGen2.5 tokenizer. Based on byte-level Byte-Pair-Encoding.
+    Construct a CodeGen25 tokenizer. Based on byte-level Byte-Pair-Encoding.
     Args:
         vocab_file (`str`):
             Path to the vocabulary file.
@@ -133,6 +133,8 @@ class CodeGen25Tokenizer(PreTrainedTokenizer):
     ):
         pad_token_added = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token
         eos_token_added = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token
+        self.add_eos_token = add_eos_token
+        self.encoder = tiktoken_tokenizer(base="gpt2", pad_token=pad_token, add_special=add_special_tokens)
         super().__init__(
             pad_token=pad_token_added,
             eos_token=eos_token_added,
@@ -140,8 +142,6 @@ class CodeGen25Tokenizer(PreTrainedTokenizer):
             add_special_tokens=add_special_tokens,
             **kwargs,
         )
-        self.add_eos_token = add_eos_token
-        self.encoder = tiktoken_tokenizer(base="gpt2", pad_token=pad_token, add_special=add_special_tokens)
 
     @property
     def vocab_size(self):
@@ -150,7 +150,7 @@ class CodeGen25Tokenizer(PreTrainedTokenizer):
 
     def get_vocab(self):
         """Returns vocab as a dict"""
-        vocab = {self._convert_id_to_token(i): i for i in range(self.vocab_size)}
+        vocab = {self.encoder.decode_single_token_bytes(i): i for i in range(self.vocab_size)}
         return vocab
 
     def _tokenize(self, text, **kwargs):
@@ -168,9 +168,7 @@ class CodeGen25Tokenizer(PreTrainedTokenizer):
         """Converts an index (integer) in a token (str) using the vocab."""
         return self.encoder.decode_single_token_bytes(index).decode("utf-8")
 
-    def _decode(self, token_ids: Union[int, List[int]], skip_special_tokens: bool = False, **kwargs):
-        if isinstance(token_ids, int):
-            token_ids = [token_ids]
+    def _decode(self, token_ids: List[int], skip_special_tokens: bool = False, **kwargs):
         if skip_special_tokens:
             token_ids = [t for t in token_ids if t not in self.all_special_ids]
         return self.encoder.decode(token_ids)
@@ -244,4 +242,4 @@ class CodeGen25Tokenizer(PreTrainedTokenizer):
 
     # has no vocab file
     def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None):
-        return ()
+        return ()