fix fast tokenizer for transformers >= "from_slow": false,
Browse files- tokenizer_config.json +1 -1
 
    	
        tokenizer_config.json
    CHANGED
    
    | 
         @@ -2,6 +2,6 @@ 
     | 
|
| 2 | 
         
             
                "unk_token": "</s>",
         
     | 
| 3 | 
         
             
                "bos_token": "<s>",
         
     | 
| 4 | 
         
             
                "eos_token": "</s>",
         
     | 
| 5 | 
         
            -
                " 
     | 
| 6 | 
         
             
                "tokenizer_class": "LlamaTokenizer"
         
     | 
| 7 | 
         
             
            }
         
     | 
| 
         | 
|
| 2 | 
         
             
                "unk_token": "</s>",
         
     | 
| 3 | 
         
             
                "bos_token": "<s>",
         
     | 
| 4 | 
         
             
                "eos_token": "</s>",
         
     | 
| 5 | 
         
            +
                "from_slow": false,
         
     | 
| 6 | 
         
             
                "tokenizer_class": "LlamaTokenizer"
         
     | 
| 7 | 
         
             
            }
         
     |