blab-jhu
/

test-32m-dec

modernbert-decoder

Model card Files Files and versions

orionweller commited on Jun 21

Commit

1dbd43f

·

verified ·

1 Parent(s): 4a0297e

Update tokenizer_config.json

Files changed (1) hide show

tokenizer_config.json +1 -1

tokenizer_config.json CHANGED Viewed

@@ -940,7 +940,7 @@
     "eos_token": "[SEP]",
     "bos_token": "[CLS]",
     "is_causal": true,
-    "tokenizer_class": "GPTNeoXTokenizer",
     "model_input_names": [
       "input_ids",
       "attention_mask"

     "eos_token": "[SEP]",
     "bos_token": "[CLS]",
     "is_causal": true,
+    "tokenizer_class": "GPTNeoXTokenizerFast",
     "model_input_names": [
       "input_ids",
       "attention_mask"