working tokenizer

.ipynb_checkpoints/eval-checkpoint.py

@@ -85,7 +85,7 @@ def main(args):
     dataset = load_dataset(args.dataset, args.config, split=args.split, use_auth_token=True)
 
     # for testing: only process the first two examples as a test
-    dataset = dataset.select(range(2))
+    # dataset = dataset.select(range(2))
 
     # load processor
     feature_extractor = AutoFeatureExtractor.from_pretrained(args.model_id)

eval.py

@@ -85,7 +85,7 @@ def main(args):
     dataset = load_dataset(args.dataset, args.config, split=args.split, use_auth_token=True)
 
     # for testing: only process the first two examples as a test
-    dataset = dataset.select(range(2))
+    # dataset = dataset.select(range(2))
 
     # load processor
     feature_extractor = AutoFeatureExtractor.from_pretrained(args.model_id)

mozilla-foundation_common_voice_8_0_fr_test_eval_results.txt

@@ -1,2 +1,2 @@
-WER: 0.0625
-CER: 0.06382978723404255
+WER: 0.21587470509795875
+CER: 0.06356032070032196