Hao Xu
commited on
Commit
·
490d0b1
1
Parent(s):
04cf55a
handling datasets with config
Browse files
app.py
CHANGED
|
@@ -177,7 +177,7 @@ def record_submission(benchmark_name, contributor, jsonl_file, hf_path, hf_split
|
|
| 177 |
return "❌ Please provide a dataset split for the HuggingFace dataset."
|
| 178 |
|
| 179 |
try:
|
| 180 |
-
if hf_config
|
| 181 |
dataset_info = load_dataset(hf_path.strip(), hf_config.strip(), split=hf_split.strip(), streaming=True, trust_remote_code=True)
|
| 182 |
else:
|
| 183 |
dataset_info = load_dataset(hf_path.strip(), split=hf_split.strip(), streaming=True, trust_remote_code=True)
|
|
@@ -195,7 +195,8 @@ def record_submission(benchmark_name, contributor, jsonl_file, hf_path, hf_split
|
|
| 195 |
'type': 'jsonl' if has_jsonl else 'hf',
|
| 196 |
'split': hf_split.strip() if has_hf else '',
|
| 197 |
'field_name': field_name.strip(),
|
| 198 |
-
'hf_path': hf_path.strip() if has_hf else ''
|
|
|
|
| 199 |
}
|
| 200 |
print(json.dumps(data))
|
| 201 |
files = {}
|
|
|
|
| 177 |
return "❌ Please provide a dataset split for the HuggingFace dataset."
|
| 178 |
|
| 179 |
try:
|
| 180 |
+
if hf_config:
|
| 181 |
dataset_info = load_dataset(hf_path.strip(), hf_config.strip(), split=hf_split.strip(), streaming=True, trust_remote_code=True)
|
| 182 |
else:
|
| 183 |
dataset_info = load_dataset(hf_path.strip(), split=hf_split.strip(), streaming=True, trust_remote_code=True)
|
|
|
|
| 195 |
'type': 'jsonl' if has_jsonl else 'hf',
|
| 196 |
'split': hf_split.strip() if has_hf else '',
|
| 197 |
'field_name': field_name.strip(),
|
| 198 |
+
'hf_path': hf_path.strip() if has_hf else '',
|
| 199 |
+
'hf_config': hf_config.strip() if has_hf else ''
|
| 200 |
}
|
| 201 |
print(json.dumps(data))
|
| 202 |
files = {}
|