| stage,source_display_name,stage_i_node_key,upstream_dataset,upstream_revision,upstream_config,recorded_licence_at_pinned_revision,target_serialized_tokens,selected_serialized_tokens,overshoot_tokens,selected_documents,share_of_stage_selected_pct,transport_repository_recorded,transport_revision_recorded | |
| stage_a,FineWeb-Edu (dedup),fineweb_edu_dedup,HuggingFaceTB/smollm-corpus,3ba9d605774198c5868892d7a8deda78031a781f,fineweb-edu-dedup,odc-by-1.0,7110526316,7110526955,639,7350945,71.1052,HuggingFaceTB/smollm-corpus,3ba9d605774198c5868892d7a8deda78031a781f | |
| stage_a,DCLM-Edu,dclm_edu,HuggingFaceTB/dclm-edu,dbad8ad71224482740cd9c9d353591adbf62fe04,default,cc-by-4.0,2031578947,2031579037,90,1593857,20.3158,HuggingFaceTB/dclm-edu,dbad8ad71224482740cd9c9d353591adbf62fe04 | |
| stage_a,Wikipedia (FineWiki EN),finewiki_en,HuggingFaceFW/finewiki,8bd13e72e6a002407649b3e898535f42ceb1aeb9,en,cc-by-sa-4.0,507894737,507896470,1733,557285,5.0790,HuggingFaceFW/finewiki,8bd13e72e6a002407649b3e898535f42ceb1aeb9 | |
| stage_a,Python-Edu,python_gate_c_full,common-pile/stackv2_edu_filtered,c354dbe88469a1153e97c6a63ac50591849654de,default,per-record metadata.license (Software Heritage permissive subset),350000000,350000772,772,487237,3.5000,NOT_RECORDED,NOT_RECORDED | |
| stage_b,FineWeb-Edu (dedup),fineweb_edu_dedup,HuggingFaceTB/smollm-corpus,3ba9d605774198c5868892d7a8deda78031a781f,fineweb-edu-dedup,odc-by-1.0,1203125000,1203125470,470,1199264,40.1041,HuggingFaceTB/smollm-corpus,3ba9d605774198c5868892d7a8deda78031a781f | |
| stage_b,DCLM-Edu,dclm_edu,HuggingFaceTB/dclm-edu,dbad8ad71224482740cd9c9d353591adbf62fe04,default,cc-by-4.0,687500000,687500443,443,538590,22.9166,HuggingFaceTB/dclm-edu,dbad8ad71224482740cd9c9d353591adbf62fe04 | |
| stage_b,structured_tutorial (Cosmopedia v2 + FinePhrase tutorial),structured_tutorial,HuggingFaceTB/smollm-corpus + HuggingFaceFW/finephrase,3ba9d605774198c5868892d7a8deda78031a781f + 78cf4a5ed0099214979c094c963e699c19163838,cosmopedia-v2 + tutorial,odc-by-1.0 (both),343750000,343750175,175,530450,11.4583,HuggingFaceTB/smollm-corpus + HuggingFaceFW/finephrase,3ba9d605774198c5868892d7a8deda78031a781f + 78cf4a5ed0099214979c094c963e699c19163838 | |
| stage_b,Python-Edu,python_gate_c_full,common-pile/stackv2_edu_filtered,c354dbe88469a1153e97c6a63ac50591849654de,default,per-record metadata.license (Software Heritage permissive subset),250000000,250000383,383,347145,8.3333,NOT_RECORDED,NOT_RECORDED | |
| stage_b,Wikipedia (FineWiki EN),finewiki_en,HuggingFaceFW/finewiki,8bd13e72e6a002407649b3e898535f42ceb1aeb9,en,cc-by-sa-4.0,171875000,171877052,2052,189465,5.7292,HuggingFaceFW/finewiki,8bd13e72e6a002407649b3e898535f42ceb1aeb9 | |
| stage_b,PES2O,pes2o,allenai/dolmino-mix-1124,a319f19eef1e257417b11ea8c30da266ae175557,pes2o,odc-by-1.0,171875000,171875364,364,625468,5.7292,allenai/dolmino-mix-1124,c58ab4b6ff990115e1ff3121953754ee2bc29501 | |
| stage_b,StackExchange,stackexchange,allenai/dolmino-mix-1124,a319f19eef1e257417b11ea8c30da266ae175557,stackexchange,cc-by-sa,171875000,171875353,353,336025,5.7292,allenai/dolmino-mix-1124,c58ab4b6ff990115e1ff3121953754ee2bc29501 | |
| stage_a,TOTAL,,,,,,10000000000,10000003234,,9989324,100.0000,, | |
| stage_b,TOTAL,,,,,,3000000000,3000004240,,3766407,100.0000,, | |
| both,GRAND TOTAL,,,,,,13000000000,13000007474,,13755731,,, | |