maint: make more nice looking

data/leaderboard-classification.csv.zip

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d9e301dfeeb8cc7092301268aad4e4c9922517b6288101a588b5a15f5a0aaca9
-size 4679

data/leaderboard-regression.csv.zip

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d9e301dfeeb8cc7092301268aad4e4c9922517b6288101a588b5a15f5a0aaca9
-size 4679

data/{leaderboard-all.csv.zip → tabarena_leaderboard.csv.zip}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d9e301dfeeb8cc7092301268aad4e4c9922517b6288101a588b5a15f5a0aaca9
-size 4679
+oid sha256:c95306a347a69e561a82562c1a6306f5a9f6819a60f458d5e350639c35cde848
+size 10202

main.py

@@ -67,7 +67,7 @@ def rename_map(model_name: str) -> str:
         "MNCA": "ModernNCA",
         "NN_TORCH": "TorchMLP",
         "FASTAI": "FastaiMLP",
-        "TABPFN": "TabPFNv2",
+        "TABPFNV2": "TabPFNv2",
         "EBM": "EBM",
         "TABDPT": "TabDPT",
         "TABICL": "TabICL",
@@ -88,8 +88,6 @@ def load_data(filename: str):
         f"Loaded dataframe with {len(df_leaderboard)} rows and columns {df_leaderboard.columns}"
     )
 
-    # sort by ELO
-    df_leaderboard = df_leaderboard.sort_values(by="elo", ascending=False)
 
     # add model family information
 
@@ -101,22 +99,51 @@ def load_data(filename: str):
     )
     df_leaderboard["method"] = df_leaderboard["method"].apply(rename_map)
 
+    # elo,elo+,elo-,mrr
+    df_leaderboard["Elo 95% CI"] = (
+        "+"
+        + df_leaderboard["elo+"].round(0).astype(int).astype(str)
+        + "/-"
+        + df_leaderboard["elo-"].round(0).astype(int).astype(str)
+    )
     # select only the columns we want to display
     df_leaderboard = df_leaderboard.loc[
-        :, ["Type", "TypeName", "method", "elo", "rank", "time_train_s", "time_infer_s"]
+        :,
+        [
+            "Type",
+            "TypeName",
+            "method",
+            "elo",
+            "Elo 95% CI",
+            "rank",
+            "normalized-error",
+            "median_time_train_s_per_1K",
+            "median_time_infer_s_per_1K",
+        ],
     ]
 
     # round for better display
-    df_leaderboard = df_leaderboard.round(1)
+    df_leaderboard[["elo", "Elo 95% CI"]] = df_leaderboard[["elo", "Elo 95% CI"]].round(0)
+    df_leaderboard[["median_time_train_s_per_1K", "rank"]] = df_leaderboard[
+        ["median_time_train_s_per_1K", "rank"]
+    ].round(2)
+    df_leaderboard[["normalized-error", "median_time_infer_s_per_1K"]] = df_leaderboard[
+        ["normalized-error", "median_time_infer_s_per_1K"]
+    ].round(3)
+
+    df_leaderboard = df_leaderboard.sort_values(by="elo", ascending=False)
+    df_leaderboard = df_leaderboard.reset_index(drop=True)
+    df_leaderboard = df_leaderboard.reset_index(names="#")
 
     # rename some columns
     return df_leaderboard.rename(
         columns={
-            "time_train_s": "training time (s) [⬇️]",
-            "time_infer_s": "inference time (s) [⬇️]",
+            "median_time_train_s_per_1K": "Median Train Time (s/1K) [⬇️]",
+            "median_time_infer_s_per_1K": "Median Predict Time (s/1K)) [⬇️]",
             "method": "Model",
             "elo": "Elo [⬆️]",
             "rank": "Rank [⬇️]",
+            "normalized-error": "Normalized Error [⬇️]",
         }
     )
 
@@ -138,6 +165,26 @@ def make_leaderboard(df_leaderboard: pd.DataFrame) -> Leaderboard:
         "(tuned + ensemble)"
     ) | df_leaderboard["Model"].str.endswith("(4h)")
 
+    # Add Imputed count postfix
+    mask = df_leaderboard["Model"].str.startswith("TabPFNv2")
+    df_leaderboard.loc[mask, "Model"] = (
+        df_leaderboard.loc[mask, "Model"] + " [35.29% IMPUTED]"
+    )
+    mask = df_leaderboard["Model"].str.startswith("TabICL")
+    df_leaderboard.loc[mask, "Model"] = (
+        df_leaderboard.loc[mask, "Model"] + " [29.41% IMPUTED]"
+    )
+
+    df_leaderboard["Imputed"] = df_leaderboard["Model"].str.startswith(
+        "TabPFNv2"
+    ) | df_leaderboard["Model"].str.startswith("TabICL")
+    df_leaderboard["Imputed"] = df_leaderboard["Imputed"].replace(
+        {
+            True: "Imputed",
+            False: "Not Imputed",
+        }
+    )
+
     return Leaderboard(
         value=df_leaderboard,
         select_columns=SelectColumns(
@@ -152,17 +199,29 @@ def make_leaderboard(df_leaderboard: pd.DataFrame) -> Leaderboard:
             "Only Default",
             "Only Tuned",
             "Only Tuned + Ensemble",
+            "Imputed",
         ],
         search_columns=["Model", "Type"],
         filter_columns=[
-            ColumnFilter(
-                "TypeFiler", type="checkboxgroup", label="Filter by Model Type"
-            ),
+            ColumnFilter("TypeFiler", type="checkboxgroup", label="Model Types."),
             ColumnFilter("Only Default", type="boolean", default=False),
             ColumnFilter("Only Tuned", type="boolean", default=False),
             ColumnFilter("Only Tuned + Ensemble", type="boolean", default=False),
+            ColumnFilter(
+                "Imputed",
+                type="checkboxgroup",
+                label="(Not) Imputed Models.",
+                info="We impute the performance for models that cannot run on all"
+                " datasets due to task or dataset size constraints (e.g. TabPFN,"
+                " TabICL). We impute with the performance of a defaultRandomForest. "
+                " We add a postfix [X% IMPUTED] to the model if any results were "
+                "imputed. The X% shows the percentage of"
+                " datasets that were imputed. In general, imputation negatively"
+                " represents the model performance, punishing the model for not"
+                " being able to run on all datasets.",
+            ),
         ],
-        bool_checkboxgroup_label="Custom Views (Exclusive, only toggle one at a time):",
+        bool_checkboxgroup_label="Custom Views (exclusive, only toggle one at a time):",
     )
 
 
@@ -174,7 +233,7 @@ def main():
 
         with gr.Tabs(elem_classes="tab-buttons"):
             with gr.TabItem("🏅 Overall", elem_id="llm-benchmark-tab-table", id=2):
-                df_leaderboard = load_data("leaderboard-all")
+                df_leaderboard = load_data("tabarena_leaderboard")
                 make_leaderboard(df_leaderboard)
 
             # TODO: decide on which subsets we want to support here.