| from __future__ import annotations |
|
|
| import base64 |
| import json |
| import os |
| import re |
| from fnmatch import fnmatch |
| from io import BytesIO |
| from pathlib import Path |
| from urllib.parse import quote |
|
|
| import gradio as gr |
| import pandas as pd |
| from huggingface_hub import HfApi, hf_hub_download |
|
|
| from theme import POVISLE_THEME |
| from views import ( |
| render_examples_tab, |
| render_leaderboard_category_tab, |
| render_leaderboard_filters, |
| render_leaderboard_task_tab, |
| ) |
|
|
| EXCLUDED_MODELS = {"LLaVA-v1.6-Vicuna-13B-HF"} |
| MODEL_TYPE_LABELS = { |
| "open": "Open-weight", |
| "proprietary": "Proprietary", |
| "random": "Random", |
| } |
| APP_CSS = """ |
| :root { |
| --povisle-bg: #f8f7f6; |
| --povisle-surface: #ffffff; |
| --povisle-soft: #efedeb; |
| --povisle-soft-red: #fff1f2; |
| --povisle-soft-blue: #edf3f9; |
| --povisle-ink: #181416; |
| --povisle-muted: #5f585b; |
| --povisle-faint: #8b8588; |
| --povisle-line: #e3dfdc; |
| --povisle-accent: #d7263d; |
| --povisle-blue: #2c5f8a; |
| --povisle-green: #2a7a4e; |
| --povisle-shadow: 0 2px 16px rgba(39, 33, 31, 0.07); |
| --povisle-shadow-lg: 0 8px 40px rgba(39, 33, 31, 0.11); |
| --povisle-serif: "Source Serif 4", Georgia, serif; |
| --povisle-sans: "DM Sans", system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; |
| --povisle-mono: "JetBrains Mono", ui-monospace, SFMono-Regular, Menlo, monospace; |
| } |
| body { |
| color-scheme: light; |
| background: var(--povisle-bg) !important; |
| color: var(--povisle-ink) !important; |
| font-family: var(--povisle-sans) !important; |
| line-height: 1.7; |
| } |
| .gradio-container { |
| max-width: 1480px !important; |
| margin: 0 auto !important; |
| padding: 0 14px 42px !important; |
| background: transparent !important; |
| color: var(--povisle-ink) !important; |
| font-family: var(--povisle-sans) !important; |
| } |
| .contain { |
| gap: 0 !important; |
| } |
| footer { |
| display: none !important; |
| } |
| .povisle-hero { |
| background: transparent; |
| border: 0; |
| border-radius: 0; |
| box-shadow: none; |
| margin: 0 auto 34px; |
| max-width: 1120px; |
| padding: 84px 28px 48px; |
| text-align: center; |
| } |
| .povisle-kicker { |
| align-items: center; |
| background: var(--povisle-soft-red); |
| border: 1px solid rgba(215, 38, 61, 0.22); |
| border-radius: 20px; |
| color: var(--povisle-accent); |
| display: inline-flex; |
| font-size: 0.78rem; |
| font-weight: 700; |
| gap: 6px; |
| letter-spacing: 0.04em; |
| line-height: 1; |
| margin: 0 0 26px; |
| padding: 7px 14px; |
| text-transform: uppercase; |
| } |
| .povisle-hero h1 { |
| color: var(--povisle-ink) !important; |
| display: block !important; |
| font-family: var(--povisle-serif) !important; |
| font-size: clamp(2.65rem, 4.5vw, 4.35rem) !important; |
| font-weight: 600 !important; |
| letter-spacing: 0 !important; |
| line-height: 1.02 !important; |
| margin: 0 auto 12px !important; |
| max-width: 1200px !important; |
| } |
| .povisle-title-accent { |
| color: var(--povisle-accent) !important; |
| } |
| .povisle-hero em { |
| color: var(--povisle-ink); |
| font-style: italic; |
| } |
| .povisle-hero p { |
| color: var(--povisle-muted); |
| font-size: 0.98rem; |
| line-height: 1.8; |
| margin: 26px auto 0; |
| max-width: 720px; |
| } |
| .povisle-authors { |
| color: var(--povisle-muted); |
| font-size: 0.9rem; |
| line-height: 1.65; |
| margin: 24px auto 0; |
| max-width: 760px; |
| } |
| .povisle-authors strong { |
| color: var(--povisle-ink); |
| font-weight: 500; |
| } |
| .povisle-links-row { |
| display: flex; |
| flex-wrap: wrap; |
| gap: 10px; |
| justify-content: center; |
| margin-top: 30px; |
| } |
| .povisle-hero a.povisle-btn, |
| .povisle-hero a.povisle-btn:hover, |
| .povisle-hero a.povisle-btn:focus, |
| .povisle-hero a.povisle-btn:visited { |
| text-decoration: none !important; |
| } |
| .povisle-btn { |
| align-items: center; |
| border: 1px solid transparent; |
| border-radius: 6px; |
| display: inline-flex; |
| font-size: 0.84rem; |
| font-weight: 500; |
| gap: 7px; |
| line-height: 1; |
| padding: 10px 18px; |
| text-decoration: none; |
| transition: all 0.18s ease; |
| } |
| .povisle-btn svg { |
| flex-shrink: 0; |
| height: 15px; |
| width: 15px; |
| } |
| .povisle-btn svg.arxiv-icon { |
| height: 22.5px; |
| width: 22.5px; |
| } |
| .povisle-btn-primary { |
| background: var(--povisle-accent); |
| color: #ffffff !important; |
| } |
| .povisle-btn-primary:hover { |
| background: #b91c33; |
| box-shadow: var(--povisle-shadow); |
| transform: translateY(-1px); |
| } |
| .povisle-btn-dataset { |
| background: var(--povisle-soft-blue); |
| border-color: rgba(44, 95, 138, 0.2); |
| color: var(--povisle-blue) !important; |
| } |
| .povisle-hero a.povisle-btn-dataset, |
| .povisle-hero a.povisle-btn-dataset:visited { |
| color: var(--povisle-blue) !important; |
| } |
| .povisle-btn-dataset:hover { |
| background: #dbe9f5; |
| transform: translateY(-1px); |
| } |
| .povisle-btn-outline { |
| background: var(--povisle-surface); |
| border-color: var(--povisle-line); |
| color: var(--povisle-ink) !important; |
| } |
| .povisle-hero a.povisle-btn-outline, |
| .povisle-hero a.povisle-btn-outline:visited { |
| color: var(--povisle-ink) !important; |
| } |
| .povisle-btn-outline:hover { |
| border-color: var(--povisle-ink); |
| box-shadow: var(--povisle-shadow); |
| transform: translateY(-1px); |
| } |
| .povisle-stat-row { |
| display: grid; |
| gap: 16px; |
| grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); |
| margin: 30px auto 0; |
| max-width: 760px; |
| } |
| .povisle-stat { |
| background: var(--povisle-bg); |
| border: 1px solid var(--povisle-line); |
| border-radius: 10px; |
| padding: 18px 14px; |
| text-align: center; |
| } |
| .povisle-stat strong { |
| color: var(--povisle-accent) !important; |
| display: block; |
| font-family: var(--povisle-serif); |
| font-size: 1.7rem; |
| font-weight: 600; |
| line-height: 1.1; |
| } |
| .povisle-stat span { |
| color: var(--povisle-faint); |
| display: block; |
| font-size: 0.78rem; |
| font-weight: 500; |
| margin-top: 5px; |
| } |
| .tab-nav { |
| justify-content: center !important; |
| margin: 0 auto !important; |
| max-width: 860px !important; |
| } |
| .tab-nav button { |
| color: var(--povisle-muted) !important; |
| font-family: var(--povisle-sans) !important; |
| font-size: 0.88rem !important; |
| font-weight: 600 !important; |
| } |
| .tabitem { |
| padding-top: 24px !important; |
| } |
| #leaderboard-section { |
| padding-top: 0 !important; |
| } |
| #leaderboard-search { |
| margin-bottom: 14px !important; |
| } |
| #leaderboard-filter-row { |
| margin-bottom: 14px !important; |
| } |
| #category-view-select { |
| margin-bottom: 14px !important; |
| } |
| .leaderboard-split-tabs { |
| margin-top: 0 !important; |
| } |
| #leaderboard-table, |
| .leaderboard-table { |
| margin-top: 0 !important; |
| } |
| .form, |
| .block { |
| border-color: var(--povisle-line) !important; |
| border-radius: 8px !important; |
| box-shadow: none !important; |
| } |
| .wrap.default { |
| background: transparent !important; |
| } |
| #leaderboard-filter-row { |
| background: var(--povisle-surface) !important; |
| border: 1px solid var(--povisle-line) !important; |
| border-radius: 8px !important; |
| gap: 0 !important; |
| overflow: hidden !important; |
| } |
| #leaderboard-filter-row > *, |
| #split-filter, |
| #type-filter, |
| #size-filter, |
| #family-filter { |
| background: var(--povisle-surface) !important; |
| border-color: transparent !important; |
| border-radius: 0 !important; |
| box-shadow: none !important; |
| } |
| #split-filter { |
| border-radius: 8px 0 0 8px !important; |
| } |
| #type-filter { |
| border-left: 0 !important; |
| border-radius: 8px 0 0 8px !important; |
| } |
| #family-filter { |
| border-radius: 0 8px 8px 0 !important; |
| } |
| #type-filter, |
| #size-filter, |
| #family-filter { |
| border-left: 1px dashed #ddd7d3 !important; |
| } |
| #type-filter { |
| border-left: 0 !important; |
| } |
| #leaderboard-filter-row .form, |
| #leaderboard-filter-row .block { |
| background: transparent !important; |
| border: 0 !important; |
| border-radius: 0 !important; |
| } |
| #leaderboard-filter-row [data-testid="token"], |
| #leaderboard-filter-row .token, |
| #leaderboard-filter-row .selected-token, |
| #leaderboard-filter-row .multiselect-token, |
| #leaderboard-filter-row div:has(> button[aria-label^="Remove"]), |
| #leaderboard-filter-row div:has(> button[aria-label^="remove"]), |
| #leaderboard-filter-row button[aria-label^="Remove"], |
| #leaderboard-filter-row button[aria-label^="remove"] { |
| align-items: center !important; |
| background: #efedeb !important; |
| border: 1px solid #ddd7d3 !important; |
| border-radius: 999px !important; |
| color: var(--povisle-ink) !important; |
| display: inline-flex !important; |
| font-size: 0.82rem !important; |
| font-weight: 500 !important; |
| gap: 6px !important; |
| line-height: 1 !important; |
| min-height: 28px !important; |
| padding: 5px 10px !important; |
| } |
| #leaderboard-filter-row [data-testid="token"] button, |
| #leaderboard-filter-row .token button, |
| #leaderboard-filter-row .selected-token button, |
| #leaderboard-filter-row .multiselect-token button { |
| background: transparent !important; |
| border: 0 !important; |
| color: var(--povisle-muted) !important; |
| min-height: auto !important; |
| padding: 0 0 0 3px !important; |
| } |
| #leaderboard-filter-row [data-testid="token"] svg, |
| #leaderboard-filter-row .token svg, |
| #leaderboard-filter-row .selected-token svg, |
| #leaderboard-filter-row .multiselect-token svg { |
| height: 12px !important; |
| width: 12px !important; |
| } |
| #leaderboard-search, |
| #category-view-select, |
| #leaderboard-table, |
| .leaderboard-table, |
| #examples-table { |
| border-radius: 10px !important; |
| } |
| #leaderboard-table, |
| .leaderboard-table, |
| #examples-table { |
| box-shadow: var(--povisle-shadow); |
| } |
| #leaderboard-table table th, |
| .leaderboard-table table th, |
| #examples-table table th { |
| background: var(--povisle-soft) !important; |
| color: var(--povisle-ink) !important; |
| font-size: 0.7rem !important; |
| font-weight: 700 !important; |
| line-height: 1.2 !important; |
| padding: 8px 10px !important; |
| white-space: normal !important; |
| } |
| #leaderboard-table table td, |
| .leaderboard-table table td, |
| #examples-table table td { |
| border-color: var(--povisle-line) !important; |
| color: var(--povisle-muted) !important; |
| } |
| #leaderboard-table table th, |
| .leaderboard-table table th, |
| #leaderboard-table table td, |
| .leaderboard-table table td { |
| min-width: 8.75rem; |
| } |
| #leaderboard-table table th:nth-child(1), |
| .leaderboard-table table th:nth-child(1), |
| #leaderboard-table table td:nth-child(1), |
| .leaderboard-table table td:nth-child(1) { |
| min-width: 7rem; |
| } |
| #leaderboard-table table th:nth-child(2), |
| .leaderboard-table table th:nth-child(2), |
| #leaderboard-table table td:nth-child(2), |
| .leaderboard-table table td:nth-child(2) { |
| min-width: 15rem; |
| } |
| #leaderboard-table table th:nth-child(3), |
| .leaderboard-table table th:nth-child(3), |
| #leaderboard-table table td:nth-child(3), |
| .leaderboard-table table td:nth-child(3) { |
| min-width: 7rem; |
| } |
| #leaderboard-table table th:nth-child(4), |
| .leaderboard-table table th:nth-child(4), |
| #leaderboard-table table td:nth-child(4), |
| .leaderboard-table table td:nth-child(4) { |
| min-width: 8rem; |
| } |
| #leaderboard-table table tbody tr:nth-child(even) td, |
| .leaderboard-table table tbody tr:nth-child(even) td, |
| #examples-table table tbody tr:nth-child(even) td { |
| background: #fbfaf9; |
| } |
| #leaderboard-table table tbody tr:hover td, |
| .leaderboard-table table tbody tr:hover td, |
| #examples-table table tbody tr:hover td { |
| background: #fff1f2 !important; |
| } |
| .povisle-section-label { |
| color: var(--povisle-accent) !important; |
| font-size: 0.72rem; |
| font-weight: 700; |
| letter-spacing: 0.1em; |
| margin-bottom: 12px; |
| text-transform: uppercase; |
| } |
| .povisle-dataset-card .povisle-section-label, |
| .povisle-leaderboard-heading .povisle-section-label, |
| .povisle-examples-heading .povisle-section-label { |
| color: var(--povisle-accent) !important; |
| } |
| .povisle-section-title { |
| color: var(--povisle-ink); |
| font-family: var(--povisle-serif); |
| font-size: clamp(1.3rem, 2.5vw, 1.7rem); |
| font-weight: 600; |
| margin: 0 0 18px; |
| } |
| .povisle-dataset-card { |
| background: transparent; |
| border-color: transparent; |
| border-radius: 0; |
| box-shadow: none; |
| margin: 30px auto 8px; |
| max-width: 1440px; |
| padding: 12px 0 4px; |
| } |
| .povisle-dataset-card p { |
| color: var(--povisle-muted); |
| font-size: 0.97rem; |
| line-height: 1.8; |
| margin: 0 0 14px; |
| max-width: 100%; |
| } |
| .povisle-dataset-card p:last-child { |
| margin-bottom: 0; |
| } |
| .povisle-about-grid { |
| align-items: start; |
| display: grid; |
| gap: 28px; |
| grid-template-columns: 1fr; |
| margin-top: 26px; |
| } |
| .povisle-about-copy { |
| margin-top: 22px; |
| } |
| .povisle-about-grid .povisle-about-copy { |
| margin-top: 0; |
| } |
| .povisle-category-layout { |
| align-items: start; |
| display: grid; |
| gap: 24px; |
| grid-template-columns: repeat(2, minmax(0, 1fr)); |
| } |
| .povisle-task-layout { |
| align-items: start; |
| display: grid; |
| gap: 24px; |
| grid-template-columns: repeat(2, minmax(0, 1fr)); |
| } |
| .povisle-creation-layout { |
| align-items: start; |
| display: grid; |
| gap: 24px; |
| grid-template-columns: repeat(2, minmax(0, 1fr)); |
| } |
| .povisle-creation-figure { |
| background: var(--povisle-surface); |
| border: 1px solid var(--povisle-line); |
| border-radius: 8px; |
| margin: 0; |
| overflow: hidden; |
| padding: 12px; |
| } |
| .povisle-creation-figure img { |
| display: block; |
| height: auto; |
| width: 100%; |
| } |
| .povisle-category-intro { |
| margin-bottom: 8px !important; |
| } |
| .povisle-about-copy h3 { |
| color: var(--povisle-ink); |
| font-family: var(--povisle-serif); |
| font-size: 1.18rem; |
| font-weight: 600; |
| line-height: 1.25; |
| margin: 0 0 10px; |
| } |
| .povisle-angled-sunburst { |
| background: var(--povisle-surface) !important; |
| border: 1px solid var(--povisle-line) !important; |
| border-radius: 8px !important; |
| box-shadow: none !important; |
| margin: 0 !important; |
| overflow: hidden !important; |
| padding: 14px !important; |
| } |
| .povisle-angled-sunburst img { |
| display: block; |
| height: clamp(260px, 36vw, 520px); |
| margin: 0 auto; |
| max-width: 720px; |
| object-fit: contain; |
| width: 100%; |
| } |
| .povisle-examples-heading { |
| margin: 38px auto 18px; |
| max-width: 1440px; |
| } |
| .povisle-leaderboard-heading { |
| margin: 26px auto 18px; |
| max-width: 1440px; |
| } |
| .povisle-leaderboard-heading .povisle-section-title { |
| margin-bottom: 10px; |
| } |
| .povisle-leaderboard-heading p { |
| color: var(--povisle-muted); |
| font-size: 0.97rem; |
| line-height: 1.7; |
| margin: 0; |
| max-width: 100%; |
| } |
| .povisle-examples-heading p { |
| color: var(--povisle-muted); |
| font-size: 0.97rem; |
| line-height: 1.8; |
| margin: 0; |
| } |
| .povisle-acknowledgement { |
| border-top: 1px solid var(--povisle-line); |
| margin: 46px auto 0; |
| max-width: 1440px; |
| padding: 24px 0 0; |
| } |
| .povisle-acknowledgement .povisle-section-title { |
| font-size: 1.15rem; |
| margin-bottom: 8px; |
| } |
| .povisle-acknowledgement p { |
| color: var(--povisle-muted); |
| font-size: 0.94rem; |
| line-height: 1.7; |
| margin: 0; |
| max-width: 100%; |
| } |
| .povisle-citation { |
| border-top: 1px solid var(--povisle-line); |
| margin: 30px auto 0; |
| max-width: 1440px; |
| padding: 24px 0 0; |
| } |
| .povisle-citation .povisle-section-title { |
| font-size: 1.15rem; |
| margin-bottom: 8px; |
| } |
| .povisle-citation pre { |
| background: var(--povisle-surface); |
| border: 1px solid var(--povisle-line); |
| border-radius: 8px; |
| color: var(--povisle-ink); |
| font-family: var(--povisle-mono); |
| font-size: 0.82rem; |
| line-height: 1.55; |
| margin: 0; |
| overflow-x: auto; |
| padding: 14px; |
| white-space: pre; |
| } |
| #examples-filter-row { |
| background: var(--povisle-surface) !important; |
| border: 1px solid var(--povisle-line) !important; |
| border-radius: 10px !important; |
| gap: 0 !important; |
| margin: 0 0 16px !important; |
| overflow: hidden !important; |
| padding: 0 !important; |
| width: 100% !important; |
| } |
| #examples-filter-row > * { |
| background: transparent !important; |
| position: relative !important; |
| } |
| #examples-filter-row > * + *::before { |
| background: #ddd7d3; |
| bottom: 14px; |
| content: ""; |
| left: 0; |
| position: absolute; |
| top: 14px; |
| width: 1px; |
| z-index: 1; |
| } |
| #examples-filter-row .block, |
| #examples-filter-row .form, |
| #examples-filter-row .wrap, |
| #examples-filter-row .wrap-inner, |
| #examples-filter-row .input-container { |
| background: transparent !important; |
| border: 0 !important; |
| border-radius: 0 !important; |
| box-shadow: none !important; |
| } |
| #examples-filter-row input, |
| #examples-filter-row textarea { |
| background: transparent !important; |
| } |
| #examples-table { |
| overflow-x: auto !important; |
| } |
| #examples-table table { |
| table-layout: fixed !important; |
| min-width: 1530px !important; |
| } |
| #examples-table table th, |
| #examples-table table td, |
| #examples-table [role="columnheader"], |
| #examples-table [role="gridcell"] { |
| line-height: 1.35 !important; |
| padding: 6px 8px !important; |
| } |
| #examples-table table td, |
| #examples-table [role="gridcell"] { |
| font-size: 0.64rem !important; |
| line-height: 1.3 !important; |
| } |
| #examples-table table th, |
| #examples-table [role="columnheader"] { |
| font-size: 0.66rem !important; |
| line-height: 1.15 !important; |
| padding: 7px 8px !important; |
| } |
| #examples-table table td *, |
| #examples-table table th *, |
| #examples-table [role="gridcell"] *, |
| #examples-table [role="columnheader"] * { |
| font-size: inherit !important; |
| line-height: inherit !important; |
| } |
| #examples-table table td, |
| #examples-table table th, |
| #examples-table [role="gridcell"], |
| #examples-table [role="columnheader"] { |
| min-width: 0 !important; |
| overflow-wrap: normal !important; |
| white-space: normal !important; |
| word-break: normal !important; |
| } |
| #examples-table table td *, |
| #examples-table table th *, |
| #examples-table [role="gridcell"] *, |
| #examples-table [role="columnheader"] * { |
| overflow-wrap: break-word !important; |
| white-space: normal !important; |
| word-break: normal !important; |
| } |
| #examples-table table td:nth-child(1), |
| #examples-table table th:nth-child(1) { |
| width: 90px !important; |
| } |
| #examples-table table td:nth-child(2), |
| #examples-table table th:nth-child(2) { |
| width: 128px !important; |
| } |
| #examples-table table td:nth-child(3), |
| #examples-table table th:nth-child(3) { |
| width: 130px !important; |
| } |
| #examples-table table td:nth-child(4), |
| #examples-table table th:nth-child(4) { |
| width: 110px !important; |
| text-align: center; |
| vertical-align: middle; |
| } |
| #examples-table table td:nth-child(5), |
| #examples-table table th:nth-child(5) { |
| width: 300px !important; |
| } |
| #examples-table table td:nth-child(6), |
| #examples-table table th:nth-child(6) { |
| width: 230px !important; |
| } |
| #examples-table table td:nth-child(6), |
| #examples-table [role="gridcell"]:nth-child(6) { |
| white-space: pre-line !important; |
| } |
| #examples-table table td:nth-child(7), |
| #examples-table table th:nth-child(7) { |
| width: 170px !important; |
| } |
| #examples-table table td:nth-child(8), |
| #examples-table table th:nth-child(8) { |
| width: 190px !important; |
| } |
| #examples-table table td:nth-child(9), |
| #examples-table table th:nth-child(9) { |
| width: 90px !important; |
| } |
| #examples-table table td:nth-child(10), |
| #examples-table table th:nth-child(10) { |
| width: 100px !important; |
| } |
| #examples-table table td:nth-child(4) img.example-image { |
| display: block; |
| margin: 0 auto; |
| max-height: 6rem; |
| max-width: 6rem; |
| object-fit: contain; |
| } |
| #leaderboard-section, |
| #examples-section { |
| background: transparent !important; |
| border: 0 !important; |
| box-shadow: none !important; |
| } |
| .povisle-category-list { |
| color: var(--povisle-muted); |
| font-size: 0.88rem; |
| line-height: 1.45; |
| margin: 4px 0 0; |
| padding-left: 1.1rem; |
| } |
| .povisle-category-list li + li { |
| margin-top: 8px; |
| } |
| .povisle-category-list strong { |
| color: var(--povisle-ink); |
| font-weight: 720; |
| } |
| .povisle-category-note { |
| margin-top: 18px !important; |
| } |
| .povisle-task-list { |
| color: var(--povisle-muted); |
| font-size: 0.9rem; |
| line-height: 1.55; |
| margin: 0; |
| padding-left: 1.1rem; |
| } |
| .povisle-task-list li + li { |
| margin-top: 8px; |
| } |
| .povisle-task-list strong { |
| color: var(--povisle-ink); |
| font-weight: 720; |
| } |
| .povisle-task-chart { |
| background: var(--povisle-surface); |
| border: 1px solid var(--povisle-line); |
| border-radius: 8px; |
| padding: 16px; |
| } |
| .povisle-task-chart-legend { |
| display: flex; |
| flex-wrap: wrap; |
| gap: 10px 14px; |
| margin-bottom: 16px; |
| } |
| .povisle-task-chart-legend span { |
| align-items: center; |
| color: var(--povisle-muted); |
| display: inline-flex; |
| font-size: 0.78rem; |
| gap: 6px; |
| line-height: 1.2; |
| } |
| .povisle-task-chart-legend span::before { |
| background: var(--segment-color); |
| border-radius: 999px; |
| content: ""; |
| display: inline-block; |
| height: 8px; |
| width: 8px; |
| } |
| .povisle-task-split + .povisle-task-split { |
| margin-top: 18px; |
| } |
| .povisle-task-split-header { |
| align-items: baseline; |
| display: flex; |
| justify-content: space-between; |
| gap: 12px; |
| margin-bottom: 8px; |
| } |
| .povisle-task-split-header span { |
| color: var(--povisle-muted); |
| font-size: 0.82rem; |
| line-height: 1.25; |
| } |
| .povisle-task-split-header strong { |
| color: var(--povisle-ink); |
| font-family: var(--povisle-mono); |
| font-size: 0.82rem; |
| font-weight: 650; |
| text-align: right; |
| } |
| .povisle-task-chart-track { |
| background: var(--povisle-soft); |
| border-radius: 999px; |
| display: flex; |
| height: 24px; |
| overflow: hidden; |
| } |
| .povisle-task-chart-segment { |
| align-items: center; |
| background: var(--segment-color); |
| color: #ffffff; |
| display: flex; |
| font-family: var(--povisle-mono); |
| font-size: 0.72rem; |
| font-weight: 650; |
| height: 100%; |
| justify-content: center; |
| line-height: 1; |
| min-width: 28px; |
| } |
| .povisle-task-chart-segment + .povisle-task-chart-segment { |
| border-left: 1px solid rgba(255, 255, 255, 0.75); |
| } |
| .block label span, |
| .block .label-wrap span { |
| color: #3c4149 !important; |
| font-weight: 680 !important; |
| } |
| @media (max-width: 760px) { |
| .gradio-container { |
| padding: 0 14px 32px !important; |
| } |
| .povisle-hero { |
| padding: 48px 18px 38px; |
| } |
| .povisle-hero h1 { |
| font-size: clamp(1.95rem, 9.5vw, 2.75rem) !important; |
| } |
| .povisle-links-row { |
| align-items: stretch; |
| flex-direction: column; |
| } |
| .povisle-btn { |
| justify-content: center; |
| } |
| .povisle-dataset-card { |
| padding: 8px 0 4px; |
| } |
| .povisle-about-grid { |
| grid-template-columns: 1fr; |
| } |
| .povisle-category-layout { |
| grid-template-columns: 1fr; |
| } |
| .povisle-task-layout { |
| grid-template-columns: 1fr; |
| } |
| .povisle-creation-layout { |
| grid-template-columns: 1fr; |
| } |
| } |
| |
| /* Leaderboard layout reset after adding nested split tabs. */ |
| #leaderboard-section, |
| #leaderboard-section > *, |
| #leaderboard-section .tabitem, |
| #leaderboard-view-tabs, |
| #leaderboard-view-tabs > *, |
| #leaderboard-view-tabs .tabitem, |
| .leaderboard-split-tabs, |
| .leaderboard-split-tabs > *, |
| .leaderboard-split-tabs .tabitem { |
| background: transparent !important; |
| } |
| #leaderboard-section { |
| display: block !important; |
| background: var(--povisle-bg) !important; |
| margin-top: 0 !important; |
| padding-top: 0 !important; |
| } |
| .tabitem:has(#leaderboard-section) { |
| padding-top: 10px !important; |
| } |
| #leaderboard-section .form, |
| #leaderboard-section .block { |
| box-shadow: none !important; |
| } |
| #leaderboard-search-row, |
| #leaderboard-filter-row, |
| #leaderboard-category-row { |
| background: var(--povisle-surface) !important; |
| border: 1px solid var(--povisle-line) !important; |
| border-radius: 8px !important; |
| box-shadow: none !important; |
| filter: none !important; |
| margin: 0 0 16px !important; |
| overflow: hidden !important; |
| } |
| #leaderboard-search, |
| #category-view-select { |
| background: transparent !important; |
| border: 0 !important; |
| border-radius: 0 !important; |
| box-shadow: none !important; |
| filter: none !important; |
| margin: 0 !important; |
| overflow: visible !important; |
| } |
| #leaderboard-search textarea, |
| #leaderboard-search input { |
| background: var(--povisle-surface) !important; |
| } |
| #leaderboard-search-row, |
| #leaderboard-search-row *, |
| #leaderboard-category-row, |
| #leaderboard-category-row * { |
| box-shadow: none !important; |
| filter: none !important; |
| } |
| #leaderboard-search, |
| #leaderboard-search *, |
| #category-view-select, |
| #category-view-select * { |
| box-shadow: none !important; |
| filter: none !important; |
| } |
| #leaderboard-search, |
| #leaderboard-search .block, |
| #leaderboard-search .form, |
| #leaderboard-search .wrap, |
| #leaderboard-search .wrap-inner, |
| #leaderboard-search .input-container, |
| #leaderboard-search textarea, |
| #leaderboard-search input, |
| #category-view-select, |
| #category-view-select .block, |
| #category-view-select .form, |
| #category-view-select .wrap, |
| #category-view-select .wrap-inner, |
| #category-view-select .input-container { |
| background: var(--povisle-surface) !important; |
| } |
| #leaderboard-search .block, |
| #leaderboard-search .form, |
| #leaderboard-search .wrap, |
| #leaderboard-search .wrap-inner, |
| #leaderboard-search .input-container, |
| #category-view-select .block, |
| #category-view-select .form, |
| #category-view-select .wrap, |
| #category-view-select .wrap-inner, |
| #category-view-select .input-container { |
| border: 0 !important; |
| border-radius: 0 !important; |
| } |
| #leaderboard-category-row { |
| background: transparent !important; |
| border: 0 !important; |
| border-radius: 0 !important; |
| padding: 12px 0 !important; |
| overflow: visible !important; |
| } |
| #leaderboard-category-row > *, |
| #leaderboard-category-row #category-view-select, |
| #leaderboard-category-row #category-view-select .block, |
| #leaderboard-category-row #category-view-select .form, |
| #leaderboard-category-row #category-view-select .wrap, |
| #leaderboard-category-row #category-view-select .wrap-inner, |
| #leaderboard-category-row #category-view-select .input-container, |
| #leaderboard-category-row #category-view-select [role="radiogroup"] { |
| background: transparent !important; |
| border: 0 !important; |
| box-shadow: none !important; |
| } |
| #category-view-select .wrap, |
| #category-view-select .wrap-inner, |
| #category-view-select .input-container, |
| #category-view-select [role="radiogroup"] { |
| align-items: center !important; |
| display: flex !important; |
| flex-wrap: wrap !important; |
| gap: 8px !important; |
| } |
| #category-view-select label { |
| align-items: center !important; |
| background: var(--povisle-soft) !important; |
| border: 1px solid var(--povisle-line) !important; |
| border-radius: 999px !important; |
| color: var(--povisle-muted) !important; |
| cursor: pointer !important; |
| display: inline-flex !important; |
| font-size: 0.86rem !important; |
| font-weight: 600 !important; |
| line-height: 1 !important; |
| margin: 0 !important; |
| max-width: 100% !important; |
| min-height: 34px !important; |
| padding: 9px 13px !important; |
| transition: background 0.16s ease, border-color 0.16s ease, color 0.16s ease; |
| } |
| #category-view-select label:hover { |
| background: #fff1f2 !important; |
| border-color: #f2b8c0 !important; |
| color: var(--povisle-ink) !important; |
| } |
| #category-view-select label:has(input:checked) { |
| background: var(--povisle-accent) !important; |
| border-color: var(--povisle-accent) !important; |
| color: #ffffff !important; |
| } |
| #category-view-select label:has(input:focus-visible) { |
| outline: 2px solid rgba(215, 38, 61, 0.26) !important; |
| outline-offset: 2px !important; |
| } |
| #category-view-select input[type="radio"] { |
| height: 1px !important; |
| margin: 0 !important; |
| opacity: 0 !important; |
| position: absolute !important; |
| width: 1px !important; |
| } |
| #category-view-select label span { |
| color: inherit !important; |
| white-space: nowrap !important; |
| } |
| #leaderboard-filter-row { |
| gap: 0 !important; |
| overflow: hidden !important; |
| } |
| #leaderboard-filter-row > * { |
| background: var(--povisle-surface) !important; |
| position: relative !important; |
| } |
| .povisle-filter-field, |
| .leaderboard-filter-field { |
| background: var(--povisle-surface) !important; |
| gap: 0 !important; |
| padding: 14px 14px 12px !important; |
| } |
| .povisle-filter-field .block, |
| .povisle-filter-field .form, |
| .povisle-filter-field .wrap, |
| .povisle-filter-field .wrap-inner, |
| .povisle-filter-field .input-container, |
| .leaderboard-filter-field .block, |
| .leaderboard-filter-field .form, |
| .leaderboard-filter-field .wrap, |
| .leaderboard-filter-field .wrap-inner, |
| .leaderboard-filter-field .input-container { |
| background: transparent !important; |
| border: 0 !important; |
| box-shadow: none !important; |
| } |
| #leaderboard-filter-row > * + *::before { |
| background: #ddd7d3; |
| bottom: 18px; |
| content: ""; |
| left: 0; |
| position: absolute; |
| top: 18px; |
| width: 1px; |
| } |
| #type-filter, |
| #size-filter, |
| #family-filter { |
| background: var(--povisle-surface) !important; |
| border: 0 !important; |
| border-radius: 0 !important; |
| box-shadow: none !important; |
| } |
| #size-filter, |
| #family-filter { |
| border-left: 1px dashed #ddd7d3 !important; |
| } |
| #leaderboard-filter-row label, |
| #leaderboard-filter-row label span, |
| #leaderboard-filter-row .label-wrap, |
| #leaderboard-filter-row .label-wrap span, |
| #leaderboard-filter-row [data-testid="block-label"], |
| #leaderboard-filter-row [data-testid="block-label"] span { |
| color: var(--povisle-ink) !important; |
| opacity: 1 !important; |
| } |
| #leaderboard-filter-row .label-wrap span, |
| #leaderboard-filter-row [data-testid="block-label"] span { |
| font-weight: 700 !important; |
| } |
| #leaderboard-filter-row input, |
| #leaderboard-filter-row textarea, |
| #leaderboard-filter-row [role="combobox"], |
| #leaderboard-filter-row [data-testid="token"], |
| #leaderboard-filter-row .token, |
| #leaderboard-filter-row .selected-token, |
| #leaderboard-filter-row .multiselect-token { |
| color: var(--povisle-ink) !important; |
| } |
| #leaderboard-view-tabs { |
| border: 0 !important; |
| box-shadow: none !important; |
| margin: 8px 0 0 !important; |
| padding: 0 !important; |
| } |
| #leaderboard-view-tabs > .tab-nav { |
| background: transparent !important; |
| border-bottom: 1px solid var(--povisle-line) !important; |
| justify-content: flex-start !important; |
| margin: 0 0 16px !important; |
| max-width: none !important; |
| padding: 0 !important; |
| } |
| #leaderboard-view-tabs > .tab-nav button { |
| font-size: 0.94rem !important; |
| font-weight: 700 !important; |
| padding: 10px 14px !important; |
| } |
| #leaderboard-view-tabs > .tabitem { |
| border: 0 !important; |
| padding: 0 !important; |
| } |
| .leaderboard-split-tabs { |
| border: 0 !important; |
| box-shadow: none !important; |
| margin: 4px 0 0 !important; |
| padding: 0 !important; |
| } |
| .leaderboard-split-tabs .tab-nav { |
| background: transparent !important; |
| border-bottom: 1px solid var(--povisle-line) !important; |
| justify-content: flex-start !important; |
| margin: 0 0 14px !important; |
| max-width: none !important; |
| padding: 0 !important; |
| } |
| .leaderboard-split-tabs .tab-nav button { |
| font-size: 0.84rem !important; |
| padding: 8px 12px !important; |
| } |
| .leaderboard-split-tabs .tabitem { |
| border: 0 !important; |
| padding: 0 !important; |
| } |
| .leaderboard-table { |
| margin-top: 0 !important; |
| } |
| """ |
| APP_HEAD = """ |
| <link rel="preconnect" href="https://fonts.googleapis.com"> |
| <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> |
| <link href="https://fonts.googleapis.com/css2?family=Source+Serif+4:ital,opsz,wght@0,8..60,300;0,8..60,400;0,8..60,600;0,8..60,700;1,8..60,300;1,8..60,400&family=DM+Sans:ital,wght@0,300;0,400;0,500;0,600;1,300&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet"> |
| """ |
| HERO_HTML = """ |
| <section class="povisle-hero"> |
| <h1> |
| <em>Jako Tako</em> or Fluent? Presenting <span class="povisle-title-accent">PoVisLE</span> |
| </h1> |
| <p> |
| We introduce PoVisLE, a monocultural vision-language benchmark for Polish designed to evaluate |
| culturally grounded multimodal understanding under a grounded evaluation paradigm, where language |
| is interpreted in interaction with visual context. The dataset contains 1,117 images and 2,366 |
| manually annotated VQA pairs. |
| </p> |
| <div class="povisle-authors"> |
| <strong>Anna Kołos</strong> · <strong>Grzegorz Statkiewicz</strong> · <strong>Karolina Seweryn</strong> · |
| <strong>Katarzyna Kowol</strong> · <strong>Karolina Piosek</strong> · <strong>Wojciech Kusa</strong><br> |
| NASK National Research Institute, Warsaw, Poland |
| </div> |
| <div class="povisle-links-row"> |
| <a href="https://huggingface.co/collections/NASK-PIB/povisle" class="povisle-btn povisle-btn-dataset" target="_blank"> |
| <svg width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" aria-hidden="true"><ellipse cx="12" cy="5" rx="9" ry="3"/><path d="M21 12c0 1.66-4 3-9 3s-9-1.34-9-3"/><path d="M3 5v14c0 1.66 4 3 9 3s9-1.34 9-3V5"/></svg> |
| Dataset (Validation) |
| </a> |
| <a href="https://github.com/NASK-NLP/PoVisLE" class="povisle-btn povisle-btn-outline" target="_blank"> |
| <svg width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" aria-hidden="true"><path d="M9 19c-5 1.5-5-2.5-7-3m14 6v-3.87a3.37 3.37 0 0 0-.94-2.61c3.14-.35 6.44-1.54 6.44-7A5.44 5.44 0 0 0 20 4.77 5.07 5.07 0 0 0 19.91 1S18.73.65 16 2.48a13.38 13.38 0 0 0-7 0C6.27.65 5.09 1 5.09 1A5.07 5.07 0 0 0 5 4.77a5.44 5.44 0 0 0-1.5 3.78c0 5.42 3.3 6.61 6.44 7A3.37 3.37 0 0 0 9 18.13V22"/></svg> |
| Code |
| </a> |
| <a href="https://arxiv.org/abs/2608.07763" class="povisle-btn povisle-btn-outline" target="_blank"> |
| <svg class="arxiv-icon" width="23" height="23" viewBox="0 0 448 512" aria-hidden="true"><path fill="currentColor" d="m 119.65,351.996 c -5.84961,0.13477 -11.1943,-3.29883 -13.502,-8.67578 -2.19629,-5.27051 -0.61914,-8.9668 4.19727,-15.8652 7.05469,-10.3799 78.8242,-96.5625 78.8242,-96.5625 l -15.8828,-14.8633 c -13.3809,-13.3779 -13.9561,-31.333 -1.50977,-43.7754 l 18.4922,-17.6133 -51.5977,-63.377 c -4.00586,-4.26758 -6.48535,-11.7559 -4.24805,-17.1309 2.27832,-5.53613 7.69727,-9.12891 13.6836,-9.07031 3.83398,0.0957 7.43262,1.87402 9.83789,4.86133 l 61.3691,57.0566 94.5762,-90.0703 c 3.19434,-3.08398 7.44336,-4.83984 11.8828,-4.91016 1.60449,0.0039 3.19824,0.245117 4.73242,0.714844 5.7793,1.80566 10.249,6.41797 11.8711,12.252 1.2998,5.47363 -0.27637,11.2334 -4.18164,15.2832 l -83.0859,100.011996 14.8789,13.834 c 11.0957,10.0029 11.1543,27.3906 0.12695,37.4688 l -16.2949,15.6309 56.2559,66.4434 0.0742,0.0859 0.0664,0.0899 c 5.02734,6.53125 7.43164,11.5615 4.83984,17.9395 -3.13379,5.96875 -8.69727,10.29 -15.2559,11.8516 -0.67676,0.0908 -1.35938,0.13574 -2.04297,0.13671 l -0.004,-0.0117 c -4.42871,-0.27051 -8.61035,-2.13086 -11.7754,-5.24023 l -0.13086,-0.10743 -0.12304,-0.11132 -65.207,-59.127 -89.8965,86.2402 c 0,0 -5.33398,6.47754 -10.9707,6.61133 z m 178.104,-33.041 c 0.47852,-0.002 0.95703,-0.0332 1.43164,-0.0957 4.84277,-1.33301 8.95605,-4.54004 11.4316,-8.91016 1.44922,-3.5625 1.00293,-6.45703 -4.19727,-13.2148 l -56.0586,-66.2188 -26.375,25.3027 64.9551,58.9062 c 2.33887,2.41504 5.46484,3.91113 8.8125,4.2168 v 0.0137 z M 192.436,227.402 334.446,57.244 c 2.78711,-3.48926 4.5293,-6.97949 3.3418,-10.9121 -1.16113,-4.30273 -4.43457,-7.71777 -8.68555,-9.05664 -1.08105,-0.333984 -2.20508,-0.503906 -3.33594,-0.505859 -3.20801,0.07617 -6.26953,1.35645 -8.57617,3.58789 l -142.033,135.284999 c -11.1719,11.1719 -9.48242,26.0195 1.39258,36.8945 z"/></svg> |
| Paper |
| </a> |
| </div> |
| <div class="povisle-stat-row" aria-label="Dataset summary"> |
| <div class="povisle-stat"><strong>1,117</strong><span>images</span></div> |
| <div class="povisle-stat"><strong>2,366</strong><span>questions</span></div> |
| <div class="povisle-stat"><strong>7</strong><span>main categories</span></div> |
| <div class="povisle-stat"><strong>3</strong><span>task formats</span></div> |
| </div> |
| </section> |
| """ |
| DATASET_CREATION_IMAGE_URL = f"/gradio_api/file={quote(str((Path('public') / 'dataset.png').resolve()))}" |
| CATEGORY_SUNBURST_URL = f"/gradio_api/file={quote(str((Path('public') / 'categories_sunburst.png').resolve()))}" |
| DATASET_DESCRIPTION_HTML = f""" |
| <section class="povisle-dataset-card"> |
| <div class="povisle-section-label">Dataset</div> |
| <h2 class="povisle-section-title">About the Benchmark</h2> |
| <p> |
| PoVisLE is a monocultural vision-language evaluation benchmark centered on Polish cultural |
| and linguistic competence. It is designed for grounded evaluation: answers should depend on |
| the interaction between the image and the question, rather than on text-only associations or |
| surface-level entity recognition. |
| |
| </p> |
| <div class="povisle-about-grid"> |
| <div class="povisle-about-copy"> |
| <h3>Tasks</h3> |
| <div class="povisle-task-layout"> |
| <div> |
| <p> |
| During annotation, each VQA pair is assigned one of three task types. These |
| are multiple-choice, binary yes/no, and open-ended questions. All questions |
| are designed to require image understanding, and the answer should not be |
| obtainable from textual knowledge alone without reference to the image. |
| </p> |
| <ul class="povisle-task-list"> |
| <li><strong>Multiple-choice questions</strong> include answer options appended below the question with letter labels.</li> |
| <li><strong>Yes/no questions</strong> use the raw question as the prompt and are evaluated as binary answers.</li> |
| <li><strong>Open-ended questions</strong> also use the raw question as the prompt, without appended answer options.</li> |
| </ul> |
| </div> |
| <div class="povisle-task-chart" aria-label="Question distribution by task format and split"> |
| <div class="povisle-task-chart-legend" aria-hidden="true"> |
| <span style="--segment-color: #d7263d;">Multiple choice</span> |
| <span style="--segment-color: #2c5f8a;">Yes/no</span> |
| <span style="--segment-color: #2a7a4e;">Open-ended</span> |
| </div> |
| <div class="povisle-task-split"> |
| <div class="povisle-task-split-header"> |
| <span>Test split</span> |
| <strong>1,960</strong> |
| </div> |
| <div class="povisle-task-chart-track" title="Test: 643 multiple-choice, 714 yes/no, 603 open-ended"> |
| <div class="povisle-task-chart-segment" style="--segment-color: #d7263d; width: 32.8%;" aria-label="Multiple choice: 643, 32.8%">643</div> |
| <div class="povisle-task-chart-segment" style="--segment-color: #2c5f8a; width: 36.4%;" aria-label="Yes/no: 714, 36.4%">714</div> |
| <div class="povisle-task-chart-segment" style="--segment-color: #2a7a4e; width: 30.8%;" aria-label="Open-ended: 603, 30.8%">603</div> |
| </div> |
| </div> |
| <div class="povisle-task-split"> |
| <div class="povisle-task-split-header"> |
| <span>Validation split</span> |
| <strong>406</strong> |
| </div> |
| <div class="povisle-task-chart-track" title="Validation: 212 multiple-choice, 154 yes/no, 40 open-ended"> |
| <div class="povisle-task-chart-segment" style="--segment-color: #d7263d; width: 52.2%;" aria-label="Multiple choice: 212, 52.2%">212</div> |
| <div class="povisle-task-chart-segment" style="--segment-color: #2c5f8a; width: 37.9%;" aria-label="Yes/no: 154, 37.9%">154</div> |
| <div class="povisle-task-chart-segment" style="--segment-color: #2a7a4e; width: 9.9%;" aria-label="Open-ended: 40, 9.9%">40</div> |
| </div> |
| </div> |
| </div> |
| </div> |
| </div> |
| <div class="povisle-about-copy"> |
| <h3>Categories</h3> |
| <div class="povisle-category-layout"> |
| <div> |
| <p class="povisle-category-intro"> |
| The taxonomy is adapted from PLCC and refined for the VQA setting. Grammar |
| and vocabulary are merged into a unified Language category, while the hierarchy |
| supports fine-grained diagnostics across cultural, linguistic, and visual |
| reasoning domains. The dataset is organized into the following main categories. |
| </p> |
| <ul class="povisle-category-list"> |
| <li><strong>Art and Entertainment</strong> covers Polish and Poland-related artistic, media, and cultural references, including architecture, film, literature, music, paintings, sculpture, sport, and media.</li> |
| <li><strong>Culture and Tradition</strong> covers shared customs, practices, symbols, cuisine, religion, traditions, pop culture, and regional or ethnic cultural variation.</li> |
| <li><strong>Geography and Nature</strong> covers Polish physical, natural, urban, infrastructural, and socio-political spaces, including landscapes, landmarks, regions, and administrative entities.</li> |
| <li><strong>History and Society</strong> covers historical and contemporary Polish social context, including the Middle Ages, World War II, post-war history, and current affairs.</li> |
| <li><strong>Language</strong> covers visually grounded Polish linguistic phenomena, such as colloquial speech, slang, dialects, regionalisms, grammar, orthography, phraseology, rhetorical figures, and semantics.</li> |
| <li><strong>Image Understanding</strong> focuses on direct recognition and interpretation of visual content in Polish or Poland-related contexts.</li> |
| <li><strong>Visual Reasoning</strong> focuses on relationships, context, and inferred information within an image.</li> |
| </ul> |
| <p class="povisle-category-note"> |
| Image Understanding and Visual Reasoning form a smaller complementary subset |
| focused on general multimodal skills while remaining embedded in Polish visual |
| and linguistic contexts. |
| </p> |
| </div> |
| <div class="povisle-angled-sunburst" aria-label="Static sunburst of categories and subcategories"> |
| <img src="{CATEGORY_SUNBURST_URL}" alt="PoVisLE category and subcategory distribution"> |
| </div> |
| </div> |
| </div> |
| </div> |
| <div class="povisle-about-copy"> |
| <h3>Dataset Creation</h3> |
| <div class="povisle-creation-layout"> |
| <div> |
| <p> |
| PoVisLE was created through manual, template-free annotation. Annotators |
| selected or reviewed images from Wikimedia Commons, other permissively |
| available public sources, and personal collections contributed for research |
| use. Each image was paired with one or more Polish VQA prompts and labeled |
| with a task type, category, and subcategory. |
| </p> |
| <p> |
| The dataset construction also included a Wikimedia-based augmentation stage |
| to increase visual diversity and reduce selection bias. Candidate images |
| were reviewed by annotators, and visually similar replacements were used |
| only when the original question remained answerable from the new image. |
| </p> |
| <p> |
| Quality assurance included cross-validation by a second annotator, metadata |
| and license checks, regular team discussion, and supervision by an expert |
| annotator. The annotation guidelines required questions to be visually |
| grounded, unambiguous, linguistically natural, and suitable for deterministic |
| evaluation. |
| </p> |
| <p> |
| The benchmark is divided into a held-out test split for final evaluation and |
| a public validation split. The validation split is intended mainly to |
| illustrate the range of question types, so it is not sampled from the same |
| distribution as the test set. In particular, it contains fewer open-ended |
| questions, which are largely retained in the held-out test split. |
| </p> |
| </div> |
| <div class="povisle-creation-figure"> |
| <img src="{DATASET_CREATION_IMAGE_URL}" alt="Overview of the PoVisLE dataset construction process"> |
| </div> |
| </div> |
| </div> |
| <div class="povisle-about-copy"> |
| <h3>Evaluation Protocol</h3> |
| <p> |
| Models receive the image and a Polish prompt specifying the expected answer format, length, |
| word order, and, where relevant, grammatical form. Macro accuracy is the main metric. |
| Multiple-choice questions use circular evaluation: answer options are cyclically rotated, and |
| a prediction is counted as correct only if the model selects the gold answer under every |
| rotation. This reduces option-position bias while keeping evaluation efficient. |
| </p> |
| <p> |
| Yes/no and open-ended questions are evaluated in a single pass. Yes/no questions require a |
| binary answer in Polish. Open-ended predictions are compared against the gold answers, with |
| correct diacritics required in all cases and correct capitalization required where relevant. |
| For selected questions, multiple answer variants are accepted through predefined inclusion |
| patterns developed through iterative human validation. |
| </p> |
| </div> |
| </section> |
| """ |
| LEADERBOARD_SPLITS = ("test", "validation") |
| RESULTS_REPO = os.getenv("RESULTS_REPO", "NASK-PIB/PoVisLE-results") |
| SOURCE_DATASET_REPO = os.getenv("SOURCE_DATASET_REPO", "NASK-PIB/PoVisLE") |
| SOURCE_DATASET_CONFIGS = [ |
| config.strip() |
| for config in os.getenv("SOURCE_DATASET_CONFIGS", "mcq,open,yn").split(",") |
| if config.strip() |
| ] |
| EXAMPLES_DATASET_SPLITS = [ |
| split.strip() |
| for split in os.getenv("EXAMPLES_DATASET_SPLITS", os.getenv("EXAMPLES_DATASET_SPLIT", "validation")).split(",") |
| if split.strip() |
| ] |
| SOURCE_DATASET_REVISION = os.getenv("SOURCE_DATASET_REVISION", "v1.2.0") |
| HF_TOKEN = os.getenv("HF_TOKEN") |
| CACHE_PATH = Path(os.getenv("HF_HOME", ".")).expanduser() |
| LOCAL_RESULTS_PATH = Path(os.getenv("LOCAL_RESULTS_PATH", CACHE_PATH / "PoVisLE-results")).expanduser() |
| PUBLIC_PATH = Path("public").expanduser() |
| PUBLIC_IMAGES_PATH = Path(os.getenv("PUBLIC_IMAGES_PATH", PUBLIC_PATH / "images")).expanduser() |
| LEADERBOARD_RELATIVE_PATH = Path("leaderboard") |
| BENCHMARK_VERSION = os.getenv("BENCHMARK_VERSION", "v1.2.0") |
| CATEGORY_DISPLAY_ORDER = [ |
| "Art & Entertainment", |
| "Culture & Tradition", |
| "Geography & Nature", |
| "History & Society", |
| "Language", |
| "Image Understanding", |
| "Visual Reasoning", |
| ] |
| CATEGORY_ORDER_ALIASES = { |
| "art entertainment": "Art & Entertainment", |
| "art and entertainment": "Art & Entertainment", |
| "culture tradition": "Culture & Tradition", |
| "culture and tradition": "Culture & Tradition", |
| "geography nature": "Geography & Nature", |
| "geography and nature": "Geography & Nature", |
| "history society": "History & Society", |
| "history and society": "History & Society", |
| "language": "Language", |
| "image understanding": "Image Understanding", |
| "visual reasoning": "Visual Reasoning", |
| } |
|
|
|
|
| def model_size_bucket(model_size: str | None) -> str: |
| if not model_size: |
| return "Unknown" |
|
|
| normalized = str(model_size).strip().upper() |
| if not normalized.endswith("B"): |
| return "Unknown" |
|
|
| try: |
| size_value = float(normalized[:-1]) |
| except ValueError: |
| return "Unknown" |
|
|
| if size_value < 10: |
| return "1-10B" |
| if size_value < 20: |
| return "10-20B" |
| if size_value < 40: |
| return "20-40B" |
| if size_value <= 100: |
| return "30-100B" |
| return ">100B" |
|
|
|
|
| def result_file_patterns() -> list[str]: |
| return [ |
| f"leaderboard/*/*/{BENCHMARK_VERSION}/test/circular_circular/results.json", |
| f"leaderboard/*/*/{BENCHMARK_VERSION}/validation/circular_circular/results.json", |
| ] |
|
|
|
|
| def matching_result_files(repo_files: list[str], patterns: list[str]) -> list[str]: |
| return sorted(path for path in repo_files if any(fnmatch(path, pattern) for pattern in patterns)) |
|
|
|
|
| def refresh_results_from_hub(local_results_path: Path) -> None: |
| patterns = result_file_patterns() |
| repo_files = HfApi(token=HF_TOKEN).list_repo_files(repo_id=RESULTS_REPO, repo_type="dataset") |
| matched_files = matching_result_files(repo_files, patterns) |
| if not matched_files: |
| raise FileNotFoundError( |
| f"No result files matching {patterns} were found in dataset {RESULTS_REPO}. " |
| f"Check RESULTS_REPO, BENCHMARK_VERSION={BENCHMARK_VERSION}, and HF_TOKEN access." |
| ) |
|
|
| local_results_path.mkdir(parents=True, exist_ok=True) |
| for filename in matched_files: |
| hf_hub_download( |
| repo_id=RESULTS_REPO, |
| filename=filename, |
| repo_type="dataset", |
| local_dir=str(local_results_path), |
| etag_timeout=30, |
| token=HF_TOKEN, |
| ) |
|
|
|
|
| def resolve_results_dir() -> Path: |
| env_path = os.environ.get("VPLCC_RESULTS_DIR") |
| if env_path: |
| candidate = Path(env_path).expanduser() |
| if candidate.exists(): |
| return candidate |
|
|
| try: |
| refresh_results_from_hub(LOCAL_RESULTS_PATH) |
| except Exception as error: |
| fallback = LOCAL_RESULTS_PATH / LEADERBOARD_RELATIVE_PATH |
| if fallback.exists(): |
| return fallback |
| raise FileNotFoundError( |
| f"Could not download results dataset from {RESULTS_REPO} and no local cache was found. ({error})" |
| ) from error |
|
|
| leaderboard_path = LOCAL_RESULTS_PATH / LEADERBOARD_RELATIVE_PATH |
| if leaderboard_path.exists(): |
| return leaderboard_path |
| raise FileNotFoundError(f"Could not find leaderboard directory inside downloaded dataset: {leaderboard_path}") |
|
|
|
|
| def format_category_label(category_name: str) -> str: |
| return str(category_name).replace("_", " ").title() |
|
|
|
|
| def make_subcategory_column_key(category_name: str, subcategory_name: str) -> str: |
| return f"__subcategory__::{category_name}::{subcategory_name}" |
|
|
|
|
| def percentage(value: float) -> float: |
| return round(float(value) * 100, 2) |
|
|
|
|
| def build_category_scores(metrics: dict) -> dict[str, float]: |
| return { |
| format_category_label(category_name): percentage(category_metrics["macro_accuracy"]) |
| for category_name, category_metrics in metrics.get("by_category", {}).items() |
| } |
|
|
|
|
| def build_subcategory_scores(metrics: dict) -> tuple[dict[str, float], dict[str, list[tuple[str, str]]]]: |
| subcategory_scores: dict[str, float] = {} |
| subcategory_groups: dict[str, list[tuple[str, str]]] = {} |
|
|
| for category_name, subcategory_metrics in metrics.get("by_category_and_subcategory", {}).items(): |
| category_label = format_category_label(category_name) |
| group_columns: list[tuple[str, str]] = [] |
|
|
| for subcategory_name, details in subcategory_metrics.items(): |
| column_key = make_subcategory_column_key(category_label, subcategory_name) |
| subcategory_label = format_category_label(subcategory_name) |
| group_columns.append((column_key, subcategory_label)) |
| subcategory_scores[column_key] = percentage(details["macro_accuracy"]) |
|
|
| if group_columns: |
| subcategory_groups[category_label] = group_columns |
|
|
| return subcategory_scores, subcategory_groups |
|
|
|
|
| def build_task_scores(metrics: dict) -> dict[str, float]: |
| task_scores = {} |
| task_column_labels = { |
| "mcq": "MCQ", |
| "yn": "Y/N", |
| "open": "Open", |
| } |
| for task_name, task_metrics in metrics.get("by_task", {}).items(): |
| normalized_task_name = normalize_result_task_name(task_name) |
| column_label = task_column_labels.get(normalized_task_name) |
| if column_label: |
| task_scores[column_label] = percentage(task_metrics["accuracy"]) |
| return task_scores |
|
|
|
|
| def iter_results_paths(results_dir: Path, split: str): |
| return sorted(results_dir.glob(f"*/*/{BENCHMARK_VERSION}/{split}/circular_circular/results.json")) |
|
|
|
|
| def normalize_result_task_name(task_name: object) -> str: |
| normalized = str(task_name).strip().lower().replace("_", "-") |
| if normalized in {"yes/no", "yes-no", "yesno"}: |
| return "yn" |
| return normalized |
|
|
|
|
| def results_path_priority(results_path: Path, payload: dict) -> tuple[int, int, int, int, str]: |
| metadata = payload.get("run_metadata", {}) |
| metrics = payload.get("metrics", {}) |
| mode = str(metadata.get("evaluation_mode") or results_path.parent.name).lower().replace("-", "_") |
| ablation_markers = ("no_image", "without_image", "no_question", "without_question") |
| is_ablation = any(metadata.get(marker) for marker in ablation_markers) or any( |
| marker in mode for marker in ablation_markers |
| ) |
| full_input_priority = 0 if is_ablation else 1 |
| expected_tasks = {normalize_result_task_name(task_name) for task_name in SOURCE_DATASET_CONFIGS} |
| result_tasks = { |
| normalize_result_task_name(task_name) |
| for task_name in (metadata.get("tasks") or metrics.get("by_task", {}).keys()) |
| } |
| all_tasks_priority = 1 if expected_tasks and expected_tasks.issubset(result_tasks) else 0 |
| total_priority = int(metrics.get("overall", {}).get("total") or 0) |
| default_mode_priority = 1 if mode == "circular_circular" else 0 |
| return full_input_priority, all_tasks_priority, total_priority, default_mode_priority, str(results_path) |
|
|
|
|
| def sort_category_columns(category_labels: set[str]) -> list[str]: |
| priority = {label: index for index, label in enumerate(CATEGORY_DISPLAY_ORDER)} |
|
|
| def category_sort_key(label: str) -> tuple[int, str]: |
| normalized = re.sub(r"[^a-z0-9]+", " ", str(label).lower()).strip() |
| canonical_label = CATEGORY_ORDER_ALIASES.get(normalized, label) |
| return priority.get(canonical_label, len(CATEGORY_DISPLAY_ORDER)), label |
|
|
| return sorted(category_labels, key=category_sort_key) |
|
|
|
|
| def safe_image_filename(task_id: object, index: object | None = None) -> str: |
| raw_name = str(task_id) |
| if index is not None: |
| raw_name = f"{raw_name}-{index}" |
| safe_name = re.sub(r"[^A-Za-z0-9_.-]+", "_", raw_name).strip("._") |
| return f"{safe_name or 'image'}.png" |
|
|
|
|
| def public_image_path(task_id: object, index: object | None = None) -> Path: |
| return PUBLIC_IMAGES_PATH.resolve() / safe_image_filename(task_id, index) |
|
|
|
|
| def public_file_url(path: Path) -> str: |
| return f"/gradio_api/file={quote(str(path.resolve()))}" |
|
|
|
|
| def pil_image_from_value(image: object): |
| if image is None: |
| return None |
|
|
| try: |
| from PIL import Image |
| except ImportError: |
| return None |
|
|
| image_bytes: bytes | None = None |
|
|
| if isinstance(image, dict): |
| raw_bytes = image.get("bytes") |
| image_path = image.get("path") |
| if raw_bytes: |
| image_bytes = raw_bytes |
| elif image_path: |
| candidate = Path(image_path) |
| if candidate.exists(): |
| image_bytes = candidate.read_bytes() |
| elif isinstance(image, bytes): |
| image_bytes = image |
| elif isinstance(image, str): |
| if image.startswith("data:image/") and "," in image: |
| image = image.split(",", 1)[1] |
| try: |
| candidate = Path(image) |
| if len(image) < 512 and candidate.exists(): |
| image_bytes = candidate.read_bytes() |
| except OSError: |
| image_bytes = None |
| if image_bytes is None: |
| try: |
| image_bytes = base64.b64decode(image, validate=True) |
| except Exception: |
| image_bytes = None |
| elif hasattr(image, "copy") and hasattr(image, "save"): |
| return image.copy() |
|
|
| if image_bytes is None: |
| return None |
|
|
| try: |
| return Image.open(BytesIO(image_bytes)).copy() |
| except Exception: |
| return None |
|
|
|
|
| def save_image_thumbnail(image: object, output_path: Path, max_size: tuple[int, int] = (128, 128)) -> bool: |
| if output_path.exists(): |
| return True |
|
|
| pil_image = pil_image_from_value(image) |
| if pil_image is None: |
| return False |
|
|
| thumbnail = pil_image.copy() |
| thumbnail.thumbnail(max_size) |
| if thumbnail.mode not in {"RGB", "RGBA"}: |
| thumbnail = thumbnail.convert("RGB") |
|
|
| output_path.parent.mkdir(parents=True, exist_ok=True) |
| thumbnail.save(output_path, format="PNG") |
| return True |
|
|
|
|
| def image_file_to_html(image_path: Path) -> str: |
| if not image_path.exists(): |
| return "" |
|
|
| return f'<img class="example-image" src="{public_file_url(image_path)}" alt="task image" loading="lazy">' |
|
|
|
|
| def format_choices(item: dict[str, object]) -> str: |
| choices = [] |
| for label in "ABCDEFGH": |
| value = item.get(label) |
| if value is None or pd.isna(value): |
| continue |
| text = str(value).strip() |
| if text: |
| choices.append(f"{label}. {text}") |
| return "\n".join(choices) |
|
|
|
|
| def format_example_value(value: object) -> object: |
| if isinstance(value, (list, dict)): |
| return json.dumps(value, ensure_ascii=False) |
| if value is None or pd.isna(value): |
| return "" |
| return value |
|
|
|
|
| def load_examples_rows() -> list[dict[str, object]]: |
| try: |
| from datasets import load_dataset |
| except ImportError as error: |
| print(f"Could not load examples because datasets is not installed: {error}") |
| return [] |
|
|
| rows: list[dict[str, object]] = [] |
| loaded_sources: set[tuple[str, str]] = set() |
| for config_name in SOURCE_DATASET_CONFIGS: |
| dataset = None |
| selected_split = None |
| errors: list[str] = [] |
| for split_name in EXAMPLES_DATASET_SPLITS: |
| load_kwargs = { |
| "path": SOURCE_DATASET_REPO, |
| "name": config_name, |
| "split": split_name, |
| "token": HF_TOKEN, |
| } |
| if SOURCE_DATASET_REVISION: |
| load_kwargs["revision"] = SOURCE_DATASET_REVISION |
|
|
| try: |
| dataset = load_dataset(**load_kwargs) |
| selected_split = split_name |
| break |
| except Exception as error: |
| errors.append(f"{split_name}: {error}") |
|
|
| if dataset is None or selected_split is None: |
| print(f"Could not load examples from {SOURCE_DATASET_REPO}/{config_name}. Tried {', '.join(errors)}.") |
| continue |
|
|
| loaded_sources.add((config_name, selected_split)) |
| for item in dataset: |
| image_path = public_image_path(item["id"]) |
| image_html = image_file_to_html(image_path) if save_image_thumbnail(item.get("image"), image_path) else "" |
| rows.append( |
| { |
| "id": item["id"], |
| "image": image_html, |
| "task": item.get("task") or config_name, |
| "category": item.get("category"), |
| "subcategory": item.get("subcategory"), |
| "question": item.get("question"), |
| "choices": format_choices(item), |
| "answer": item.get("answer"), |
| "include": format_example_value(item.get("include")), |
| "check_casing": format_example_value(item.get("check_casing")), |
| "check_diacritics": format_example_value(item.get("check_diacritics")), |
| } |
| ) |
|
|
| if rows: |
| sources = ", ".join(f"{config}/{split}" for config, split in sorted(loaded_sources)) |
| print(f"Loaded {len(rows)} examples from {SOURCE_DATASET_REPO}: {sources}.") |
| else: |
| print(f"No examples were loaded from {SOURCE_DATASET_REPO}.") |
| return rows |
|
|
|
|
| def load_dashboard_data() -> tuple[pd.DataFrame, list[str], dict[str, list[tuple[str, str]]], pd.DataFrame, str]: |
| results_dir = resolve_results_dir() |
| leaderboard_rows = [] |
| latest_results_by_model: dict[tuple[str, str], tuple[Path, tuple[int, int, int, int, str]]] = {} |
| category_labels: set[str] = set() |
| all_subcategory_groups: dict[str, list[tuple[str, str]]] = {} |
|
|
| for split in LEADERBOARD_SPLITS: |
| for results_path in iter_results_paths(results_dir, split): |
| payload = json.loads(results_path.read_text(encoding="utf-8")) |
| metadata = payload["run_metadata"] |
| metadata_split = metadata.get("split") |
| if metadata_split and str(metadata_split).lower() != split: |
| continue |
| model_name = metadata["model_name"] |
| if model_name in EXCLUDED_MODELS: |
| continue |
| priority = results_path_priority(results_path, payload) |
| key = (split, model_name) |
| if key not in latest_results_by_model or priority > latest_results_by_model[key][1]: |
| latest_results_by_model[key] = (results_path, priority) |
|
|
| for (split, model_name), (results_path, _) in sorted(latest_results_by_model.items()): |
| payload = json.loads(results_path.read_text(encoding="utf-8")) |
| metadata = payload["run_metadata"] |
| metrics = payload["metrics"] |
| category_scores = build_category_scores(metrics) |
| subcategory_scores, subcategory_groups = build_subcategory_scores(metrics) |
| task_scores = build_task_scores(metrics) |
| overall_score = percentage(metrics["overall"]["macro_accuracy"]) |
| category_labels.update(category_scores) |
| for category_label, columns in subcategory_groups.items(): |
| known_column_keys = {column_key for column_key, _ in all_subcategory_groups.get(category_label, [])} |
| all_subcategory_groups.setdefault(category_label, []) |
| for column_key, column_label in columns: |
| if column_key not in known_column_keys: |
| all_subcategory_groups[category_label].append((column_key, column_label)) |
| known_column_keys.add(column_key) |
| leaderboard_rows.append( |
| { |
| "Split": split.title(), |
| "Org": metadata.get("org", "unknown"), |
| "Model": model_name, |
| "Type": MODEL_TYPE_LABELS.get(metadata["model_type"], metadata["model_type"].title()), |
| "Family": metadata.get("model_family", "unknown"), |
| "Size": metadata.get("model_size", "unknown"), |
| "Size range": model_size_bucket(metadata.get("model_size")), |
| "Overall": overall_score, |
| **task_scores, |
| **category_scores, |
| **subcategory_scores, |
| } |
| ) |
|
|
| category_columns = sort_category_columns(category_labels) |
| leaderboard = pd.DataFrame(leaderboard_rows) |
| for task_column in ["MCQ", "Y/N", "Open"]: |
| if task_column not in leaderboard: |
| leaderboard[task_column] = pd.NA |
| for category_column in category_columns: |
| if category_column not in leaderboard: |
| leaderboard[category_column] = pd.NA |
| for subcategory_columns in all_subcategory_groups.values(): |
| for column_key, _ in subcategory_columns: |
| if column_key not in leaderboard: |
| leaderboard[column_key] = pd.NA |
|
|
| leaderboard = leaderboard.sort_values(by=["Split", "Overall", *category_columns], ascending=[True, False, *([False] * len(category_columns))]) |
| leaderboard.insert(0, "Rank", range(1, len(leaderboard) + 1)) |
| examples_df = pd.DataFrame( |
| load_examples_rows(), |
| columns=[ |
| "id", |
| "image", |
| "task", |
| "category", |
| "subcategory", |
| "question", |
| "choices", |
| "answer", |
| "include", |
| "check_casing", |
| "check_diacritics", |
| ], |
| ) |
| return leaderboard, category_columns, all_subcategory_groups, examples_df, str(results_dir) |
|
|
|
|
| PUBLIC_IMAGES_PATH.mkdir(parents=True, exist_ok=True) |
| gr.set_static_paths(paths=[PUBLIC_PATH.resolve(), PUBLIC_IMAGES_PATH.resolve()]) |
|
|
| LEADERBOARD_DF, CATEGORY_COLUMNS, SUBCATEGORY_GROUPS, EXAMPLES_DF, RESULTS_DIR = load_dashboard_data() |
|
|
| with gr.Blocks( |
| title="PoVisLE", |
| ) as demo: |
| gr.HTML(HERO_HTML) |
| with gr.Group(elem_id="leaderboard-section"): |
| gr.HTML( |
| """ |
| <section class="povisle-leaderboard-heading"> |
| <div class="povisle-section-label">Results</div> |
| <h2 class="povisle-section-title">Leaderboard</h2> |
| <p>Compare model performance by category or task type across the test and validation splits. Overall and category scores report <strong>macro accuracy</strong>, while task columns report task-level accuracy.</p> |
| </section> |
| """ |
| ) |
| search_box, type_filter, size_filter, family_filter = render_leaderboard_filters(LEADERBOARD_DF) |
| with gr.Tabs(elem_id="leaderboard-view-tabs"): |
| with gr.Tab("By category"): |
| render_leaderboard_category_tab( |
| LEADERBOARD_DF, |
| CATEGORY_COLUMNS, |
| SUBCATEGORY_GROUPS, |
| search_box, |
| type_filter, |
| size_filter, |
| family_filter, |
| ) |
| with gr.Tab("By task"): |
| render_leaderboard_task_tab( |
| LEADERBOARD_DF, |
| search_box, |
| type_filter, |
| size_filter, |
| family_filter, |
| ) |
| gr.HTML(DATASET_DESCRIPTION_HTML) |
| render_examples_tab(EXAMPLES_DF) |
| gr.HTML( |
| """ |
| <section class="povisle-acknowledgement"> |
| <h2 class="povisle-section-title">Acknowledgement</h2> |
| <p>This work was supported by the Polish Ministry of Digital Affairs (subsidy no. 4/WII/DBI/2026). The computational resources were provided by the Polish high-performance computing infrastructure PLGrid (HPC Center: ACK Cyfronet AGH) under computational grant no. PLG/2026/019138.</p> |
| </section> |
| <section class="povisle-citation"> |
| <h2 class="povisle-section-title">Citation</h2> |
| <pre>@article{kolos2026povisle, |
| title = {Jako Tako or Fluent? Presenting PoVisLE: A Polish Vision-Language Evaluation}, |
| author = {Ko{\\l}os, Anna and Statkiewicz, Grzegorz and Seweryn, Karolina and Kowol, Katarzyna and Piosek, Karolina and Kusa, Wojciech}, |
| journal = {arXiv preprint}, |
| year = {2026} |
| }</pre> |
| </section> |
| """ |
| ) |
|
|
|
|
| if __name__ == "__main__": |
| demo.launch( |
| theme=POVISLE_THEME, |
| css=APP_CSS, |
| head=APP_HEAD, |
| ) |
|
|
