from datetime import datetime, timezone import json import os import re import shutil from typing import Optional import gradio as gr import pandas as pd from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns from src.about import ( CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, INTRODUCTION_TEXT, LLM_BENCHMARKS_TEXT, TITLE, ) from src.display.css_html_js import custom_css from src.envs import RESULTS_PATH, SUBMISSIONS_PATH from src.leaderboard.load_results import ( DISPLAY_METADATA_COLUMNS, V1_0_DOMAINS, V1_1_DOMAINS, ResultsValidationError, build_dataframe, load_records, ) from src.leaderboard.schema import SCHEMA V1_1_LABEL = "V1.1 \u2014 5 datasets (adds M5)" V1_0_LABEL = "V1.0 \u2014 original 4 datasets" VERSION_DOMAINS = { V1_1_LABEL: V1_1_DOMAINS, V1_0_LABEL: V1_0_DOMAINS, } RANKING_METRIC = "overall_t1_t3_t4_acc" def load_leaderboard_data() -> tuple[dict[str, tuple[pd.DataFrame, list[str]]], Optional[str]]: """Build one ranked view per benchmark version from the same result records.""" try: records = load_records(RESULTS_PATH) views = {} for label, domains in VERSION_DOMAINS.items(): df, column_order = build_dataframe(records, domains=domains) if RANKING_METRIC in df.columns: df = df.sort_values(RANKING_METRIC, ascending=False, na_position="last") views[label] = (df, column_order) return views, None except ResultsValidationError as exc: fallback_cols = list(SCHEMA.identity_fields) + list(SCHEMA.required_metrics) empty = pd.DataFrame(columns=fallback_cols) return {label: (empty, fallback_cols) for label in VERSION_DOMAINS}, str(exc) VERSION_VIEWS, LOAD_ERROR = load_leaderboard_data() # Unfiltered five-domain view: the By Domain tab reports per-dataset results for every # entry, including those evaluated only on the four V1.0 domains. LEADERBOARD_DF, COLUMN_ORDER = VERSION_VIEWS[V1_1_LABEL] DATASET_PREFIX_MAP = { "FreshRetailNet": "FreshRetailNet", "PSML": "PSML", "Causal Chambers": "CausalChambers", "MIMIC": "MIMIC", "M5": "M5", } DATASET_PREFIXES = [f"{prefix}_" for prefix in DATASET_PREFIX_MAP.values()] def is_dataset_metric(column: str) -> bool: return any(column.startswith(prefix) for prefix in DATASET_PREFIXES) BASE_COLUMNS = list(SCHEMA.identity_fields) + list(DISPLAY_METADATA_COLUMNS) + list(SCHEMA.required_metrics) ALL_DATASET_COLUMNS = [c for c in COLUMN_ORDER if is_dataset_metric(c)] M5_COLUMNS = [column for column in ALL_DATASET_COLUMNS if column.startswith("M5_")] NON_M5_DATASET_COLUMNS = [ column for column in ALL_DATASET_COLUMNS if column not in M5_COLUMNS ] AGGREGATE_FORECAST_COLUMNS = [ "overall_t1_t3_t4_acc", "overall_mcq_acc", "T2_MAE", "T2_sMAPE", "T4_MAE", "T4_sMAPE", "MIMIC_T2_OW_sMAPE", "MIMIC_T2_OW_RMSSE", "MIMIC_T4_OW_sMAPE", "MIMIC_T4_OW_RMSSE", ] def aggregate_columns(column_order: list[str]) -> list[str]: return BASE_COLUMNS + [c for c in AGGREGATE_FORECAST_COLUMNS if c in column_order] def complete_entries(df: pd.DataFrame) -> pd.DataFrame: """Entries evaluated on every domain of the selected version. A version's leaderboard ranks only entries measured on its full domain set, so V1.0-only entries are absent from V1.1. V1.0 stays backward compatible: it lists those entries together with V1.1 entries reduced to the same four domains. """ if "benchmark_complete" not in df.columns: return df return df[df["benchmark_complete"]] BY_DOMAIN_COLUMNS = BASE_COLUMNS + ALL_DATASET_COLUMNS DISPLAY_LABELS = { "agent_name": "Agent", "agent_type": "Agent Type", "base_model": "Base Model", "paper": "Paper", "evaluation_scope": "Evaluation Scope", "domain_coverage": "Domain Coverage", "benchmark_complete": "All Domains Evaluated", "overall_mcq_acc": "Overall MCQ Acc", "overall_t1_t3_t4_acc": "Overall T1/T3/T4 Acc", "T1_acc": "Overall T1 Acc", "T2_acc": "Overall T2 Acc", "T3_acc": "Overall T3 Acc", "T4_acc": "Overall T4 Acc", "T2_MAE": "Overall T2 MAE", "T2_sMAPE": "Overall T2 sMAPE", "T4_MAE": "Overall T4 MAE", "T4_sMAPE": "Overall T4 sMAPE", "MIMIC_T2_OW_sMAPE": "MIMIC T2 OW-sMAPE", "MIMIC_T2_OW_RMSSE": "MIMIC T2 OW-RMSSE", "MIMIC_T4_OW_sMAPE": "MIMIC T4 OW-sMAPE", "MIMIC_T4_OW_RMSSE": "MIMIC T4 OW-RMSSE", } def display_label(column: str) -> str: if column in DISPLAY_LABELS: return DISPLAY_LABELS[column] match = re.fullmatch(r"(.+)_T([1-4])_(acc|MAE|sMAPE|OW_sMAPE|OW_RMSSE)", column) if match is None: return column domain, tier, metric = match.groups() domain = "Causal Chambers" if domain == "CausalChambers" else domain return f"{domain} T{tier} {metric.replace('_', '-')}" def column_types(column_order: list[str]) -> list[str]: types = [] for col in column_order: if col in SCHEMA.identity_fields or col in {"evaluation_scope", "domain_coverage"}: types.append("str") elif col == "paper": types.append("markdown") elif col == "benchmark_complete": types.append("bool") else: types.append("number") return types def init_leaderboard(dataframe, column_order, default_columns=None): if dataframe is None or dataframe.empty: dataframe = pd.DataFrame(columns=column_order) dataframe = dataframe.reindex(columns=column_order) display_columns = [display_label(column) for column in column_order] dataframe = dataframe.rename(columns=dict(zip(column_order, display_columns))) required_cols = list(SCHEMA.identity_fields) + ["evaluation_scope", "domain_coverage"] cant_deselect = [display_label(c) for c in required_cols if c in column_order] default_columns = default_columns or column_order default_selection = [display_label(c) for c in default_columns if c in column_order] search_columns = [ display_label(c) for c in ["agent_name", "base_model"] if c in column_order ] return Leaderboard( value=dataframe, datatype=column_types(column_order), select_columns=SelectColumns( default_selection=default_selection, cant_deselect=cant_deselect, label="Select Columns to Display:", ), search_columns=search_columns, filter_columns=[ ColumnFilter(display_label("agent_type"), type="checkboxgroup", label="Agent Type"), ], interactive=False, ) def select_version(choice: str): """Show the panel for the selected benchmark version, hide the other.""" is_v1_1 = choice == V1_1_LABEL return gr.update(visible=is_v1_1), gr.update(visible=not is_v1_1) def save_submission(uploaded_file) -> str: if uploaded_file is None: return "Please upload a results file." file_path = uploaded_file.name if hasattr(uploaded_file, "name") else str(uploaded_file) extension = os.path.splitext(file_path)[1].lower() if extension not in {".zip", ".rar"}: return "Please upload a .zip or .rar submission package." os.makedirs(SUBMISSIONS_PATH, exist_ok=True) timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") out_path = os.path.join(SUBMISSIONS_PATH, f"submission_{timestamp}{extension}") shutil.copyfile(file_path, out_path) return "Submission received for manual review." def build_example_record_markdown() -> str: try: records = load_records(RESULTS_PATH) if not records: return "No example data available." example = records[0] return "Example record (JSON):\n```json\n" + json.dumps(example, indent=2) + "\n```" except Exception as exc: return f"Could not load example record: {exc}" EXAMPLE_RECORD_MD = build_example_record_markdown() demo = gr.Blocks(css=custom_css, analytics_enabled=False) with demo: gr.HTML(TITLE) gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") if LOAD_ERROR: gr.Markdown(f"**Data validation error:** {LOAD_ERROR}", elem_classes="markdown-text") with gr.Tabs(elem_classes="tab-buttons") as tabs: with gr.TabItem("🏅 Leaderboard", elem_id="tab-leaderboard", id=0): version_selector = gr.Radio( choices=[V1_1_LABEL, V1_0_LABEL], value=V1_1_LABEL, label="Benchmark version", info=( "V1.0 restricts every entry to the four original domains and re-ranks " "them, so all entries are compared on the same domain set." ), ) v1_1_df, v1_1_columns = VERSION_VIEWS[V1_1_LABEL] v1_0_df, v1_0_columns = VERSION_VIEWS[V1_0_LABEL] v1_1_df = complete_entries(v1_1_df) v1_0_df = complete_entries(v1_0_df) with gr.Column(visible=True) as v1_1_panel: gr.Markdown( f"**Current entries:** {len(v1_1_df)} agent configurations evaluated on all " "five V1.1 datasets. Entries measured only on the four V1.0 domains are not " "listed here — switch to **V1.0** to see them alongside these entries reduced " "to the same four domains. Rows are ranked by **Overall T1/T3/T4 Acc**, " "weighted by the corresponding MCQ question counts.\n\n" "⚠️ M5 was evaluated in a later round than the other four domains, after the " "DeepSeek API used for the original baselines was withdrawn. For the " "**AgentScope**, **CAMEL**, **MetaGPT** and **TimeSeries Scientist** " "`deepseek-chat` rows the overall score therefore averages across two API " "versions, and small gaps between them are not meaningful. See the **About** " "tab for details." ) init_leaderboard(v1_1_df, aggregate_columns(v1_1_columns)) with gr.Column(visible=False) as v1_0_panel: gr.Markdown( f"**Current entries:** {len(v1_0_df)} agent configurations on the four V1.0 " "domains (FreshRetailNet, PSML, Causal Chambers, MIMIC). This view is backward " "compatible: it lists V1.0-only entries together with V1.1 entries reduced to " "their four-domain subset, with **Overall T1/T3/T4 Acc** recomputed from those " "domains only and the ranking recalculated — so every row is compared on the " "same domain set.\n\n" "Because M5 is excluded, this view is free of the M5 API-version seam " "described in the **About** tab." ) init_leaderboard(v1_0_df, aggregate_columns(v1_0_columns)) version_selector.change( select_version, version_selector, [v1_1_panel, v1_0_panel], ) with gr.TabItem("🧭 By Domain", elem_id="tab-by-domain", id=1): gr.Markdown( f"Per-dataset results for all {len(LEADERBOARD_DF)} entries, including those " "evaluated only on the four V1.0 domains — **Domain Coverage** marks which is " "which. The newest evaluation domain, **M5**, is shown first. Use **Select " "Columns to Display** to add forecasting metrics or other domains." ) domain_default_columns = BASE_COLUMNS + M5_COLUMNS + [ column for column in NON_M5_DATASET_COLUMNS if column.endswith("_acc") ] init_leaderboard( LEADERBOARD_DF, BY_DOMAIN_COLUMNS, default_columns=domain_default_columns, ) with gr.TabItem("📤 Submit Results", elem_id="tab-submit", id=2): gr.Markdown( ( "Upload submission files for manual review.\n\n" "Required files:\n" "1. `results_on_dev_dataset.json`: task-level metrics in leaderboard format.\n" "2. `results_on_test_dataset.json`: per-example test outputs with at least " "`id`, `tier`, `source_dataset`, `label`, and `output` " "(required when the sample contains forecasting).\n\n" "Please also include model architecture code and LLM/system details for verification." ), elem_classes="markdown-text", ) gr.Markdown(EXAMPLE_RECORD_MD, elem_classes="markdown-text") submission_file = gr.File( label="Submission package (.zip or .rar)", file_types=[".zip", ".rar"], ) submit_button = gr.Button("Submit for Review") submission_status = gr.Markdown() submit_button.click(save_submission, [submission_file], submission_status) with gr.TabItem("📝 About", elem_id="tab-about", id=3): gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text") gr.Markdown(f"## Citation\n{CITATION_BUTTON_LABEL}", elem_classes="markdown-text") gr.Markdown(f"```bibtex\n{CITATION_BUTTON_TEXT.strip()}\n```", elem_classes="markdown-text") # Citation section hidden for now. # with gr.Row(): # with gr.Accordion("📙 Citation", open=False): # citation_button = gr.Textbox( # value=CITATION_BUTTON_TEXT, # label=CITATION_BUTTON_LABEL, # lines=20, # elem_id="citation-button", # show_copy_button=True, # ) demo.launch()