Download app.py from Melady/TemporalBench_Leaderboard: direct link, hf CLI and curl.
- Browser
- Download file 14 kB
-
https://huggingface.co/spaces/Melady/TemporalBench_Leaderboard/resolve/main/app.py
- Command line
-
hf download hf://spaces/Melady/TemporalBench_Leaderboard/app.py
-
curl -L -o app.py https://huggingface.co/spaces/Melady/TemporalBench_Leaderboard/resolve/main/app.py
14 kB
| from datetime import datetime, timezone | |
| import json | |
| import os | |
| import re | |
| import shutil | |
| from typing import Optional | |
| import gradio as gr | |
| import pandas as pd | |
| from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns | |
| from src.about import ( | |
| CITATION_BUTTON_LABEL, | |
| CITATION_BUTTON_TEXT, | |
| INTRODUCTION_TEXT, | |
| LLM_BENCHMARKS_TEXT, | |
| TITLE, | |
| ) | |
| from src.display.css_html_js import custom_css | |
| from src.envs import RESULTS_PATH, SUBMISSIONS_PATH | |
| from src.leaderboard.load_results import ( | |
| DISPLAY_METADATA_COLUMNS, | |
| V1_0_DOMAINS, | |
| V1_1_DOMAINS, | |
| ResultsValidationError, | |
| build_dataframe, | |
| load_records, | |
| ) | |
| from src.leaderboard.schema import SCHEMA | |
| V1_1_LABEL = "V1.1 \u2014 5 datasets (adds M5)" | |
| V1_0_LABEL = "V1.0 \u2014 original 4 datasets" | |
| VERSION_DOMAINS = { | |
| V1_1_LABEL: V1_1_DOMAINS, | |
| V1_0_LABEL: V1_0_DOMAINS, | |
| } | |
| RANKING_METRIC = "overall_t1_t3_t4_acc" | |
| def load_leaderboard_data() -> tuple[dict[str, tuple[pd.DataFrame, list[str]]], Optional[str]]: | |
| """Build one ranked view per benchmark version from the same result records.""" | |
| try: | |
| records = load_records(RESULTS_PATH) | |
| views = {} | |
| for label, domains in VERSION_DOMAINS.items(): | |
| df, column_order = build_dataframe(records, domains=domains) | |
| if RANKING_METRIC in df.columns: | |
| df = df.sort_values(RANKING_METRIC, ascending=False, na_position="last") | |
| views[label] = (df, column_order) | |
| return views, None | |
| except ResultsValidationError as exc: | |
| fallback_cols = list(SCHEMA.identity_fields) + list(SCHEMA.required_metrics) | |
| empty = pd.DataFrame(columns=fallback_cols) | |
| return {label: (empty, fallback_cols) for label in VERSION_DOMAINS}, str(exc) | |
| VERSION_VIEWS, LOAD_ERROR = load_leaderboard_data() | |
| # Unfiltered five-domain view: the By Domain tab reports per-dataset results for every | |
| # entry, including those evaluated only on the four V1.0 domains. | |
| LEADERBOARD_DF, COLUMN_ORDER = VERSION_VIEWS[V1_1_LABEL] | |
| DATASET_PREFIX_MAP = { | |
| "FreshRetailNet": "FreshRetailNet", | |
| "PSML": "PSML", | |
| "Causal Chambers": "CausalChambers", | |
| "MIMIC": "MIMIC", | |
| "M5": "M5", | |
| } | |
| DATASET_PREFIXES = [f"{prefix}_" for prefix in DATASET_PREFIX_MAP.values()] | |
| def is_dataset_metric(column: str) -> bool: | |
| return any(column.startswith(prefix) for prefix in DATASET_PREFIXES) | |
| BASE_COLUMNS = list(SCHEMA.identity_fields) + list(DISPLAY_METADATA_COLUMNS) + list(SCHEMA.required_metrics) | |
| ALL_DATASET_COLUMNS = [c for c in COLUMN_ORDER if is_dataset_metric(c)] | |
| M5_COLUMNS = [column for column in ALL_DATASET_COLUMNS if column.startswith("M5_")] | |
| NON_M5_DATASET_COLUMNS = [ | |
| column for column in ALL_DATASET_COLUMNS if column not in M5_COLUMNS | |
| ] | |
| AGGREGATE_FORECAST_COLUMNS = [ | |
| "overall_t1_t3_t4_acc", | |
| "overall_mcq_acc", | |
| "T2_MAE", | |
| "T2_sMAPE", | |
| "T4_MAE", | |
| "T4_sMAPE", | |
| "MIMIC_T2_OW_sMAPE", | |
| "MIMIC_T2_OW_RMSSE", | |
| "MIMIC_T4_OW_sMAPE", | |
| "MIMIC_T4_OW_RMSSE", | |
| ] | |
| def aggregate_columns(column_order: list[str]) -> list[str]: | |
| return BASE_COLUMNS + [c for c in AGGREGATE_FORECAST_COLUMNS if c in column_order] | |
| def complete_entries(df: pd.DataFrame) -> pd.DataFrame: | |
| """Entries evaluated on every domain of the selected version. | |
| A version's leaderboard ranks only entries measured on its full domain set, so | |
| V1.0-only entries are absent from V1.1. V1.0 stays backward compatible: it lists | |
| those entries together with V1.1 entries reduced to the same four domains. | |
| """ | |
| if "benchmark_complete" not in df.columns: | |
| return df | |
| return df[df["benchmark_complete"]] | |
| BY_DOMAIN_COLUMNS = BASE_COLUMNS + ALL_DATASET_COLUMNS | |
| DISPLAY_LABELS = { | |
| "agent_name": "Agent", | |
| "agent_type": "Agent Type", | |
| "base_model": "Base Model", | |
| "paper": "Paper", | |
| "evaluation_scope": "Evaluation Scope", | |
| "domain_coverage": "Domain Coverage", | |
| "benchmark_complete": "All Domains Evaluated", | |
| "overall_mcq_acc": "Overall MCQ Acc", | |
| "overall_t1_t3_t4_acc": "Overall T1/T3/T4 Acc", | |
| "T1_acc": "Overall T1 Acc", | |
| "T2_acc": "Overall T2 Acc", | |
| "T3_acc": "Overall T3 Acc", | |
| "T4_acc": "Overall T4 Acc", | |
| "T2_MAE": "Overall T2 MAE", | |
| "T2_sMAPE": "Overall T2 sMAPE", | |
| "T4_MAE": "Overall T4 MAE", | |
| "T4_sMAPE": "Overall T4 sMAPE", | |
| "MIMIC_T2_OW_sMAPE": "MIMIC T2 OW-sMAPE", | |
| "MIMIC_T2_OW_RMSSE": "MIMIC T2 OW-RMSSE", | |
| "MIMIC_T4_OW_sMAPE": "MIMIC T4 OW-sMAPE", | |
| "MIMIC_T4_OW_RMSSE": "MIMIC T4 OW-RMSSE", | |
| } | |
| def display_label(column: str) -> str: | |
| if column in DISPLAY_LABELS: | |
| return DISPLAY_LABELS[column] | |
| match = re.fullmatch(r"(.+)_T([1-4])_(acc|MAE|sMAPE|OW_sMAPE|OW_RMSSE)", column) | |
| if match is None: | |
| return column | |
| domain, tier, metric = match.groups() | |
| domain = "Causal Chambers" if domain == "CausalChambers" else domain | |
| return f"{domain} T{tier} {metric.replace('_', '-')}" | |
| def column_types(column_order: list[str]) -> list[str]: | |
| types = [] | |
| for col in column_order: | |
| if col in SCHEMA.identity_fields or col in {"evaluation_scope", "domain_coverage"}: | |
| types.append("str") | |
| elif col == "paper": | |
| types.append("markdown") | |
| elif col == "benchmark_complete": | |
| types.append("bool") | |
| else: | |
| types.append("number") | |
| return types | |
| def init_leaderboard(dataframe, column_order, default_columns=None): | |
| if dataframe is None or dataframe.empty: | |
| dataframe = pd.DataFrame(columns=column_order) | |
| dataframe = dataframe.reindex(columns=column_order) | |
| display_columns = [display_label(column) for column in column_order] | |
| dataframe = dataframe.rename(columns=dict(zip(column_order, display_columns))) | |
| required_cols = list(SCHEMA.identity_fields) + ["evaluation_scope", "domain_coverage"] | |
| cant_deselect = [display_label(c) for c in required_cols if c in column_order] | |
| default_columns = default_columns or column_order | |
| default_selection = [display_label(c) for c in default_columns if c in column_order] | |
| search_columns = [ | |
| display_label(c) | |
| for c in ["agent_name", "base_model"] | |
| if c in column_order | |
| ] | |
| return Leaderboard( | |
| value=dataframe, | |
| datatype=column_types(column_order), | |
| select_columns=SelectColumns( | |
| default_selection=default_selection, | |
| cant_deselect=cant_deselect, | |
| label="Select Columns to Display:", | |
| ), | |
| search_columns=search_columns, | |
| filter_columns=[ | |
| ColumnFilter(display_label("agent_type"), type="checkboxgroup", label="Agent Type"), | |
| ], | |
| interactive=False, | |
| ) | |
| def select_version(choice: str): | |
| """Show the panel for the selected benchmark version, hide the other.""" | |
| is_v1_1 = choice == V1_1_LABEL | |
| return gr.update(visible=is_v1_1), gr.update(visible=not is_v1_1) | |
| def save_submission(uploaded_file) -> str: | |
| if uploaded_file is None: | |
| return "Please upload a results file." | |
| file_path = uploaded_file.name if hasattr(uploaded_file, "name") else str(uploaded_file) | |
| extension = os.path.splitext(file_path)[1].lower() | |
| if extension not in {".zip", ".rar"}: | |
| return "Please upload a .zip or .rar submission package." | |
| os.makedirs(SUBMISSIONS_PATH, exist_ok=True) | |
| timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") | |
| out_path = os.path.join(SUBMISSIONS_PATH, f"submission_{timestamp}{extension}") | |
| shutil.copyfile(file_path, out_path) | |
| return "Submission received for manual review." | |
| def build_example_record_markdown() -> str: | |
| try: | |
| records = load_records(RESULTS_PATH) | |
| if not records: | |
| return "No example data available." | |
| example = records[0] | |
| return "Example record (JSON):\n```json\n" + json.dumps(example, indent=2) + "\n```" | |
| except Exception as exc: | |
| return f"Could not load example record: {exc}" | |
| EXAMPLE_RECORD_MD = build_example_record_markdown() | |
| demo = gr.Blocks(css=custom_css, analytics_enabled=False) | |
| with demo: | |
| gr.HTML(TITLE) | |
| gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") | |
| if LOAD_ERROR: | |
| gr.Markdown(f"**Data validation error:** {LOAD_ERROR}", elem_classes="markdown-text") | |
| with gr.Tabs(elem_classes="tab-buttons") as tabs: | |
| with gr.TabItem("π Leaderboard", elem_id="tab-leaderboard", id=0): | |
| version_selector = gr.Radio( | |
| choices=[V1_1_LABEL, V1_0_LABEL], | |
| value=V1_1_LABEL, | |
| label="Benchmark version", | |
| info=( | |
| "V1.0 restricts every entry to the four original domains and re-ranks " | |
| "them, so all entries are compared on the same domain set." | |
| ), | |
| ) | |
| v1_1_df, v1_1_columns = VERSION_VIEWS[V1_1_LABEL] | |
| v1_0_df, v1_0_columns = VERSION_VIEWS[V1_0_LABEL] | |
| v1_1_df = complete_entries(v1_1_df) | |
| v1_0_df = complete_entries(v1_0_df) | |
| with gr.Column(visible=True) as v1_1_panel: | |
| gr.Markdown( | |
| f"**Current entries:** {len(v1_1_df)} agent configurations evaluated on all " | |
| "five V1.1 datasets. Entries measured only on the four V1.0 domains are not " | |
| "listed here β switch to **V1.0** to see them alongside these entries reduced " | |
| "to the same four domains. Rows are ranked by **Overall T1/T3/T4 Acc**, " | |
| "weighted by the corresponding MCQ question counts.\n\n" | |
| "β οΈ M5 was evaluated in a later round than the other four domains, after the " | |
| "DeepSeek API used for the original baselines was withdrawn. For the " | |
| "**AgentScope**, **CAMEL**, **MetaGPT** and **TimeSeries Scientist** " | |
| "`deepseek-chat` rows the overall score therefore averages across two API " | |
| "versions, and small gaps between them are not meaningful. See the **About** " | |
| "tab for details." | |
| ) | |
| init_leaderboard(v1_1_df, aggregate_columns(v1_1_columns)) | |
| with gr.Column(visible=False) as v1_0_panel: | |
| gr.Markdown( | |
| f"**Current entries:** {len(v1_0_df)} agent configurations on the four V1.0 " | |
| "domains (FreshRetailNet, PSML, Causal Chambers, MIMIC). This view is backward " | |
| "compatible: it lists V1.0-only entries together with V1.1 entries reduced to " | |
| "their four-domain subset, with **Overall T1/T3/T4 Acc** recomputed from those " | |
| "domains only and the ranking recalculated β so every row is compared on the " | |
| "same domain set.\n\n" | |
| "Because M5 is excluded, this view is free of the M5 API-version seam " | |
| "described in the **About** tab." | |
| ) | |
| init_leaderboard(v1_0_df, aggregate_columns(v1_0_columns)) | |
| version_selector.change( | |
| select_version, | |
| version_selector, | |
| [v1_1_panel, v1_0_panel], | |
| ) | |
| with gr.TabItem("π§ By Domain", elem_id="tab-by-domain", id=1): | |
| gr.Markdown( | |
| f"Per-dataset results for all {len(LEADERBOARD_DF)} entries, including those " | |
| "evaluated only on the four V1.0 domains β **Domain Coverage** marks which is " | |
| "which. The newest evaluation domain, **M5**, is shown first. Use **Select " | |
| "Columns to Display** to add forecasting metrics or other domains." | |
| ) | |
| domain_default_columns = BASE_COLUMNS + M5_COLUMNS + [ | |
| column for column in NON_M5_DATASET_COLUMNS if column.endswith("_acc") | |
| ] | |
| init_leaderboard( | |
| LEADERBOARD_DF, | |
| BY_DOMAIN_COLUMNS, | |
| default_columns=domain_default_columns, | |
| ) | |
| with gr.TabItem("π€ Submit Results", elem_id="tab-submit", id=2): | |
| gr.Markdown( | |
| ( | |
| "Upload submission files for manual review.\n\n" | |
| "Required files:\n" | |
| "1. `results_on_dev_dataset.json`: task-level metrics in leaderboard format.\n" | |
| "2. `results_on_test_dataset.json`: per-example test outputs with at least " | |
| "`id`, `tier`, `source_dataset`, `label`, and `output` " | |
| "(required when the sample contains forecasting).\n\n" | |
| "Please also include model architecture code and LLM/system details for verification." | |
| ), | |
| elem_classes="markdown-text", | |
| ) | |
| gr.Markdown(EXAMPLE_RECORD_MD, elem_classes="markdown-text") | |
| submission_file = gr.File( | |
| label="Submission package (.zip or .rar)", | |
| file_types=[".zip", ".rar"], | |
| ) | |
| submit_button = gr.Button("Submit for Review") | |
| submission_status = gr.Markdown() | |
| submit_button.click(save_submission, [submission_file], submission_status) | |
| with gr.TabItem("π About", elem_id="tab-about", id=3): | |
| gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text") | |
| gr.Markdown(f"## Citation\n{CITATION_BUTTON_LABEL}", elem_classes="markdown-text") | |
| gr.Markdown(f"```bibtex\n{CITATION_BUTTON_TEXT.strip()}\n```", elem_classes="markdown-text") | |
| # Citation section hidden for now. | |
| # with gr.Row(): | |
| # with gr.Accordion("π Citation", open=False): | |
| # citation_button = gr.Textbox( | |
| # value=CITATION_BUTTON_TEXT, | |
| # label=CITATION_BUTTON_LABEL, | |
| # lines=20, | |
| # elem_id="citation-button", | |
| # show_copy_button=True, | |
| # ) | |
| demo.launch() | |