Ray0202
Add TimeCopilot v4-pro results, version toggle, and paper links
72dff92
Raw History Blame Contribute Delete
14 kB
from datetime import datetime, timezone
import json
import os
import re
import shutil
from typing import Optional
import gradio as gr
import pandas as pd
from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns
from src.about import (
CITATION_BUTTON_LABEL,
CITATION_BUTTON_TEXT,
INTRODUCTION_TEXT,
LLM_BENCHMARKS_TEXT,
TITLE,
)
from src.display.css_html_js import custom_css
from src.envs import RESULTS_PATH, SUBMISSIONS_PATH
from src.leaderboard.load_results import (
DISPLAY_METADATA_COLUMNS,
V1_0_DOMAINS,
V1_1_DOMAINS,
ResultsValidationError,
build_dataframe,
load_records,
)
from src.leaderboard.schema import SCHEMA
V1_1_LABEL = "V1.1 \u2014 5 datasets (adds M5)"
V1_0_LABEL = "V1.0 \u2014 original 4 datasets"
VERSION_DOMAINS = {
V1_1_LABEL: V1_1_DOMAINS,
V1_0_LABEL: V1_0_DOMAINS,
}
RANKING_METRIC = "overall_t1_t3_t4_acc"
def load_leaderboard_data() -> tuple[dict[str, tuple[pd.DataFrame, list[str]]], Optional[str]]:
"""Build one ranked view per benchmark version from the same result records."""
try:
records = load_records(RESULTS_PATH)
views = {}
for label, domains in VERSION_DOMAINS.items():
df, column_order = build_dataframe(records, domains=domains)
if RANKING_METRIC in df.columns:
df = df.sort_values(RANKING_METRIC, ascending=False, na_position="last")
views[label] = (df, column_order)
return views, None
except ResultsValidationError as exc:
fallback_cols = list(SCHEMA.identity_fields) + list(SCHEMA.required_metrics)
empty = pd.DataFrame(columns=fallback_cols)
return {label: (empty, fallback_cols) for label in VERSION_DOMAINS}, str(exc)
VERSION_VIEWS, LOAD_ERROR = load_leaderboard_data()
# Unfiltered five-domain view: the By Domain tab reports per-dataset results for every
# entry, including those evaluated only on the four V1.0 domains.
LEADERBOARD_DF, COLUMN_ORDER = VERSION_VIEWS[V1_1_LABEL]
DATASET_PREFIX_MAP = {
"FreshRetailNet": "FreshRetailNet",
"PSML": "PSML",
"Causal Chambers": "CausalChambers",
"MIMIC": "MIMIC",
"M5": "M5",
}
DATASET_PREFIXES = [f"{prefix}_" for prefix in DATASET_PREFIX_MAP.values()]
def is_dataset_metric(column: str) -> bool:
return any(column.startswith(prefix) for prefix in DATASET_PREFIXES)
BASE_COLUMNS = list(SCHEMA.identity_fields) + list(DISPLAY_METADATA_COLUMNS) + list(SCHEMA.required_metrics)
ALL_DATASET_COLUMNS = [c for c in COLUMN_ORDER if is_dataset_metric(c)]
M5_COLUMNS = [column for column in ALL_DATASET_COLUMNS if column.startswith("M5_")]
NON_M5_DATASET_COLUMNS = [
column for column in ALL_DATASET_COLUMNS if column not in M5_COLUMNS
]
AGGREGATE_FORECAST_COLUMNS = [
"overall_t1_t3_t4_acc",
"overall_mcq_acc",
"T2_MAE",
"T2_sMAPE",
"T4_MAE",
"T4_sMAPE",
"MIMIC_T2_OW_sMAPE",
"MIMIC_T2_OW_RMSSE",
"MIMIC_T4_OW_sMAPE",
"MIMIC_T4_OW_RMSSE",
]
def aggregate_columns(column_order: list[str]) -> list[str]:
return BASE_COLUMNS + [c for c in AGGREGATE_FORECAST_COLUMNS if c in column_order]
def complete_entries(df: pd.DataFrame) -> pd.DataFrame:
"""Entries evaluated on every domain of the selected version.
A version's leaderboard ranks only entries measured on its full domain set, so
V1.0-only entries are absent from V1.1. V1.0 stays backward compatible: it lists
those entries together with V1.1 entries reduced to the same four domains.
"""
if "benchmark_complete" not in df.columns:
return df
return df[df["benchmark_complete"]]
BY_DOMAIN_COLUMNS = BASE_COLUMNS + ALL_DATASET_COLUMNS
DISPLAY_LABELS = {
"agent_name": "Agent",
"agent_type": "Agent Type",
"base_model": "Base Model",
"paper": "Paper",
"evaluation_scope": "Evaluation Scope",
"domain_coverage": "Domain Coverage",
"benchmark_complete": "All Domains Evaluated",
"overall_mcq_acc": "Overall MCQ Acc",
"overall_t1_t3_t4_acc": "Overall T1/T3/T4 Acc",
"T1_acc": "Overall T1 Acc",
"T2_acc": "Overall T2 Acc",
"T3_acc": "Overall T3 Acc",
"T4_acc": "Overall T4 Acc",
"T2_MAE": "Overall T2 MAE",
"T2_sMAPE": "Overall T2 sMAPE",
"T4_MAE": "Overall T4 MAE",
"T4_sMAPE": "Overall T4 sMAPE",
"MIMIC_T2_OW_sMAPE": "MIMIC T2 OW-sMAPE",
"MIMIC_T2_OW_RMSSE": "MIMIC T2 OW-RMSSE",
"MIMIC_T4_OW_sMAPE": "MIMIC T4 OW-sMAPE",
"MIMIC_T4_OW_RMSSE": "MIMIC T4 OW-RMSSE",
}
def display_label(column: str) -> str:
if column in DISPLAY_LABELS:
return DISPLAY_LABELS[column]
match = re.fullmatch(r"(.+)_T([1-4])_(acc|MAE|sMAPE|OW_sMAPE|OW_RMSSE)", column)
if match is None:
return column
domain, tier, metric = match.groups()
domain = "Causal Chambers" if domain == "CausalChambers" else domain
return f"{domain} T{tier} {metric.replace('_', '-')}"
def column_types(column_order: list[str]) -> list[str]:
types = []
for col in column_order:
if col in SCHEMA.identity_fields or col in {"evaluation_scope", "domain_coverage"}:
types.append("str")
elif col == "paper":
types.append("markdown")
elif col == "benchmark_complete":
types.append("bool")
else:
types.append("number")
return types
def init_leaderboard(dataframe, column_order, default_columns=None):
if dataframe is None or dataframe.empty:
dataframe = pd.DataFrame(columns=column_order)
dataframe = dataframe.reindex(columns=column_order)
display_columns = [display_label(column) for column in column_order]
dataframe = dataframe.rename(columns=dict(zip(column_order, display_columns)))
required_cols = list(SCHEMA.identity_fields) + ["evaluation_scope", "domain_coverage"]
cant_deselect = [display_label(c) for c in required_cols if c in column_order]
default_columns = default_columns or column_order
default_selection = [display_label(c) for c in default_columns if c in column_order]
search_columns = [
display_label(c)
for c in ["agent_name", "base_model"]
if c in column_order
]
return Leaderboard(
value=dataframe,
datatype=column_types(column_order),
select_columns=SelectColumns(
default_selection=default_selection,
cant_deselect=cant_deselect,
label="Select Columns to Display:",
),
search_columns=search_columns,
filter_columns=[
ColumnFilter(display_label("agent_type"), type="checkboxgroup", label="Agent Type"),
],
interactive=False,
)
def select_version(choice: str):
"""Show the panel for the selected benchmark version, hide the other."""
is_v1_1 = choice == V1_1_LABEL
return gr.update(visible=is_v1_1), gr.update(visible=not is_v1_1)
def save_submission(uploaded_file) -> str:
if uploaded_file is None:
return "Please upload a results file."
file_path = uploaded_file.name if hasattr(uploaded_file, "name") else str(uploaded_file)
extension = os.path.splitext(file_path)[1].lower()
if extension not in {".zip", ".rar"}:
return "Please upload a .zip or .rar submission package."
os.makedirs(SUBMISSIONS_PATH, exist_ok=True)
timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
out_path = os.path.join(SUBMISSIONS_PATH, f"submission_{timestamp}{extension}")
shutil.copyfile(file_path, out_path)
return "Submission received for manual review."
def build_example_record_markdown() -> str:
try:
records = load_records(RESULTS_PATH)
if not records:
return "No example data available."
example = records[0]
return "Example record (JSON):\n```json\n" + json.dumps(example, indent=2) + "\n```"
except Exception as exc:
return f"Could not load example record: {exc}"
EXAMPLE_RECORD_MD = build_example_record_markdown()
demo = gr.Blocks(css=custom_css, analytics_enabled=False)
with demo:
gr.HTML(TITLE)
gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
if LOAD_ERROR:
gr.Markdown(f"**Data validation error:** {LOAD_ERROR}", elem_classes="markdown-text")
with gr.Tabs(elem_classes="tab-buttons") as tabs:
with gr.TabItem("πŸ… Leaderboard", elem_id="tab-leaderboard", id=0):
version_selector = gr.Radio(
choices=[V1_1_LABEL, V1_0_LABEL],
value=V1_1_LABEL,
label="Benchmark version",
info=(
"V1.0 restricts every entry to the four original domains and re-ranks "
"them, so all entries are compared on the same domain set."
),
)
v1_1_df, v1_1_columns = VERSION_VIEWS[V1_1_LABEL]
v1_0_df, v1_0_columns = VERSION_VIEWS[V1_0_LABEL]
v1_1_df = complete_entries(v1_1_df)
v1_0_df = complete_entries(v1_0_df)
with gr.Column(visible=True) as v1_1_panel:
gr.Markdown(
f"**Current entries:** {len(v1_1_df)} agent configurations evaluated on all "
"five V1.1 datasets. Entries measured only on the four V1.0 domains are not "
"listed here β€” switch to **V1.0** to see them alongside these entries reduced "
"to the same four domains. Rows are ranked by **Overall T1/T3/T4 Acc**, "
"weighted by the corresponding MCQ question counts.\n\n"
"⚠️ M5 was evaluated in a later round than the other four domains, after the "
"DeepSeek API used for the original baselines was withdrawn. For the "
"**AgentScope**, **CAMEL**, **MetaGPT** and **TimeSeries Scientist** "
"`deepseek-chat` rows the overall score therefore averages across two API "
"versions, and small gaps between them are not meaningful. See the **About** "
"tab for details."
)
init_leaderboard(v1_1_df, aggregate_columns(v1_1_columns))
with gr.Column(visible=False) as v1_0_panel:
gr.Markdown(
f"**Current entries:** {len(v1_0_df)} agent configurations on the four V1.0 "
"domains (FreshRetailNet, PSML, Causal Chambers, MIMIC). This view is backward "
"compatible: it lists V1.0-only entries together with V1.1 entries reduced to "
"their four-domain subset, with **Overall T1/T3/T4 Acc** recomputed from those "
"domains only and the ranking recalculated β€” so every row is compared on the "
"same domain set.\n\n"
"Because M5 is excluded, this view is free of the M5 API-version seam "
"described in the **About** tab."
)
init_leaderboard(v1_0_df, aggregate_columns(v1_0_columns))
version_selector.change(
select_version,
version_selector,
[v1_1_panel, v1_0_panel],
)
with gr.TabItem("🧭 By Domain", elem_id="tab-by-domain", id=1):
gr.Markdown(
f"Per-dataset results for all {len(LEADERBOARD_DF)} entries, including those "
"evaluated only on the four V1.0 domains β€” **Domain Coverage** marks which is "
"which. The newest evaluation domain, **M5**, is shown first. Use **Select "
"Columns to Display** to add forecasting metrics or other domains."
)
domain_default_columns = BASE_COLUMNS + M5_COLUMNS + [
column for column in NON_M5_DATASET_COLUMNS if column.endswith("_acc")
]
init_leaderboard(
LEADERBOARD_DF,
BY_DOMAIN_COLUMNS,
default_columns=domain_default_columns,
)
with gr.TabItem("πŸ“€ Submit Results", elem_id="tab-submit", id=2):
gr.Markdown(
(
"Upload submission files for manual review.\n\n"
"Required files:\n"
"1. `results_on_dev_dataset.json`: task-level metrics in leaderboard format.\n"
"2. `results_on_test_dataset.json`: per-example test outputs with at least "
"`id`, `tier`, `source_dataset`, `label`, and `output` "
"(required when the sample contains forecasting).\n\n"
"Please also include model architecture code and LLM/system details for verification."
),
elem_classes="markdown-text",
)
gr.Markdown(EXAMPLE_RECORD_MD, elem_classes="markdown-text")
submission_file = gr.File(
label="Submission package (.zip or .rar)",
file_types=[".zip", ".rar"],
)
submit_button = gr.Button("Submit for Review")
submission_status = gr.Markdown()
submit_button.click(save_submission, [submission_file], submission_status)
with gr.TabItem("πŸ“ About", elem_id="tab-about", id=3):
gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
gr.Markdown(f"## Citation\n{CITATION_BUTTON_LABEL}", elem_classes="markdown-text")
gr.Markdown(f"```bibtex\n{CITATION_BUTTON_TEXT.strip()}\n```", elem_classes="markdown-text")
# Citation section hidden for now.
# with gr.Row():
# with gr.Accordion("πŸ“™ Citation", open=False):
# citation_button = gr.Textbox(
# value=CITATION_BUTTON_TEXT,
# label=CITATION_BUTTON_LABEL,
# lines=20,
# elem_id="citation-button",
# show_copy_button=True,
# )
demo.launch()