Benchmarks

# Run a local tasks file (interactive agent selection)
hud eval tasks.json

# Run a hosted dataset with Claude
hud eval hud-evals/SheetBench-50 claude --full

from hud.agents import ClaudeAgent
from hud.datasets import run_dataset

results = await run_dataset(
    name="SheetBench Eval",
    dataset="hud-evals/SheetBench-50",
    agent_class=ClaudeAgent,
    max_concurrent=50,
)

hud analyze hudpython/hud-remote-browser:latest

import uuid

web_tasks = []
web_tasks.append({
    "id": str(uuid.uuid4()),
    "prompt": "Navigate to the documentation page",
    "mcp_config": {
        "hud": {
            "url": "https://mcp.hud.ai/v3/mcp",
            "headers": {
                "Authorization": "Bearer ${HUD_API_KEY}",
                "Mcp-Image": "hudpython/hud-remote-browser:latest"
            }
        }
    },
    "setup_tool": {"name": "setup", "arguments": {"name": "navigate", "arguments": {"url": "https://example.com"}}},
    "evaluate_tool": {"name": "evaluate", "arguments": {"name": "url_match", "arguments": {"pattern": ".*/docs.*"}}},
    "metadata": {"difficulty": "easy", "category": "navigation"}
})

from hud.datasets import save_tasks

save_tasks(
  web_tasks,
  repo_id="my-org/web-navigation-benchmark",
  private=False,
  tags=["web", "navigation", "automation"],
)

from hud.datasets import run_dataset
from hud.agents import ClaudeAgent

results = await run_dataset(
    name="Claude Sonnet SheetBench",
    dataset="hud-evals/SheetBench-50",
    agent_class=ClaudeAgent,
)

# Open https://hud.ai/leaderboards/hud-evals/SheetBench-50
# Click "My Jobs" to see runs and create a scorecard