Converting Harbor Benchmarks

# 1. Clone the benchmark
git clone https://github.com/laude-institute/terminal-bench-2.git

# 2. Convert to HUD format
hud convert ./terminal-bench-2/ --output ./tb2-hud

# 3. Deploy all environments (~3 min per environment, leave it running)
hud deploy ./tb2-hud --all

# 4. Run evaluation
hud eval ./tb2-hud/taskset.json

task-name/
├── task.toml              # Config (timeout, metadata)
├── instruction.md         # Agent prompt
├── environment/
│   ├── Dockerfile         # Container setup
│   └── (build context)    # Any files the Dockerfile references
├── tests/
│   └── test.sh            # Verification script
└── solution/              # Ignored

hud-harbor-dataset/
├── env.py                 # MCP environment with run-task scenario
├── Dockerfile.hud         # Harbor Dockerfile + HUD MCP layer
├── pyproject.toml         # Dependencies
├── (build context files)  # Copied from environment/
└── tasks/
    └── task-name/
        ├── instruction.md
        ├── task.toml
        └── tests/test.sh

hud eval ./tb2-hud/taskset.json

import asyncio
import hud
from hud.agents.claude import ClaudeAgent
from hud.eval.task import Task

async def main():
    task = Task(
        env={"name": "hud-harbor-terminal-bench-2-sample-g1"},
        scenario="hud-harbor-terminal-bench-2-sample-g1:run-task",
        args={"task_id": "build-pmars"},
    )

    agent = ClaudeAgent.create(model="claude-sonnet-4-20250514")

    async with hud.eval(task, name="harbor-demo") as ctx:
        result = await agent.run(ctx, max_steps=30)

    print(f"Reward: {ctx.reward}")

asyncio.run(main())

import json
from pathlib import Path

from hud.eval.task import Task

taskset = json.loads(Path("./tb2-hud/taskset.json").read_text())
tasks = [Task(**t) for t in taskset]