Grounding Tools

Agent: "click the red submit button"
         ↓
    [Screenshot]
         ↓
    [Vision Model: (450, 320)]
         ↓
    Computer: click(x=450, y=320)

from hud.tools.grounding import GroundedComputerTool, Grounder, GrounderConfig

config = GrounderConfig(
    api_base="https://api.openai.com/v1",
    model="gpt-4o",
    api_key="your-api-key",
)
grounder = Grounder(config=config)

grounded = GroundedComputerTool(
    grounder=grounder,
    ctx=env,                        # Environment context
    computer_tool_name="computer",  # Name of computer tool to use
)

# Click using description
await grounded(
    action="click",
    element_description="the blue login button at the top",
    screenshot_b64=current_screenshot,
)

# Scroll at element
await grounded(
    action="scroll",
    element_description="the main content area",
    scroll_x=0,
    scroll_y=-100,
    screenshot_b64=current_screenshot,
)

# Drag between elements
await grounded(
    action="drag",
    start_element_description="the file icon",
    end_element_description="the trash folder",
    screenshot_b64=current_screenshot,
)

# No grounding needed for these
await grounded(action="type", text="Hello!")
await grounded(action="keypress", keys=["ctrl", "s"])

from hud.tools.grounding import Grounder, GrounderConfig

# Basic config
config = GrounderConfig(
    api_base="https://api.openai.com/v1",
    model="gpt-4o",
)
grounder = Grounder(config=config)

# With custom settings
config = GrounderConfig(
    api_base="https://openrouter.ai/api/v1",
    model="qwen/qwen-2.5-vl-7b-instruct",
    api_key="your-openrouter-key",
    output_format="pixels",
)
grounder = Grounder(config=config)

coords = await grounder.predict_click(
    image_b64=screenshot_base64,
    instruction="the submit button",
)
# Returns: (x, y) or None if not found

from hud import Environment
from hud.tools import AnthropicComputerTool
from hud.tools.grounding import GroundedComputerTool, Grounder, GrounderConfig

# Setup environment with computer tool
env = Environment("grounded-env")
env.add_tool(AnthropicComputerTool())

# Create grounder
config = GrounderConfig(
    api_base="https://api.openai.com/v1",
    model="gpt-4o",
    api_key="your-api-key",
)
grounder = Grounder(config=config)

async with env:
    # Wrap environment for grounded calls
    grounded = GroundedComputerTool(grounder=grounder, ctx=env)
    
    # Take screenshot via environment
    result = await env.call_tool("computer", action="screenshot")
    
    # Use grounded tool for element-based actions
    await grounded(
        action="click",
        element_description="the login button",
        screenshot_b64=result.content[0].data,  # base64 from screenshot
    )

from hud.agents import create_agent
import hud

task = env("my_task")
agent = create_agent("gpt-4o")

async with hud.eval(task) as ctx:
    await agent.run(ctx)