# Closed-source cloud models × NativeOpenHands (CodeAct) agent on 5 use-case benchmarks.
#
# Models: Claude Opus 4.6, Claude Sonnet 4.6, GPT 5.4, GPT 5 Mini,
#          Gemini 3.1 Pro, Gemini 3.1 Flash
# Agent: NativeOpenHandsAgent (code-execution-centric, Action/Observation loop)
#
# Prerequisites: source .env (OPENAI_API_KEY, ANTHROPIC_API_KEY, GOOGLE_API_KEY)

[meta]
name = "use-case-cloud-openhands"
description = "Cloud models with NativeOpenHandsAgent (CodeAct) on use-case benchmarks"

[defaults]
temperature = 0.0
max_tokens = 2048

[judge]
model = "gpt-5-mini-2025-08-07"

[run]
max_workers = 4
output_dir = "results/use-cases-cloud-openhands/"
seed = 42
telemetry = true

# --- Models Under Test ---

[[models]]
name = "claude-opus-4-6"
engine = "cloud"
provider = "anthropic"

[[models]]
name = "claude-sonnet-4-6"
engine = "cloud"
provider = "anthropic"

[[models]]
name = "gpt-5.4"
engine = "cloud"
provider = "openai"

[[models]]
name = "gpt-5-mini"
engine = "cloud"
provider = "openai"

[[models]]
name = "gemini-3.1-pro-preview"
engine = "cloud"
provider = "google"

[[models]]
name = "gemini-3-flash-preview"
engine = "cloud"
provider = "google"

# --- Benchmarks ---

[[benchmarks]]
name = "email_triage"
backend = "jarvis-agent"
agent = "native_openhands"
tools = ["think"]
max_samples = 30

[[benchmarks]]
name = "morning_brief"
backend = "jarvis-agent"
agent = "native_openhands"
tools = ["think", "web_search"]
max_samples = 15

[[benchmarks]]
name = "research_mining"
backend = "jarvis-agent"
agent = "native_openhands"
tools = ["think", "web_search"]
max_samples = 30

[[benchmarks]]
name = "knowledge_base"
backend = "jarvis-agent"
agent = "native_openhands"
tools = ["think"]
max_samples = 30

[[benchmarks]]
name = "coding_task"
backend = "jarvis-agent"
agent = "native_openhands"
tools = ["think", "code_interpreter"]
max_samples = 29
