# PaperArena evaluation with MonitorOperativeAgent
# Benchmark: scientific literature reasoning (MC/CA/OA)
# Paper: arXiv:2510.10909, best frontier: Gemini 2.5 Pro at 38.8%

[meta]
name = "paperarena-monitor-operative"
description = "PaperArena scientific reasoning with MonitorOperativeAgent"

[defaults]
temperature = 0.6
max_tokens = 32768

[judge]
model = "gpt-5-mini-2025-08-07"
temperature = 0.0
max_tokens = 1024

[run]
max_workers = 1
output_dir = "results/paperarena-monitor-operative/"
seed = 42
telemetry = true
gpu_metrics = true

[[models]]
name = "Qwen/Qwen3.5-35B-A3B"
engine = "vllm"
param_count_b = 35.0
active_params_b = 3.0
gpu_peak_tflops = 989.5
gpu_peak_bandwidth_gb_s = 3350.0
num_gpus = 2

[[benchmarks]]
name = "paperarena"
backend = "jarvis-agent"
agent = "monitor_operative"
max_samples = 50
tools = [
    "think", "calculator", "code_interpreter",
    "web_search", "file_read", "memory_store", "memory_search",
]
