# MonitorOperativeAgent evaluation on Phase 25 text-to-text benchmarks
# Machine: 2x H100-80GB (GPUs 6-7), engine: vLLM, model: Qwen3.5-35B-A3B
# Inference: Qwen3.5 thinking mode (temperature=0.6, top_p=0.95, top_k=20)
# Benchmarks: LogHub (log anomaly detection), LifelongAgentBench (sequential SQL)
# Note: AMA-Bench excluded (no public dataset yet, paper arxiv:2602.22769)
#       DeepPlanning/PaperArena: see dedicated config files

[meta]
name = "qwen3.5-35b-monitor-operative"
description = "Evaluate MonitorOperativeAgent on Phase 25 text-to-text benchmarks"

[defaults]
temperature = 0.6          # Qwen3.5 thinking mode (precise analysis)
max_tokens = 32768         # Qwen3.5 recommended for general queries

[judge]
model = "gpt-5-mini-2025-08-07"
temperature = 0.0
max_tokens = 1024

[run]
max_workers = 1            # Sequential for agent state safety
output_dir = "results/qwen3.5-35b-monitor-operative/"
seed = 42
telemetry = true
gpu_metrics = true

[[models]]
name = "Qwen/Qwen3.5-35B-A3B"
engine = "vllm"
param_count_b = 35.0
active_params_b = 3.0
gpu_peak_tflops = 989.5        # H100 SXM FP16 peak TFLOPS
gpu_peak_bandwidth_gb_s = 3350.0  # H100 SXM memory bandwidth
num_gpus = 2

# --- Phase 25 Text-to-Text Benchmarks ---

[[benchmarks]]
name = "loghub"
backend = "jarvis-agent"
agent = "monitor_operative"
max_samples = 50
tools = [
    "think", "calculator", "code_interpreter",
    "memory_store", "memory_search", "memory_retrieve",
]

[[benchmarks]]
name = "lifelong-agent"
backend = "jarvis-agent"
agent = "monitor_operative"
max_samples = 50
tools = [
    "think", "calculator", "code_interpreter",
    "memory_store", "memory_search", "memory_retrieve",
]
