# GLM-4.7-FP8 eval with NativeOpenHandsAgent on GAIA only.
# Machine: 8x A100-80GB, engine: vLLM, TP=8

[meta]
name = "glm-4.7-fp8-openhands-gaia"
description = "Evaluate GLM-4.7-FP8 via NativeOpenHandsAgent on GAIA"

[defaults]
temperature = 0.0
max_tokens = 2048

[judge]
model = "gpt-5-mini-2025-08-07"
temperature = 0.0
max_tokens = 1024

[run]
max_workers = 4
output_dir = "results/glm-4.7-fp8-openhands-gaia/"
seed = 42
telemetry = true
gpu_metrics = true

# --- Model Under Test ---

[[models]]
name = "zai-org/GLM-4.7-FP8"
engine = "vllm"
param_count_b = 30.0              # 30B total MoE params
active_params_b = 3.0             # ~3B active params per token
gpu_peak_tflops = 312.0           # A100 SXM FP16 peak TFLOPS
gpu_peak_bandwidth_gb_s = 2039.0  # A100 SXM memory bandwidth
num_gpus = 8                      # TP=8

# --- Benchmarks ---

# Agentic: GAIA — requires file reading, web search, multi-step reasoning
[[benchmarks]]
name = "gaia"
backend = "jarvis-agent"
agent = "native_openhands"
tools = ["code_interpreter", "web_search", "file_read", "calculator", "think"]
max_samples = 50
