# Open-source models × Orchestrator agent on 5 use-case benchmarks.
#
# Models: Qwen3.5-397B-A17B, Qwen3.5-122B-A10B, GLM-5, gpt-oss-120b, GLM-4.7-Flash
# Agent: OrchestratorAgent (multi-turn tool-calling loop)
# Engine: vLLM (large MoE models need tensor parallelism)
#
# Setup:
#   - Start vLLM with each model, e.g.:
#     vllm serve unsloth/Qwen3.5-397B-A17B-GGUF --tensor-parallel-size 8
#   - Or use Ollama for smaller models:
#     ollama pull glm4.7-flash

[meta]
name = "use-case-opensource-orchestrator"
description = "Open-source models with OrchestratorAgent on use-case benchmarks"

[defaults]
temperature = 0.0
max_tokens = 2048

[judge]
model = "gpt-5-mini-2025-08-07"

[run]
max_workers = 4
output_dir = "results/use-cases-opensource-orchestrator/"
seed = 42
telemetry = true
gpu_metrics = true

# --- Models Under Test ---

[[models]]
name = "unsloth/Qwen3.5-397B-A17B-GGUF"
engine = "vllm"
param_count_b = 397.0
active_params_b = 17.0
num_gpus = 8

[[models]]
name = "Qwen/Qwen3.5-122B-A10B-FP8"
engine = "vllm"
param_count_b = 122.0
active_params_b = 10.0
num_gpus = 4

[[models]]
name = "unsloth/GLM-5-GGUF"
engine = "vllm"
param_count_b = 100.0
num_gpus = 4

[[models]]
name = "unsloth/gpt-oss-120b-GGUF"
engine = "vllm"
param_count_b = 120.0
num_gpus = 4

[[models]]
name = "unsloth/GLM-4.7-Flash-GGUF"
engine = "vllm"
param_count_b = 30.0
active_params_b = 3.0
num_gpus = 2

# --- Benchmarks ---

[[benchmarks]]
name = "email_triage"
backend = "jarvis-agent"
agent = "orchestrator"
tools = ["think"]
max_samples = 30

[[benchmarks]]
name = "morning_brief"
backend = "jarvis-agent"
agent = "orchestrator"
tools = ["think", "web_search"]
max_samples = 15

[[benchmarks]]
name = "research_mining"
backend = "jarvis-agent"
agent = "orchestrator"
tools = ["think", "web_search"]
max_samples = 30

[[benchmarks]]
name = "knowledge_base"
backend = "jarvis-agent"
agent = "orchestrator"
tools = ["think"]
max_samples = 30

[[benchmarks]]
name = "coding_task"
backend = "jarvis-agent"
agent = "orchestrator"
tools = ["think", "code_interpreter"]
max_samples = 29
