# Smoke eval — Apple Silicon platform
# Target hardware: Apple Silicon Mac (M1/M2/M3/M4) with 16+ GB unified memory
# Purpose: verify full-stack metrics collection with minimal samples (3 per benchmark)
# Engines: MLX (primary), Ollama (GGUF)
# Run: uv run python -m openjarvis.evals run -c src/openjarvis/evals/configs/smoke_apple.toml

[meta]
name = "smoke-apple"
description = "Full-stack smoke test on Apple Silicon — MLX + Ollama, 3 samples each"

[defaults]
temperature = 0.0
max_tokens = 2048

[judge]
model = "gpt-5-mini-2025-08-07"
temperature = 0.0
max_tokens = 512

[run]
max_workers = 2
output_dir = "results/smoke-apple/"
seed = 42
telemetry = true
gpu_metrics = true

# ── Models ────────────────────────────────────────────────────────────────────

# MLX Qwen 4-bit — fits Apple Silicon 16 GB unified memory
[[models]]
name = "mlx-community/Qwen2.5-7B-4bit"
engine = "mlx"
param_count_b = 7.0

# Ollama GGUF — pull with: ollama pull qwen3.5:2b
[[models]]
name = "qwen3.5:2b"
engine = "ollama"

# ── Benchmarks ────────────────────────────────────────────────────────────────
# 6 benchmarks × 3 samples = 18 runs per model

[[benchmarks]]
name = "supergpqa"
backend = "jarvis-direct"
max_samples = 3

[[benchmarks]]
name = "simpleqa"
backend = "jarvis-direct"
max_samples = 3

[[benchmarks]]
name = "doc_qa"
backend = "jarvis-agent"
agent = "simple"
tools = ["think"]
max_samples = 3

[[benchmarks]]
name = "coding_assistant"
backend = "jarvis-agent"
agent = "native_react"
tools = ["file_read", "code_interpreter", "shell_exec", "think"]
max_samples = 3

[[benchmarks]]
name = "daily_digest"
backend = "jarvis-agent"
agent = "orchestrator"
tools = ["web_search", "think"]
max_samples = 3

[[benchmarks]]
name = "browser_assistant"
backend = "jarvis-agent"
agent = "orchestrator"
tools = ["web_search", "browser", "think"]
max_samples = 3
