mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-30 19:02:16 +00:00
Jon Saad-FalconandClaude Opus 4.6
990d7d8a79
Add ReAct and OpenHands agents, WebSearch and CodeInterpreter tools, full MCP protocol layer (server/client/transport), Gemini cloud engine support, 12 new model specs (4 local MoE + 8 cloud), trace system, and comprehensive test coverage across all dimensions (hardware, engine, memory, agents, tools, MCP, integration). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
240 lines
8.7 KiB
Python
240 lines
8.7 KiB
Python
"""Tests for the extended model catalog -- all new ModelSpec entries."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from openjarvis.core.registry import ModelRegistry
|
|
from openjarvis.core.types import ModelSpec
|
|
from openjarvis.intelligence.model_catalog import (
|
|
BUILTIN_MODELS,
|
|
merge_discovered_models,
|
|
register_builtin_models,
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _get_spec(model_id: str) -> ModelSpec:
|
|
"""Lookup a spec from the BUILTIN_MODELS list (not registry)."""
|
|
for spec in BUILTIN_MODELS:
|
|
if spec.model_id == model_id:
|
|
return spec
|
|
raise KeyError(model_id)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Local models
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestLocalModelSpecs:
|
|
"""Verify every new local model has correct fields."""
|
|
|
|
def test_gpt_oss_120b_model_id(self) -> None:
|
|
spec = _get_spec("gpt-oss:120b")
|
|
assert spec.model_id == "gpt-oss:120b"
|
|
|
|
def test_gpt_oss_120b_params(self) -> None:
|
|
spec = _get_spec("gpt-oss:120b")
|
|
assert spec.parameter_count_b == 117.0
|
|
assert spec.active_parameter_count_b == 5.1
|
|
|
|
def test_gpt_oss_120b_context(self) -> None:
|
|
spec = _get_spec("gpt-oss:120b")
|
|
assert spec.context_length == 131072
|
|
|
|
def test_gpt_oss_120b_engines(self) -> None:
|
|
spec = _get_spec("gpt-oss:120b")
|
|
assert "vllm" in spec.supported_engines
|
|
assert "ollama" in spec.supported_engines
|
|
|
|
def test_gpt_oss_120b_architecture(self) -> None:
|
|
spec = _get_spec("gpt-oss:120b")
|
|
assert spec.metadata["architecture"] == "moe"
|
|
|
|
def test_qwen3_8b_model_id(self) -> None:
|
|
spec = _get_spec("qwen3:8b")
|
|
assert spec.model_id == "qwen3:8b"
|
|
|
|
def test_qwen3_8b_params(self) -> None:
|
|
spec = _get_spec("qwen3:8b")
|
|
assert spec.parameter_count_b == 8.2
|
|
|
|
def test_qwen3_8b_context(self) -> None:
|
|
spec = _get_spec("qwen3:8b")
|
|
assert spec.context_length == 32768
|
|
|
|
def test_qwen3_8b_engines(self) -> None:
|
|
spec = _get_spec("qwen3:8b")
|
|
for e in ("vllm", "ollama", "llamacpp", "sglang"):
|
|
assert e in spec.supported_engines
|
|
|
|
def test_qwen3_8b_architecture(self) -> None:
|
|
spec = _get_spec("qwen3:8b")
|
|
assert spec.metadata["architecture"] == "dense"
|
|
|
|
def test_glm_47_flash_params(self) -> None:
|
|
spec = _get_spec("glm-4.7-flash")
|
|
assert spec.parameter_count_b == 30.0
|
|
assert spec.active_parameter_count_b == 3.0
|
|
|
|
def test_glm_47_flash_context(self) -> None:
|
|
spec = _get_spec("glm-4.7-flash")
|
|
assert spec.context_length == 131072
|
|
|
|
def test_glm_47_flash_engines(self) -> None:
|
|
spec = _get_spec("glm-4.7-flash")
|
|
assert "vllm" in spec.supported_engines
|
|
assert "sglang" in spec.supported_engines
|
|
|
|
def test_trinity_mini_params(self) -> None:
|
|
spec = _get_spec("trinity-mini")
|
|
assert spec.parameter_count_b == 26.0
|
|
assert spec.active_parameter_count_b == 3.0
|
|
|
|
def test_trinity_mini_context(self) -> None:
|
|
spec = _get_spec("trinity-mini")
|
|
assert spec.context_length == 128000
|
|
|
|
def test_trinity_mini_engines(self) -> None:
|
|
spec = _get_spec("trinity-mini")
|
|
assert "vllm" in spec.supported_engines
|
|
assert "llamacpp" in spec.supported_engines
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cloud models
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCloudModelSpecs:
|
|
"""Verify every new cloud model has correct fields."""
|
|
|
|
def test_gpt_5_mini_provider(self) -> None:
|
|
spec = _get_spec("gpt-5-mini")
|
|
assert spec.provider == "openai"
|
|
assert spec.requires_api_key is True
|
|
|
|
def test_gpt_5_mini_context(self) -> None:
|
|
spec = _get_spec("gpt-5-mini")
|
|
assert spec.context_length == 400000
|
|
|
|
def test_gpt_5_mini_pricing(self) -> None:
|
|
spec = _get_spec("gpt-5-mini")
|
|
assert spec.metadata["pricing_input"] == pytest.approx(0.25)
|
|
assert spec.metadata["pricing_output"] == pytest.approx(2.00)
|
|
|
|
def test_claude_opus_4_6_provider(self) -> None:
|
|
spec = _get_spec("claude-opus-4-6")
|
|
assert spec.provider == "anthropic"
|
|
assert spec.requires_api_key is True
|
|
|
|
def test_claude_opus_4_6_context(self) -> None:
|
|
spec = _get_spec("claude-opus-4-6")
|
|
assert spec.context_length == 200000
|
|
|
|
def test_claude_sonnet_4_6_provider(self) -> None:
|
|
spec = _get_spec("claude-sonnet-4-6")
|
|
assert spec.provider == "anthropic"
|
|
assert spec.requires_api_key is True
|
|
|
|
def test_claude_haiku_4_5_provider(self) -> None:
|
|
spec = _get_spec("claude-haiku-4-5")
|
|
assert spec.provider == "anthropic"
|
|
assert spec.requires_api_key is True
|
|
|
|
def test_claude_haiku_4_5_pricing(self) -> None:
|
|
spec = _get_spec("claude-haiku-4-5")
|
|
assert spec.metadata["pricing_input"] == pytest.approx(1.00)
|
|
assert spec.metadata["pricing_output"] == pytest.approx(5.00)
|
|
|
|
def test_gemini_25_pro_provider(self) -> None:
|
|
spec = _get_spec("gemini-2.5-pro")
|
|
assert spec.provider == "google"
|
|
assert spec.requires_api_key is True
|
|
assert spec.context_length == 1000000
|
|
|
|
def test_gemini_25_flash_provider(self) -> None:
|
|
spec = _get_spec("gemini-2.5-flash")
|
|
assert spec.provider == "google"
|
|
assert spec.context_length == 1000000
|
|
|
|
def test_gemini_3_pro_provider(self) -> None:
|
|
spec = _get_spec("gemini-3-pro")
|
|
assert spec.provider == "google"
|
|
assert spec.requires_api_key is True
|
|
|
|
def test_gemini_3_flash_provider(self) -> None:
|
|
spec = _get_spec("gemini-3-flash")
|
|
assert spec.provider == "google"
|
|
assert spec.requires_api_key is True
|
|
|
|
def test_gemini_3_pro_pricing(self) -> None:
|
|
spec = _get_spec("gemini-3-pro")
|
|
assert spec.metadata["pricing_input"] == pytest.approx(2.00)
|
|
assert spec.metadata["pricing_output"] == pytest.approx(12.00)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Discovery / invariant tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestModelDiscovery:
|
|
def test_local_models_have_engine_compat(self) -> None:
|
|
"""Every local model has at least one supported engine."""
|
|
for spec in BUILTIN_MODELS:
|
|
if not spec.requires_api_key:
|
|
assert len(spec.supported_engines) >= 1, (
|
|
f"{spec.model_id} has no supported engines"
|
|
)
|
|
|
|
def test_cloud_models_require_api_key(self) -> None:
|
|
"""All cloud models have requires_api_key=True."""
|
|
cloud_ids = {
|
|
"gpt-4o", "gpt-4o-mini", "gpt-5-mini",
|
|
"claude-sonnet-4-20250514", "claude-opus-4-20250514",
|
|
"claude-opus-4-6", "claude-sonnet-4-6", "claude-haiku-4-5",
|
|
"gemini-2.5-pro", "gemini-2.5-flash", "gemini-3-pro", "gemini-3-flash",
|
|
}
|
|
for spec in BUILTIN_MODELS:
|
|
if spec.model_id in cloud_ids:
|
|
assert spec.requires_api_key is True, (
|
|
f"{spec.model_id} should require API key"
|
|
)
|
|
|
|
def test_moe_models_have_active_params(self) -> None:
|
|
"""MoE models have active_parameter_count_b set."""
|
|
moe_ids = {"gpt-oss:120b", "glm-4.7-flash", "trinity-mini"}
|
|
for spec in BUILTIN_MODELS:
|
|
if spec.model_id in moe_ids:
|
|
assert spec.active_parameter_count_b is not None
|
|
assert spec.active_parameter_count_b > 0
|
|
|
|
def test_all_models_have_context_length(self) -> None:
|
|
"""No model has zero or None context length."""
|
|
for spec in BUILTIN_MODELS:
|
|
assert spec.context_length > 0, (
|
|
f"{spec.model_id} has context_length={spec.context_length}"
|
|
)
|
|
|
|
def test_merge_discovered_preserves_new(self) -> None:
|
|
"""merge_discovered_models works for all new model IDs."""
|
|
register_builtin_models()
|
|
new_ids = [
|
|
"gpt-oss:120b", "glm-4.7-flash", "trinity-mini",
|
|
"gpt-5-mini", "claude-opus-4-6", "gemini-3-pro",
|
|
]
|
|
# Merging known IDs should not raise
|
|
merge_discovered_models("vllm", new_ids)
|
|
for mid in new_ids:
|
|
assert ModelRegistry.contains(mid)
|
|
|
|
def test_register_builtin_is_idempotent(self) -> None:
|
|
"""Calling register_builtin_models twice does not raise."""
|
|
register_builtin_models()
|
|
register_builtin_models() # should not raise
|
|
assert ModelRegistry.contains("qwen3:8b")
|