mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-30 10:52:15 +00:00
Jon Saad-FalconandClaude Opus 4.6
301e9cd2d4
Complete implementation across six development phases (v0.1 through v1.0): - Core: Registry system, config, event bus, types (Phase 0) - Intelligence + Inference: Model routing, Ollama/vLLM/llama.cpp/Cloud engines (Phase 1) - Memory: SQLite/FAISS/ColBERT/BM25/Hybrid backends, document ingest, context injection (Phase 2) - Agents: Simple/Orchestrator/Custom/OpenClaw agents, tool system (Phase 3) - Learning: HeuristicRouter, reward functions, GRPO stub, telemetry aggregation (Phase 4) - SDK: Jarvis class, OpenClaw protocol/transport, benchmarks, Docker deployment (Phase 5) 520 tests passing, 8 skipped (optional deps). Ruff lint clean. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
83 lines
2.4 KiB
Python
83 lines
2.4 KiB
Python
"""Tests for the latency benchmark."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import MagicMock
|
|
|
|
import pytest
|
|
|
|
from openjarvis.bench.latency import LatencyBenchmark
|
|
from openjarvis.core.registry import BenchmarkRegistry
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _register_latency():
|
|
"""Re-register latency benchmark after registry clear."""
|
|
from openjarvis.bench.latency import ensure_registered
|
|
|
|
ensure_registered()
|
|
|
|
|
|
def _make_engine():
|
|
engine = MagicMock()
|
|
engine.engine_id = "mock"
|
|
engine.generate.return_value = {
|
|
"content": "Hello",
|
|
"usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},
|
|
}
|
|
return engine
|
|
|
|
|
|
class TestLatencyBenchmark:
|
|
def test_registration(self):
|
|
assert BenchmarkRegistry.contains("latency")
|
|
assert BenchmarkRegistry.get("latency") is LatencyBenchmark
|
|
|
|
def test_name(self):
|
|
b = LatencyBenchmark()
|
|
assert b.name == "latency"
|
|
|
|
def test_description(self):
|
|
b = LatencyBenchmark()
|
|
assert "latency" in b.description.lower()
|
|
|
|
def test_run_with_mock_engine(self):
|
|
engine = _make_engine()
|
|
b = LatencyBenchmark()
|
|
result = b.run(engine, "test-model", num_samples=3)
|
|
assert result.benchmark_name == "latency"
|
|
assert result.model == "test-model"
|
|
assert result.engine == "mock"
|
|
assert result.samples == 3
|
|
assert result.errors == 0
|
|
|
|
def test_metrics_keys(self):
|
|
engine = _make_engine()
|
|
b = LatencyBenchmark()
|
|
result = b.run(engine, "test-model", num_samples=3)
|
|
expected_keys = {
|
|
"mean_latency", "p50_latency", "p95_latency",
|
|
"min_latency", "max_latency",
|
|
}
|
|
assert set(result.metrics.keys()) == expected_keys
|
|
|
|
def test_sample_count(self):
|
|
engine = _make_engine()
|
|
b = LatencyBenchmark()
|
|
b.run(engine, "test-model", num_samples=5)
|
|
assert engine.generate.call_count == 5
|
|
|
|
def test_run_with_errors(self):
|
|
engine = _make_engine()
|
|
engine.generate.side_effect = RuntimeError("fail")
|
|
b = LatencyBenchmark()
|
|
result = b.run(engine, "test-model", num_samples=3)
|
|
assert result.errors == 3
|
|
assert result.metrics == {}
|
|
|
|
def test_ensure_registered(self):
|
|
from openjarvis.bench.latency import ensure_registered
|
|
|
|
ensure_registered() # should not raise
|
|
assert BenchmarkRegistry.contains("latency")
|