mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-30 19:02:16 +00:00
Jon Saad-FalconandClaude Opus 4.6
301e9cd2d4
Complete implementation across six development phases (v0.1 through v1.0): - Core: Registry system, config, event bus, types (Phase 0) - Intelligence + Inference: Model routing, Ollama/vLLM/llama.cpp/Cloud engines (Phase 1) - Memory: SQLite/FAISS/ColBERT/BM25/Hybrid backends, document ingest, context injection (Phase 2) - Agents: Simple/Orchestrator/Custom/OpenClaw agents, tool system (Phase 3) - Learning: HeuristicRouter, reward functions, GRPO stub, telemetry aggregation (Phase 4) - SDK: Jarvis class, OpenClaw protocol/transport, benchmarks, Docker deployment (Phase 5) 520 tests passing, 8 skipped (optional deps). Ruff lint clean. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
78 lines
2.5 KiB
Python
78 lines
2.5 KiB
Python
"""Tests for the throughput benchmark."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import MagicMock
|
|
|
|
import pytest
|
|
|
|
from openjarvis.bench.throughput import ThroughputBenchmark
|
|
from openjarvis.core.registry import BenchmarkRegistry
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _register_throughput():
|
|
"""Re-register throughput benchmark after registry clear."""
|
|
from openjarvis.bench.throughput import ensure_registered
|
|
|
|
ensure_registered()
|
|
|
|
|
|
def _make_engine(completion_tokens=10):
|
|
engine = MagicMock()
|
|
engine.engine_id = "mock"
|
|
engine.generate.return_value = {
|
|
"content": "Hello world",
|
|
"usage": {
|
|
"prompt_tokens": 5,
|
|
"completion_tokens": completion_tokens,
|
|
"total_tokens": 5 + completion_tokens,
|
|
},
|
|
}
|
|
return engine
|
|
|
|
|
|
class TestThroughputBenchmark:
|
|
def test_registration(self):
|
|
assert BenchmarkRegistry.contains("throughput")
|
|
assert BenchmarkRegistry.get("throughput") is ThroughputBenchmark
|
|
|
|
def test_run_with_mock(self):
|
|
engine = _make_engine()
|
|
b = ThroughputBenchmark()
|
|
result = b.run(engine, "test-model", num_samples=3)
|
|
assert result.benchmark_name == "throughput"
|
|
assert result.model == "test-model"
|
|
assert result.engine == "mock"
|
|
assert result.samples == 3
|
|
|
|
def test_metrics_keys(self):
|
|
engine = _make_engine()
|
|
b = ThroughputBenchmark()
|
|
result = b.run(engine, "test-model", num_samples=3)
|
|
expected_keys = {"tokens_per_second", "total_tokens", "total_time_seconds"}
|
|
assert set(result.metrics.keys()) == expected_keys
|
|
|
|
def test_tokens_per_second_calc(self):
|
|
engine = _make_engine(completion_tokens=10)
|
|
b = ThroughputBenchmark()
|
|
result = b.run(engine, "test-model", num_samples=5)
|
|
# 5 samples * 10 tokens each = 50 total tokens
|
|
assert result.metrics["total_tokens"] == 50.0
|
|
assert result.metrics["tokens_per_second"] > 0
|
|
|
|
def test_sample_count(self):
|
|
engine = _make_engine()
|
|
b = ThroughputBenchmark()
|
|
b.run(engine, "test-model", num_samples=7)
|
|
assert engine.generate.call_count == 7
|
|
|
|
def test_zero_latency_handling(self):
|
|
"""All errors should result in 0 tokens_per_second."""
|
|
engine = _make_engine()
|
|
engine.generate.side_effect = RuntimeError("fail")
|
|
b = ThroughputBenchmark()
|
|
result = b.run(engine, "test-model", num_samples=3)
|
|
assert result.errors == 3
|
|
assert result.metrics["tokens_per_second"] == 0.0
|