mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-28 05:12:26 +00:00
`jarvis eval run --base-url ... --api-key ...` was silently dropped for jarvis-direct/jarvis-agent (_build_backend only forwarded the flags to hermes/openclaw) and ignored by terminalbench-native, which hardcoded api_base="http://localhost:8000/v1". Worse, with --base-url set the engine-discovery fallback silently substituted ANY healthy local engine (observed: requested vllm + healthy endpoint at --base-url, got OllamaEngine@localhost:11434 — the requested URL was never contacted). Changes: - _OpenAICompatibleEngine gains an api_key param (Bearer Authorization header on the httpx client; {ENGINE_ID}_API_KEY env fallback with hyphen-sanitized names; no header when unset). - New non-registered OpenAICompatEngine + normalize_openai_base_url() (strips a single literal trailing "/v1" so request paths don't double). - SystemBuilder.engine_instance() injects a pre-built engine; build() health-checks it and fails loudly naming the host instead of falling back to discovery. Discovery substitution after an explicit -e key now logs a warning. - JarvisDirectBackend/JarvisAgentBackend accept base_url/api_key; on base_url they pin an OpenAICompatEngine to that endpoint with a fail-fast pre-flight (actionable error naming the URL and probe). - _build_backend forwards base_url/api_key to first-party backends on the CLI path; _run_terminalbench_native receives --base-url as api_base (single /v1 suffix) and exports OPENAI_API_KEY around the in-process harness run (terminus-2 routes via LiteLLM). - Suite TOML [backend.external] stays scoped to hermes/openclaw (suite_mode=True in the suite drivers) — first-party suite semantics are explicitly deferred. The config-host path is untouched. - Help text updated on both CLI surfaces; KNOWN_BACKENDS now lists hermes/openclaw/terminalbench-native. Fixes the eval-CLI endpoint gap reported by the downstream team. Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
110 lines
4.6 KiB
Python
110 lines
4.6 KiB
Python
"""API-key (Authorization header) support in the OpenAI-compat engine base."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import httpx
|
|
import pytest
|
|
import respx
|
|
|
|
from openjarvis.core.types import Message, Role
|
|
from openjarvis.engine.openai_compat_engines import (
|
|
OpenAICompatEngine,
|
|
VLLMEngine,
|
|
normalize_openai_base_url,
|
|
)
|
|
|
|
_CHAT_RESPONSE = {
|
|
"choices": [{"message": {"content": "ok"}, "finish_reason": "stop"}],
|
|
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
|
"model": "m",
|
|
}
|
|
|
|
|
|
class TestAuthorizationHeader:
|
|
def test_bearer_header_sent_when_api_key_set(self) -> None:
|
|
engine = OpenAICompatEngine(host="http://testhost:9000", api_key="sk-test")
|
|
with respx.mock:
|
|
route = respx.post("http://testhost:9000/v1/chat/completions").mock(
|
|
return_value=httpx.Response(200, json=_CHAT_RESPONSE)
|
|
)
|
|
engine.generate([Message(role=Role.USER, content="hi")], model="m")
|
|
assert route.calls.last.request.headers["Authorization"] == "Bearer sk-test"
|
|
|
|
def test_no_authorization_header_without_api_key(self) -> None:
|
|
engine = OpenAICompatEngine(host="http://testhost:9000")
|
|
with respx.mock:
|
|
route = respx.post("http://testhost:9000/v1/chat/completions").mock(
|
|
return_value=httpx.Response(200, json=_CHAT_RESPONSE)
|
|
)
|
|
engine.generate([Message(role=Role.USER, content="hi")], model="m")
|
|
assert "authorization" not in route.calls.last.request.headers
|
|
|
|
def test_health_check_sends_bearer_header(self) -> None:
|
|
engine = OpenAICompatEngine(host="http://testhost:9000", api_key="sk-test")
|
|
with respx.mock:
|
|
route = respx.get("http://testhost:9000/v1/models").mock(
|
|
return_value=httpx.Response(200, json={"data": []})
|
|
)
|
|
assert engine.health() is True
|
|
assert route.calls.last.request.headers["Authorization"] == "Bearer sk-test"
|
|
|
|
def test_env_var_fallback_sanitizes_hyphen(
|
|
self, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
# engine_id "openai-compat" must map to OPENAI_COMPAT_API_KEY —
|
|
# shells cannot set hyphenated env-var names.
|
|
monkeypatch.setenv("OPENAI_COMPAT_API_KEY", "sk-env")
|
|
engine = OpenAICompatEngine(host="http://testhost:9000")
|
|
with respx.mock:
|
|
route = respx.get("http://testhost:9000/v1/models").mock(
|
|
return_value=httpx.Response(200, json={"data": []})
|
|
)
|
|
engine.health()
|
|
assert route.calls.last.request.headers["Authorization"] == "Bearer sk-env"
|
|
|
|
def test_vllm_env_var_fallback(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("VLLM_API_KEY", "sk-vllm")
|
|
engine = VLLMEngine(host="http://testhost:8000")
|
|
with respx.mock:
|
|
route = respx.get("http://testhost:8000/v1/models").mock(
|
|
return_value=httpx.Response(200, json={"data": []})
|
|
)
|
|
engine.health()
|
|
assert route.calls.last.request.headers["Authorization"] == "Bearer sk-vllm"
|
|
|
|
def test_explicit_api_key_beats_env_var(
|
|
self, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
monkeypatch.setenv("OPENAI_COMPAT_API_KEY", "sk-env")
|
|
engine = OpenAICompatEngine(host="http://testhost:9000", api_key="sk-explicit")
|
|
assert engine._api_key == "sk-explicit"
|
|
|
|
|
|
class TestNormalizeOpenAIBaseUrl:
|
|
@pytest.mark.parametrize(
|
|
("url", "expected"),
|
|
[
|
|
("http://h:8000", "http://h:8000"),
|
|
("http://h:8000/", "http://h:8000"),
|
|
("http://h:8000/v1", "http://h:8000"),
|
|
("http://h:8000/v1/", "http://h:8000"),
|
|
("http://h:8000/gateway/v1", "http://h:8000/gateway"),
|
|
# Only a literal trailing "/v1" is stripped — never other paths.
|
|
("http://h:8000/v1x", "http://h:8000/v1x"),
|
|
("http://h:8000/v2", "http://h:8000/v2"),
|
|
],
|
|
)
|
|
def test_normalization(self, url: str, expected: str) -> None:
|
|
assert normalize_openai_base_url(url) == expected
|
|
|
|
def test_engine_requests_have_single_v1_prefix(self) -> None:
|
|
"""End to end: a user-supplied .../v1 URL must not produce /v1/v1."""
|
|
host = normalize_openai_base_url("http://testhost:9000/v1")
|
|
engine = OpenAICompatEngine(host=host)
|
|
with respx.mock:
|
|
route = respx.get("http://testhost:9000/v1/models").mock(
|
|
return_value=httpx.Response(200, json={"data": []})
|
|
)
|
|
assert engine.health() is True
|
|
assert route.calls.last.request.url.path == "/v1/models"
|