Files
OpenJarvis/tests/engine/test_openai_compat_api_key.py
T
50993dfa4d fix(evals): honor --base-url/--api-key for first-party eval backends (#535)
`jarvis eval run --base-url ... --api-key ...` was silently dropped for
jarvis-direct/jarvis-agent (_build_backend only forwarded the flags to
hermes/openclaw) and ignored by terminalbench-native, which hardcoded
api_base="http://localhost:8000/v1". Worse, with --base-url set the
engine-discovery fallback silently substituted ANY healthy local engine
(observed: requested vllm + healthy endpoint at --base-url, got
OllamaEngine@localhost:11434 — the requested URL was never contacted).

Changes:
- _OpenAICompatibleEngine gains an api_key param (Bearer Authorization
  header on the httpx client; {ENGINE_ID}_API_KEY env fallback with
  hyphen-sanitized names; no header when unset).
- New non-registered OpenAICompatEngine + normalize_openai_base_url()
  (strips a single literal trailing "/v1" so request paths don't double).
- SystemBuilder.engine_instance() injects a pre-built engine; build()
  health-checks it and fails loudly naming the host instead of falling
  back to discovery. Discovery substitution after an explicit -e key now
  logs a warning.
- JarvisDirectBackend/JarvisAgentBackend accept base_url/api_key; on
  base_url they pin an OpenAICompatEngine to that endpoint with a
  fail-fast pre-flight (actionable error naming the URL and probe).
- _build_backend forwards base_url/api_key to first-party backends on
  the CLI path; _run_terminalbench_native receives --base-url as
  api_base (single /v1 suffix) and exports OPENAI_API_KEY around the
  in-process harness run (terminus-2 routes via LiteLLM).
- Suite TOML [backend.external] stays scoped to hermes/openclaw
  (suite_mode=True in the suite drivers) — first-party suite semantics
  are explicitly deferred. The config-host path is untouched.
- Help text updated on both CLI surfaces; KNOWN_BACKENDS now lists
  hermes/openclaw/terminalbench-native.

Fixes the eval-CLI endpoint gap reported by the downstream team.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-11 13:49:37 -07:00

110 lines
4.6 KiB
Python

"""API-key (Authorization header) support in the OpenAI-compat engine base."""
from __future__ import annotations
import httpx
import pytest
import respx
from openjarvis.core.types import Message, Role
from openjarvis.engine.openai_compat_engines import (
OpenAICompatEngine,
VLLMEngine,
normalize_openai_base_url,
)
_CHAT_RESPONSE = {
"choices": [{"message": {"content": "ok"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
"model": "m",
}
class TestAuthorizationHeader:
def test_bearer_header_sent_when_api_key_set(self) -> None:
engine = OpenAICompatEngine(host="http://testhost:9000", api_key="sk-test")
with respx.mock:
route = respx.post("http://testhost:9000/v1/chat/completions").mock(
return_value=httpx.Response(200, json=_CHAT_RESPONSE)
)
engine.generate([Message(role=Role.USER, content="hi")], model="m")
assert route.calls.last.request.headers["Authorization"] == "Bearer sk-test"
def test_no_authorization_header_without_api_key(self) -> None:
engine = OpenAICompatEngine(host="http://testhost:9000")
with respx.mock:
route = respx.post("http://testhost:9000/v1/chat/completions").mock(
return_value=httpx.Response(200, json=_CHAT_RESPONSE)
)
engine.generate([Message(role=Role.USER, content="hi")], model="m")
assert "authorization" not in route.calls.last.request.headers
def test_health_check_sends_bearer_header(self) -> None:
engine = OpenAICompatEngine(host="http://testhost:9000", api_key="sk-test")
with respx.mock:
route = respx.get("http://testhost:9000/v1/models").mock(
return_value=httpx.Response(200, json={"data": []})
)
assert engine.health() is True
assert route.calls.last.request.headers["Authorization"] == "Bearer sk-test"
def test_env_var_fallback_sanitizes_hyphen(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
# engine_id "openai-compat" must map to OPENAI_COMPAT_API_KEY —
# shells cannot set hyphenated env-var names.
monkeypatch.setenv("OPENAI_COMPAT_API_KEY", "sk-env")
engine = OpenAICompatEngine(host="http://testhost:9000")
with respx.mock:
route = respx.get("http://testhost:9000/v1/models").mock(
return_value=httpx.Response(200, json={"data": []})
)
engine.health()
assert route.calls.last.request.headers["Authorization"] == "Bearer sk-env"
def test_vllm_env_var_fallback(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("VLLM_API_KEY", "sk-vllm")
engine = VLLMEngine(host="http://testhost:8000")
with respx.mock:
route = respx.get("http://testhost:8000/v1/models").mock(
return_value=httpx.Response(200, json={"data": []})
)
engine.health()
assert route.calls.last.request.headers["Authorization"] == "Bearer sk-vllm"
def test_explicit_api_key_beats_env_var(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("OPENAI_COMPAT_API_KEY", "sk-env")
engine = OpenAICompatEngine(host="http://testhost:9000", api_key="sk-explicit")
assert engine._api_key == "sk-explicit"
class TestNormalizeOpenAIBaseUrl:
@pytest.mark.parametrize(
("url", "expected"),
[
("http://h:8000", "http://h:8000"),
("http://h:8000/", "http://h:8000"),
("http://h:8000/v1", "http://h:8000"),
("http://h:8000/v1/", "http://h:8000"),
("http://h:8000/gateway/v1", "http://h:8000/gateway"),
# Only a literal trailing "/v1" is stripped — never other paths.
("http://h:8000/v1x", "http://h:8000/v1x"),
("http://h:8000/v2", "http://h:8000/v2"),
],
)
def test_normalization(self, url: str, expected: str) -> None:
assert normalize_openai_base_url(url) == expected
def test_engine_requests_have_single_v1_prefix(self) -> None:
"""End to end: a user-supplied .../v1 URL must not produce /v1/v1."""
host = normalize_openai_base_url("http://testhost:9000/v1")
engine = OpenAICompatEngine(host=host)
with respx.mock:
route = respx.get("http://testhost:9000/v1/models").mock(
return_value=httpx.Response(200, json={"data": []})
)
assert engine.health() is True
assert route.calls.last.request.url.path == "/v1/models"