From 5270149ea59f5c9eb4be23e4ab3e037a7cab8b31 Mon Sep 17 00:00:00 2001 From: Gilbert Barajas Date: Mon, 1 Jun 2026 11:23:43 -0500 Subject: [PATCH] fix(engine): modernize apple_fm_shim for the public apple-fm-sdk (#377) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(engine): modernize apple_fm_shim for the public apple-fm-sdk Apple released the official Foundation Models Python SDK as `apple/python-apple-fm-sdk` (import name `apple_fm_sdk`, distribution name `apple-fm-sdk`). The shim's `import apple_fm` predates the public SDK and the per-call API has since moved as well; running the shim against current `apple-fm-sdk` v0.1.1 fails on import, then on the `/health` call shape, and again on every `respond` / `stream_response` keyword. This change brings the shim up to date with the real SDK without altering its external OpenAI-compatible contract: - Import `apple_fm_sdk` (the public name). Update the missing-package error to point at the GitHub repo since the SDK isn't on PyPI; installation is `uv pip install -e `. - `SystemLanguageModel.is_available()` is an instance method and now returns `(bool, SystemLanguageModelUnavailableReason | None)`. The health endpoint instantiates the model and unpacks the tuple, and surfaces the reason string when the model is unavailable. - `LanguageModelSession.respond` and `.stream_response` no longer accept `max_tokens` / `temperature` positionally; they take a `GenerationOptions` instance via the `options` kwarg. The shim now builds a `GenerationOptions(temperature=..., maximum_response_tokens=...)` from each `ChatRequest` and threads it through both paths. `temperature` is now actually honored (the previous code dropped it silently). - Add `response_model=None` to the `/v1/chat/completions` decorator so FastAPI doesn't try to build a Pydantic field for the `JSONResponse | StreamingResponse` union return type — that fails with the FastAPI version pinned in the `server` extra. - Update the module docstring to reference macOS 26 + Apple Intelligence (the SDK's actual minimum), not macOS 15. ## How was this tested? - `uv pip install -e ./python-apple-fm-sdk` against a local clone of Apple's repo on macOS 26 / M5 Max with Apple Intelligence enabled. - `uv sync --extra dev --extra server` then `uv run uvicorn openjarvis.engine.apple_fm_shim:app --host 127.0.0.1 --port 8079`. - `GET /health` → `{"status": "ok"}` (200). - `GET /v1/models` → lists `apple-fm`. - `POST /v1/chat/completions` with messages + temperature + max_tokens → returns a real Apple Intelligence completion. - End-to-end through OpenJarvis: add `[engine.apple_fm] host = "http://localhost:8079"` to `~/.openjarvis/config.toml`, then `jarvis ask --engine apple_fm --model apple-fm "..."` returns the same Apple FM response routed through the OpenAI-compatible engine wrapper. - `uv run ruff check src/openjarvis/engine/apple_fm_shim.py` and `uv run ruff format --check src/openjarvis/engine/apple_fm_shim.py` both pass. * test(engine): add stubbed-SDK tests for apple_fm_shim migration Covers the apple_fm -> apple_fm_sdk migration: GenerationOptions carries temperature + max_tokens and is passed via options= to respond() and stream_response(), and /health unpacks the (available, reason) tuple from SystemLanguageModel().is_available(). The real apple-fm-sdk is not installable in CI (not on PyPI; macOS 26 + Apple Intelligence only), so the tests inject a stub SDK into sys.modules. They verify the shim's OpenAI-compat wiring, not Apple's real SDK behavior. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) --- src/openjarvis/engine/apple_fm_shim.py | 59 ++++++++--- tests/engine/test_apple_fm_shim.py | 137 +++++++++++++++++++++++++ 2 files changed, 180 insertions(+), 16 deletions(-) create mode 100644 tests/engine/test_apple_fm_shim.py diff --git a/src/openjarvis/engine/apple_fm_shim.py b/src/openjarvis/engine/apple_fm_shim.py index 215735e4..49941a5d 100644 --- a/src/openjarvis/engine/apple_fm_shim.py +++ b/src/openjarvis/engine/apple_fm_shim.py @@ -1,12 +1,14 @@ """Apple Foundation Models shim. -Thin FastAPI server exposing Apple FM as OpenAI-compatible API. -Only runs on macOS 15+ with Apple Silicon. Wraps python-apple-fm-sdk's -LanguageModelSession as /v1/chat/completions and /v1/models endpoints. +Thin FastAPI server exposing Apple Intelligence's on-device foundation +model as an OpenAI-compatible API. Only runs on macOS 26+ with Apple +Intelligence enabled. Wraps Apple's `apple-fm-sdk`'s +``LanguageModelSession`` as ``/v1/chat/completions`` and ``/v1/models`` +endpoints. -**Token counts:** The Apple FM SDK does not expose token counts. The shim -returns 0 for all token counts. Throughput and energy benchmarks will -reflect this limitation. +**Token counts:** The Apple FM SDK does not expose token counts. The +shim returns 0 for all token counts. Throughput and energy benchmarks +will reflect this limitation. Usage: uvicorn openjarvis.engine.apple_fm_shim:app \ @@ -26,10 +28,15 @@ if platform.system() != "Darwin": sys.exit(1) try: - import apple_fm # type: ignore[import-untyped] + import apple_fm_sdk # type: ignore[import-untyped] except ImportError: print( - "apple_fm_shim: pip install python-apple-fm-sdk", + "apple_fm_shim: apple-fm-sdk is not available. The SDK is not on\n" + "PyPI yet; clone https://github.com/apple/python-apple-fm-sdk and\n" + "install from source:\n" + " git clone https://github.com/apple/python-apple-fm-sdk\n" + " uv pip install -e ./python-apple-fm-sdk\n" + "Requires macOS 26+, Xcode 26+, and Apple Intelligence enabled.", file=sys.stderr, ) sys.exit(1) @@ -70,12 +77,31 @@ def _build_prompt(messages: list[ChatMessage]) -> str: return "\n".join(parts) +def _generation_options(req: ChatRequest) -> apple_fm_sdk.GenerationOptions: + """Build the per-request GenerationOptions from a ChatRequest. + + Apple FM doesn't take ``max_tokens`` / ``temperature`` as positional + args to ``respond`` / ``stream_response`` — they live on a + ``GenerationOptions`` object passed via the ``options`` kwarg. + """ + return apple_fm_sdk.GenerationOptions( + temperature=req.temperature, + maximum_response_tokens=req.max_tokens, + ) + + @app.get("/health") def health() -> JSONResponse: - is_available = apple_fm.SystemLanguageModel.is_available() - status = "ok" if is_available else "unavailable" - code = 200 if is_available else 503 - return JSONResponse({"status": status}, status_code=code) + # SystemLanguageModel.is_available() is an *instance* method that + # returns (bool, reason | None). Unpack so we can both gate the + # response code and surface the reason for unavailability. + available, reason = apple_fm_sdk.SystemLanguageModel().is_available() + if available: + return JSONResponse({"status": "ok"}, status_code=200) + return JSONResponse( + {"status": "unavailable", "reason": str(reason) if reason else None}, + status_code=503, + ) @app.get("/v1/models") @@ -90,12 +116,13 @@ def list_models() -> JSONResponse: ) -@app.post("/v1/chat/completions") +@app.post("/v1/chat/completions", response_model=None) async def chat_completions( req: ChatRequest, ) -> JSONResponse | StreamingResponse: prompt = _build_prompt(req.messages) - session = apple_fm.LanguageModelSession() + session = apple_fm_sdk.LanguageModelSession() + options = _generation_options(req) if req.stream: @@ -103,7 +130,7 @@ async def chat_completions( cid = f"chatcmpl-{uuid.uuid4().hex[:12]}" async for token in session.stream_response( prompt, - max_tokens=req.max_tokens, + options=options, ): chunk = { "id": cid, @@ -140,7 +167,7 @@ async def chat_completions( media_type="text/event-stream", ) - text = await session.respond(prompt, max_tokens=req.max_tokens) + text = await session.respond(prompt, options=options) cid = f"chatcmpl-{uuid.uuid4().hex[:12]}" return JSONResponse( { diff --git a/tests/engine/test_apple_fm_shim.py b/tests/engine/test_apple_fm_shim.py new file mode 100644 index 00000000..afd43a67 --- /dev/null +++ b/tests/engine/test_apple_fm_shim.py @@ -0,0 +1,137 @@ +"""Tests for the Apple FM shim. + +The real ``apple-fm-sdk`` is not on PyPI and only runs on macOS 26+ with +Apple Intelligence, so these tests inject a stub SDK into ``sys.modules`` +before importing the shim. They verify the shim's OpenAI-compat wiring +against the stub's recorded calls — they do NOT exercise Apple's real SDK. +""" + +from __future__ import annotations + +import importlib +import platform +import sys +import types +from typing import Any + +import pytest + +fastapi = pytest.importorskip("fastapi") +from fastapi.testclient import TestClient # noqa: E402 + + +def _install_stub_sdk( + *, + available: tuple[bool, Any] = (True, None), + stream_tokens: list[str] | None = None, + respond_text: str = "Hello from Apple FM.", +) -> dict[str, Any]: + """Inject a fake ``apple_fm_sdk`` module and return a recorder dict. + + The recorder captures the GenerationOptions the shim builds and the + args passed to ``stream_response`` / ``respond`` so tests can assert + the shim wires ``options=`` through (PR #377) rather than the old + ``max_tokens=`` positional kwarg. + """ + rec: dict[str, Any] = {"options": [], "stream_calls": [], "respond_calls": []} + sdk = types.ModuleType("apple_fm_sdk") + + class GenerationOptions: + def __init__(self, temperature: float = 1.0, maximum_response_tokens: int = 0): + self.temperature = temperature + self.maximum_response_tokens = maximum_response_tokens + rec["options"].append(self) + + class SystemLanguageModel: + def is_available(self): # instance method returning (bool, reason) + return available + + class LanguageModelSession: + async def stream_response(self, prompt: str, *, options: Any = None): + rec["stream_calls"].append({"prompt": prompt, "options": options}) + for tok in stream_tokens or []: + yield tok + + async def respond(self, prompt: str, *, options: Any = None) -> str: + rec["respond_calls"].append({"prompt": prompt, "options": options}) + return respond_text + + sdk.GenerationOptions = GenerationOptions # type: ignore[attr-defined] + sdk.SystemLanguageModel = SystemLanguageModel # type: ignore[attr-defined] + sdk.LanguageModelSession = LanguageModelSession # type: ignore[attr-defined] + + sys.modules["apple_fm_sdk"] = sdk + return rec + + +@pytest.fixture +def shim(monkeypatch): + """Import a fresh copy of the shim against the stub SDK on any platform.""" + # The module bails with sys.exit unless it sees Darwin + the SDK import. + monkeypatch.setattr(platform, "system", lambda: "Darwin") + rec = _install_stub_sdk(stream_tokens=["Sure! ", "Sure! The ", "Sure! The answer."]) + sys.modules.pop("openjarvis.engine.apple_fm_shim", None) + mod = importlib.import_module("openjarvis.engine.apple_fm_shim") + mod = importlib.reload(mod) + yield mod, rec + sys.modules.pop("openjarvis.engine.apple_fm_shim", None) + sys.modules.pop("apple_fm_sdk", None) + + +class TestAppleFmShimSdkMigration: + def test_options_carry_temperature_and_max_tokens(self, shim): + mod, rec = shim + client = TestClient(mod.app) + resp = client.post( + "/v1/chat/completions", + json={ + "messages": [{"role": "user", "content": "hi"}], + "temperature": 0.3, + "max_tokens": 42, + "stream": False, + }, + ) + assert resp.status_code == 200 + # The shim must build a GenerationOptions and pass it as options=. + assert rec["options"], "shim never constructed GenerationOptions" + opt = rec["options"][-1] + assert opt.temperature == 0.3 + assert opt.maximum_response_tokens == 42 + assert rec["respond_calls"][-1]["options"] is opt + assert resp.json()["choices"][0]["message"]["content"] == "Hello from Apple FM." + + def test_stream_passes_options_not_max_tokens(self, shim): + mod, rec = shim + client = TestClient(mod.app) + with client.stream( + "POST", + "/v1/chat/completions", + json={ + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + }, + ) as resp: + assert resp.status_code == 200 + body = "".join(resp.iter_text()) + assert rec["stream_calls"], "stream_response was never called" + # options object passed through; not the legacy max_tokens kwarg. + assert rec["stream_calls"][-1]["options"] is rec["options"][-1] + assert "data: [DONE]" in body + + def test_health_unpacks_is_available_tuple(self, monkeypatch): + monkeypatch.setattr(platform, "system", lambda: "Darwin") + rec = _install_stub_sdk(available=(False, "Apple Intelligence disabled")) + sys.modules.pop("openjarvis.engine.apple_fm_shim", None) + mod = importlib.import_module("openjarvis.engine.apple_fm_shim") + mod = importlib.reload(mod) + try: + client = TestClient(mod.app) + resp = client.get("/health") + assert resp.status_code == 503 + data = resp.json() + assert data["status"] == "unavailable" + assert data["reason"] == "Apple Intelligence disabled" + finally: + sys.modules.pop("openjarvis.engine.apple_fm_shim", None) + sys.modules.pop("apple_fm_sdk", None) + _ = rec