mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-27 21:05:34 +00:00
lemonade: update default host and recommended model
This commit is contained in:
committed by
krypticmouse
parent
36482b1a27
commit
a65cd12ea2
@@ -116,7 +116,7 @@ All providers produce the same output format consumed by agents:
|
||||
| **LM Studio** | `lmstudio` | OpenAI-compatible | 1234 | No (GPU optional) | Desktop GUI, easy model management |
|
||||
| **Exo** | `exo` | OpenAI-compatible | 52415 | No (distributed) | Distributed inference across heterogeneous devices |
|
||||
| **Nexa** | `nexa` | OpenAI-compatible | 18181 | No (CPU/GPU) | On-device inference with GGUF models |
|
||||
| **Lemonade** | `lemonade` | OpenAI-compatible | 8000 | AMD GPU/NPU | AMD consumer GPUs (RDNA), Ryzen AI NPUs |
|
||||
| **Lemonade** | `lemonade` | OpenAI-compatible | 13305 | AMD GPU/NPU | AMD consumer GPUs (RDNA), Ryzen AI NPUs |
|
||||
| **Uzu** | `uzu` | OpenAI-compatible | 8000 | Varies | Uzu inference runtime |
|
||||
| **Apple FM** | `apple_fm` | OpenAI-compatible | 8079 | Apple Silicon | Apple Foundation Model on-device inference |
|
||||
| **LiteLLM** | `litellm` | OpenAI-compatible | — | No | Unified proxy to 100+ LLM providers |
|
||||
@@ -135,7 +135,7 @@ The Ollama backend communicates via Ollama's native HTTP API at `/api/chat` and
|
||||
|
||||
The vLLM backend uses the OpenAI-compatible `/v1/chat/completions` API. It is recommended for datacenter GPUs (NVIDIA A100, H100, L40, A10, A30 and AMD MI300, MI325, MI350, MI355).
|
||||
|
||||
- **Default host:** `http://localhost:8000`
|
||||
- **Default host:** `http://localhost:13305`
|
||||
- **Health check:** `GET /v1/models`
|
||||
- **Tool fallback:** If the server returns HTTP 400 when tools are included, the engine automatically retries without tools
|
||||
|
||||
@@ -204,7 +204,7 @@ The Nexa backend connects to the Nexa SDK on-device inference server via a FastA
|
||||
|
||||
The Lemonade backend connects to the [Lemonade](https://lemonade-server.ai/) inference server, which is optimized for AMD consumer GPUs (RDNA architecture) and Ryzen AI Neural Processing Units (NPUs). It uses the OpenAI-compatible `/v1/chat/completions` API.
|
||||
|
||||
- **Default host:** `http://localhost:8000`
|
||||
- **Default host:** `http://localhost:13305`
|
||||
- **Health check:** `GET /v1/models`
|
||||
- **Install:** Visit [lemonade-server.ai](https://lemonade-server.ai/) for platform-specific installation instructions
|
||||
- **Best for:** Ryzen AI GPUs and NPUs, and AMD-based desktop and laptop systems
|
||||
@@ -357,7 +357,7 @@ host = "http://localhost:30000"
|
||||
# binary_path = ""
|
||||
|
||||
# [engine.lemonade]
|
||||
# host = "http://localhost:8000"
|
||||
# host = "http://localhost:13305"
|
||||
```
|
||||
|
||||
The `EngineConfig` dataclass and its per-engine sub-dataclasses map these settings:
|
||||
@@ -370,7 +370,7 @@ The `EngineConfig` dataclass and its per-engine sub-dataclasses map these settin
|
||||
| `SGLangEngineConfig` | `host` | `http://localhost:30000` | SGLang server URL |
|
||||
| `LlamaCppEngineConfig` | `host` | `http://localhost:8080` | llama.cpp server URL |
|
||||
| `LlamaCppEngineConfig` | `binary_path` | `""` | Path to llama.cpp binary (for managed mode) |
|
||||
| `LemonadeEngineConfig` | `host` | `http://localhost:8000` | Lemonade server URL |
|
||||
| `LemonadeEngineConfig` | `host` | `http://localhost:13305` | Lemonade server URL |
|
||||
|
||||
!!! note "Backward compatibility"
|
||||
The old flat field names `ollama_host`, `vllm_host`, `llamacpp_host`, `llamacpp_path`, `sglang_host`, and `lemonade_host` under `[engine]` are still accepted as backward-compatible properties on `EngineConfig`. New configurations should use the nested sub-section format.
|
||||
|
||||
@@ -280,14 +280,14 @@ _MODEL_TIERS = [
|
||||
(64, "qwen3.5:27b"),
|
||||
]
|
||||
_MODEL_TIER_FALLBACK = "qwen3.5:27b"
|
||||
_LEMONADE_DEFAULT_MODEL = "Qwen3.6-35B-A3B-GGUF"
|
||||
|
||||
|
||||
def recommend_model(hw: HardwareInfo, engine: str) -> str:
|
||||
"""Suggest the best Qwen3.5 model that fits the detected hardware.
|
||||
"""Suggest a default model for the selected engine and hardware.
|
||||
|
||||
Uses an explicit tier table mapping available memory to model size.
|
||||
Falls back to scanning the full catalog if the tiered model is not
|
||||
compatible with the selected engine.
|
||||
For Lemonade, prefer the validated Qwen3.6 35B A3B GGUF default.
|
||||
For other local engines, use the generic Qwen3.5 tier mapping.
|
||||
"""
|
||||
from openjarvis.intelligence.model_catalog import BUILTIN_MODELS
|
||||
|
||||
@@ -295,6 +295,9 @@ def recommend_model(hw: HardwareInfo, engine: str) -> str:
|
||||
if available_gb <= 0:
|
||||
return ""
|
||||
|
||||
if engine == "lemonade":
|
||||
return _LEMONADE_DEFAULT_MODEL
|
||||
|
||||
# Build a lookup for quick engine-compatibility checks
|
||||
catalog = {spec.model_id: spec for spec in BUILTIN_MODELS}
|
||||
|
||||
@@ -422,7 +425,7 @@ class GemmaCppEngineConfig:
|
||||
class LemonadeEngineConfig:
|
||||
"""Per-engine config for Lemonade."""
|
||||
|
||||
host: str = "http://localhost:8000"
|
||||
host: str = "http://localhost:13305"
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -13,7 +13,7 @@ _ENGINES = {
|
||||
"nexa": ("NexaEngine", "http://localhost:18181", "/v1"),
|
||||
"uzu": ("UzuEngine", "http://localhost:8000", ""),
|
||||
"apple_fm": ("AppleFmEngine", "http://localhost:8079", "/v1"),
|
||||
"lemonade": ("LemonadeEngine", "http://localhost:8000", "/v1"),
|
||||
"lemonade": ("LemonadeEngine", "http://localhost:13305", "/v1"),
|
||||
}
|
||||
|
||||
for _key, (_cls_name, _default_host, _api_prefix) in _ENGINES.items():
|
||||
|
||||
@@ -37,10 +37,12 @@ class TestDefaults:
|
||||
assert ec.vllm.host == "http://localhost:8000"
|
||||
assert ec.sglang.host == "http://localhost:30000"
|
||||
assert ec.llamacpp.host == "http://localhost:8080"
|
||||
assert ec.lemonade.host == "http://localhost:13305"
|
||||
assert ec.llamacpp.binary_path == ""
|
||||
# Backward-compat properties still work
|
||||
assert ec.ollama_host == ""
|
||||
assert ec.vllm_host == "http://localhost:8000"
|
||||
assert ec.lemonade_host == "http://localhost:13305"
|
||||
|
||||
|
||||
class TestRecommendEngine:
|
||||
@@ -93,6 +95,17 @@ class TestTomlLoading:
|
||||
assert cfg.engine.default == "vllm"
|
||||
assert cfg.memory.default_backend == "faiss"
|
||||
|
||||
def test_loads_nested_lemonade_host_override(self, tmp_path: Path) -> None:
|
||||
toml_file = tmp_path / "config.toml"
|
||||
toml_file.write_text(
|
||||
'[engine]\ndefault = "lemonade"\n\n'
|
||||
'[engine.lemonade]\nhost = "http://custom-lemonade:19000"\n'
|
||||
)
|
||||
cfg = load_config(toml_file)
|
||||
assert cfg.engine.default == "lemonade"
|
||||
assert cfg.engine.lemonade.host == "http://custom-lemonade:19000"
|
||||
assert cfg.engine.lemonade_host == "http://custom-lemonade:19000"
|
||||
|
||||
|
||||
class TestGenerateToml:
|
||||
def test_contains_engine_section(self) -> None:
|
||||
@@ -213,6 +226,7 @@ class TestNestedEngineConfig:
|
||||
assert ec.vllm.host == "http://localhost:8000"
|
||||
assert ec.sglang.host == "http://localhost:30000"
|
||||
assert ec.llamacpp.host == "http://localhost:8080"
|
||||
assert ec.lemonade.host == "http://localhost:13305"
|
||||
assert ec.llamacpp.binary_path == ""
|
||||
|
||||
def test_backward_compat_setter(self) -> None:
|
||||
@@ -250,6 +264,17 @@ class TestNestedEngineConfig:
|
||||
assert cfg.engine.ollama.host == "http://old:11434"
|
||||
assert cfg.engine.vllm.host == "http://old:8000"
|
||||
|
||||
def test_loads_old_flat_lemonade_host(self, tmp_path: Path) -> None:
|
||||
toml_file = tmp_path / "config.toml"
|
||||
toml_file.write_text(
|
||||
'[engine]\ndefault = "lemonade"\n'
|
||||
'lemonade_host = "http://legacy-lemonade:19191"\n'
|
||||
)
|
||||
cfg = load_config(toml_file)
|
||||
assert cfg.engine.default == "lemonade"
|
||||
assert cfg.engine.lemonade.host == "http://legacy-lemonade:19191"
|
||||
assert cfg.engine.lemonade_host == "http://legacy-lemonade:19191"
|
||||
|
||||
|
||||
class TestNestedLearningConfig:
|
||||
def test_defaults(self) -> None:
|
||||
|
||||
@@ -86,6 +86,20 @@ class TestRecommendModelGpu:
|
||||
# available = 288 GB → tier fallback qwen3.5:27b, valid for vllm
|
||||
assert result == "qwen3.5:27b"
|
||||
|
||||
def test_amd_lemonade_picks_qwen36_35b_a3b(self) -> None:
|
||||
hw = HardwareInfo(
|
||||
platform="linux",
|
||||
ram_gb=64.0,
|
||||
gpu=GpuInfo(
|
||||
vendor="amd",
|
||||
name="Radeon RX 7900 XTX",
|
||||
vram_gb=24.0,
|
||||
count=1,
|
||||
),
|
||||
)
|
||||
result = recommend_model(hw, "lemonade")
|
||||
assert result == "Qwen3.6-35B-A3B-GGUF"
|
||||
|
||||
|
||||
class TestRecommendModelEdgeCases:
|
||||
"""Edge cases."""
|
||||
|
||||
@@ -32,7 +32,7 @@ _OPENAI_COMPAT_ENGINES = [
|
||||
("nexa", "http://testhost:18181", NexaEngine),
|
||||
("uzu", "http://testhost:8000", UzuEngine),
|
||||
("apple_fm", "http://testhost:8079", AppleFmEngine),
|
||||
("lemonade", "http://testhost:8000", LemonadeEngine),
|
||||
("lemonade", "http://testhost:13305", LemonadeEngine),
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ from openjarvis.engine.openai_compat_engines import LemonadeEngine
|
||||
@pytest.fixture()
|
||||
def engine() -> LemonadeEngine:
|
||||
EngineRegistry.register_value("lemonade", LemonadeEngine)
|
||||
return LemonadeEngine(host="http://testhost:8000")
|
||||
return LemonadeEngine(host="http://testhost:13305")
|
||||
|
||||
|
||||
class TestLemonadeEngineBasics:
|
||||
@@ -27,7 +27,7 @@ class TestLemonadeEngineBasics:
|
||||
assert LemonadeEngine.engine_id == "lemonade"
|
||||
|
||||
def test_default_host(self) -> None:
|
||||
assert LemonadeEngine._default_host == "http://localhost:8000"
|
||||
assert LemonadeEngine._default_host == "http://localhost:13305"
|
||||
|
||||
def test_api_prefix(self) -> None:
|
||||
assert LemonadeEngine._api_prefix == "/v1"
|
||||
@@ -36,11 +36,19 @@ class TestLemonadeEngineBasics:
|
||||
EngineRegistry.register_value("lemonade", LemonadeEngine)
|
||||
assert EngineRegistry.get("lemonade") is LemonadeEngine
|
||||
|
||||
def test_env_var_overrides_default_host(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("LEMONADE_HOST", "http://env-lemonade:17777")
|
||||
engine = LemonadeEngine()
|
||||
try:
|
||||
assert engine._host == "http://env-lemonade:17777"
|
||||
finally:
|
||||
engine.close()
|
||||
|
||||
|
||||
class TestLemonadeGenerate:
|
||||
def test_generate_uses_v1_prefix(self, engine: LemonadeEngine) -> None:
|
||||
with respx.mock:
|
||||
respx.post("http://testhost:8000/v1/chat/completions").mock(
|
||||
respx.post("http://testhost:13305/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
@@ -67,7 +75,7 @@ class TestLemonadeGenerate:
|
||||
|
||||
def test_generate_connection_error(self, engine: LemonadeEngine) -> None:
|
||||
with respx.mock:
|
||||
respx.post("http://testhost:8000/v1/chat/completions").mock(
|
||||
respx.post("http://testhost:13305/v1/chat/completions").mock(
|
||||
side_effect=httpx.ConnectError("refused")
|
||||
)
|
||||
with pytest.raises(EngineConnectionError):
|
||||
@@ -80,14 +88,14 @@ class TestLemonadeGenerate:
|
||||
class TestLemonadeHealth:
|
||||
def test_health_true(self, engine: LemonadeEngine) -> None:
|
||||
with respx.mock:
|
||||
respx.get("http://testhost:8000/v1/models").mock(
|
||||
respx.get("http://testhost:13305/v1/models").mock(
|
||||
return_value=httpx.Response(200, json={"data": []})
|
||||
)
|
||||
assert engine.health() is True
|
||||
|
||||
def test_health_false(self, engine: LemonadeEngine) -> None:
|
||||
with respx.mock:
|
||||
respx.get("http://testhost:8000/v1/models").mock(
|
||||
respx.get("http://testhost:13305/v1/models").mock(
|
||||
side_effect=httpx.ConnectError("refused")
|
||||
)
|
||||
assert engine.health() is False
|
||||
@@ -96,7 +104,7 @@ class TestLemonadeHealth:
|
||||
class TestLemonadeListModels:
|
||||
def test_list_models_uses_v1_prefix(self, engine: LemonadeEngine) -> None:
|
||||
with respx.mock:
|
||||
respx.get("http://testhost:8000/v1/models").mock(
|
||||
respx.get("http://testhost:13305/v1/models").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
|
||||
Reference in New Issue
Block a user