admin/OpenJarvis
1
0
Fork 0
mirror of https://github.com/open-jarvis/OpenJarvis.git synced 2026-07-29 18:40:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: hardware-aware model recommendation in jarvis init (#188)

Browse Source 
Closes #132 — jarvis init now recommends a Qwen3 dense model based on
available memory using an explicit tier table:

  ≤8 GB  → qwen3:1.7b  (~1.1 GB)
  8-16GB → qwen3:4b    (~2.5 GB)
 16-32GB → qwen3:8b    (~5.2 GB)
  32GB+  → qwen3:14b   (~9.3 GB)

Adds qwen3:0.6b, 1.7b, 4b, 14b, 30b to the model catalog. Falls back
to scanning all compatible models if the tiered pick doesn't support
the selected engine. Shows the user why the model was chosen and how
to download later if declined.
This commit is contained in:
Robby Manihani
2026-04-03 22:19:27 -07:00
committed by GitHub
parent e0dccd1b8d
commit 35b8991cc2
4 changed files with 191 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/openjarvis/cli/init_cmd.py
+11 -2
View File
@@ -505,14 +505,23 @@ sources = ["hackernews", "news_rss"]
else:
spec = find_model_spec(model)
size_gb = estimated_download_gb(spec.parameter_count_b) if spec else 0
from openjarvis.core.config import _available_memory_gb
avail = _available_memory_gb(hw)
console.print(
f"\n [bold]Recommended model:[/bold] {model} (~{size_gb:.1f} GB estimated)"
f"\n [bold]Recommended model:[/bold] {model} (~{size_gb:.1f} GB)"
f" [dim](selected for {avail:.0f} GB available memory)[/dim]"
)
if not no_download and spec:
prompt = f" Download {model} (~{size_gb:.1f} GB estimated) now?"
prompt = f" Download {model} (~{size_gb:.1f} GB) now?"
if click.confirm(prompt, default=True):
_do_download(selected_engine, model, spec, console)
else:
console.print(
f"\n Skipped. Download later with:\n"
f" [bold]jarvis model pull {model}[/bold]"
)
if not skip_scan:
_quick_privacy_check(console)
src/openjarvis/core/config.py
+51 -24
View File
@@ -228,40 +228,67 @@ def recommend_engine(hw: HardwareInfo) -> str:
return "llamacpp"
def recommend_model(hw: HardwareInfo, engine: str) -> str:
"""Suggest the largest Qwen3.5 model that fits the detected hardware.
def _available_memory_gb(hw: HardwareInfo) -> float:
"""Return usable memory in GB for model loading."""
gpu = hw.gpu
if gpu and gpu.vram_gb > 0:
return gpu.vram_gb * max(gpu.count, 1) * 0.9
if hw.ram_gb > 0:
return (hw.ram_gb - 4) * 0.8
return 0.0
Uses llmfit-style VRAM estimation: Q4_K_M quantization is ~0.5 bytes/param
with 10% overhead. For MoE models Ollama loads full model weights, so we
use ``parameter_count_b`` (total), not ``active_parameter_count_b``.
# Explicit tier table: (max_ram_gb, model_id).
# Walked in order — first tier where available_gb <= max_ram is chosen.
_MODEL_TIERS = [
(8, "qwen3:1.7b"),
(16, "qwen3:4b"),
(32, "qwen3:8b"),
(64, "qwen3:14b"),
]
_MODEL_TIER_FALLBACK = "qwen3:14b"
def recommend_model(hw: HardwareInfo, engine: str) -> str:
"""Suggest a Qwen3 dense model that fits the detected hardware.
Uses an explicit tier table mapping available memory to model size.
Falls back to scanning the full catalog if the tiered model is not
compatible with the selected engine.
"""
from openjarvis.intelligence.model_catalog import BUILTIN_MODELS
# Determine available memory in GB
gpu = hw.gpu
if gpu and gpu.vram_gb > 0:
available_gb = gpu.vram_gb * max(gpu.count, 1) * 0.9
elif hw.ram_gb > 0:
available_gb = (hw.ram_gb - 4) * 0.8
else:
available_gb = _available_memory_gb(hw)
if available_gb <= 0:
return ""
# Filter Qwen3.5 models compatible with the chosen engine
# Build a lookup for quick engine-compatibility checks
catalog = {spec.model_id: spec for spec in BUILTIN_MODELS}
# Try explicit tier mapping first
model_id = _MODEL_TIER_FALLBACK
for max_ram, tier_model in _MODEL_TIERS:
if available_gb <= max_ram:
model_id = tier_model
break
spec = catalog.get(model_id)
if spec and engine in spec.supported_engines:
return model_id
# Fallback: scan all Qwen3 dense models for engine compatibility
candidates = [
spec
for spec in BUILTIN_MODELS
if spec.provider == "alibaba"
and spec.model_id.startswith("qwen3.5:")
and engine in spec.supported_engines
s
for s in BUILTIN_MODELS
if s.provider == "alibaba"
and s.model_id.startswith("qwen3:")
and engine in s.supported_engines
]
# Sort by parameter count descending — pick the largest that fits
candidates.sort(key=lambda s: s.parameter_count_b, reverse=True)
for spec in candidates:
estimated_gb = spec.parameter_count_b * 0.5 * 1.1
for s in candidates:
estimated_gb = s.parameter_count_b * 0.5 * 1.1
if estimated_gb <= available_gb:
return spec.model_id
return s.model_id
return ""
src/openjarvis/intelligence/model_catalog.py
+62 -1
View File
@@ -11,18 +11,79 @@ BUILTIN_MODELS: List[ModelSpec] = [
# -----------------------------------------------------------------------
# Local models — Dense
# -----------------------------------------------------------------------
ModelSpec(
model_id="qwen3:0.6b",
name="Qwen3 0.6B",
parameter_count_b=0.6,
context_length=40960,
supported_engines=("ollama", "vllm", "llamacpp", "sglang", "mlx", "lemonade"),
provider="alibaba",
metadata={
"architecture": "dense",
"hf_repo": "Qwen/Qwen3-0.6B",
},
),
ModelSpec(
model_id="qwen3:1.7b",
name="Qwen3 1.7B",
parameter_count_b=1.7,
context_length=40960,
supported_engines=("ollama", "vllm", "llamacpp", "sglang", "mlx", "lemonade"),
provider="alibaba",
metadata={
"architecture": "dense",
"hf_repo": "Qwen/Qwen3-1.7B",
},
),
ModelSpec(
model_id="qwen3:4b",
name="Qwen3 4B",
parameter_count_b=4.0,
context_length=262144,
supported_engines=("ollama", "vllm", "llamacpp", "sglang", "mlx", "lemonade"),
provider="alibaba",
metadata={
"architecture": "dense",
"hf_repo": "Qwen/Qwen3-4B",
},
),
ModelSpec(
model_id="qwen3:8b",
name="Qwen3 8B",
parameter_count_b=8.2,
context_length=32768,
supported_engines=("vllm", "ollama", "llamacpp", "sglang", "lemonade"),
supported_engines=("vllm", "ollama", "llamacpp", "sglang", "mlx", "lemonade"),
provider="alibaba",
metadata={
"architecture": "dense",
"hf_repo": "Qwen/Qwen3-8B",
},
),
ModelSpec(
model_id="qwen3:14b",
name="Qwen3 14B",
parameter_count_b=14.0,
context_length=40960,
supported_engines=("ollama", "vllm", "llamacpp", "sglang", "lemonade"),
provider="alibaba",
metadata={
"architecture": "dense",
"hf_repo": "Qwen/Qwen3-14B",
},
),
ModelSpec(
model_id="qwen3:30b",
name="Qwen3 30B",
parameter_count_b=30.0,
context_length=262144,
min_vram_gb=18.0,
supported_engines=("ollama", "vllm", "sglang"),
provider="alibaba",
metadata={
"architecture": "dense",
"hf_repo": "Qwen/Qwen3-30B",
},
),
ModelSpec(
model_id="qwen3:32b",
name="Qwen3 32B",
tests/core/test_recommend_model.py
+67 -72
View File
@@ -5,98 +5,88 @@ from __future__ import annotations
from openjarvis.core.config import GpuInfo, HardwareInfo, recommend_model
class TestRecommendModelTiers:
"""Tier-based model recommendation (Qwen3 dense)."""
def test_8gb_ram_picks_qwen3_1_7b(self) -> None:
hw = HardwareInfo(platform="linux", ram_gb=8.0, gpu=None)
result = recommend_model(hw, "llamacpp")
# available = (8 - 4) * 0.8 = 3.2 GB → ≤8 tier → qwen3:1.7b
assert result == "qwen3:1.7b"
def test_16gb_ram_picks_qwen3_4b(self) -> None:
hw = HardwareInfo(platform="linux", ram_gb=16.0, gpu=None)
result = recommend_model(hw, "llamacpp")
# available = (16 - 4) * 0.8 = 9.6 GB → ≤16 tier → qwen3:4b
assert result == "qwen3:4b"
def test_32gb_ram_picks_qwen3_8b(self) -> None:
hw = HardwareInfo(platform="linux", ram_gb=32.0, gpu=None)
result = recommend_model(hw, "llamacpp")
# available = (32 - 4) * 0.8 = 22.4 GB → ≤32 tier → qwen3:8b
assert result == "qwen3:8b"
def test_64gb_ram_picks_qwen3_14b(self) -> None:
hw = HardwareInfo(platform="linux", ram_gb=64.0, gpu=None)
result = recommend_model(hw, "llamacpp")
# available = (64 - 4) * 0.8 = 48 GB → >32 → qwen3:14b
assert result == "qwen3:14b"
class TestRecommendModelGpu:
"""GPU-based model recommendation."""
def test_24gb_gpu_picks_qwen35_35b(self) -> None:
def test_24gb_gpu_picks_qwen3_14b(self) -> None:
hw = HardwareInfo(
platform="linux",
ram_gb=64.0,
gpu=GpuInfo(vendor="nvidia", name="RTX 4090", vram_gb=24.0, count=1),
)
result = recommend_model(hw, "ollama")
# 35B * 0.5 * 1.1 = 19.25 GB; available = 24 * 0.9 = 21.6 → fits
assert result == "qwen3.5:35b"
# available = 24 * 0.9 = 21.6 GB → ≤32 tier → qwen3:8b
# Actually 21.6 ≤ 32, so tier is qwen3:8b
assert result == "qwen3:8b"
def test_8gb_gpu_picks_qwen35_14b(self) -> None:
def test_8gb_gpu_picks_qwen3_1_7b(self) -> None:
hw = HardwareInfo(
platform="linux",
ram_gb=32.0,
gpu=GpuInfo(vendor="nvidia", name="RTX 3070", vram_gb=8.0, count=1),
)
result = recommend_model(hw, "ollama")
# 14B * 0.5 * 1.1 = 7.7 GB; available = 8 * 0.9 = 7.2 → too big
# 8B * 0.5 * 1.1 = 4.4 GB; available = 7.2 → fits
assert result == "qwen3.5:9b"
# available = 8 * 0.9 = 7.2 GB → ≤8 tier → qwen3:1.7b
assert result == "qwen3:1.7b"
def test_4gb_gpu_picks_qwen35_4b(self) -> None:
def test_4gb_gpu_picks_qwen3_1_7b(self) -> None:
hw = HardwareInfo(
platform="linux",
ram_gb=16.0,
gpu=GpuInfo(vendor="nvidia", name="GTX 1650", vram_gb=4.0, count=1),
)
result = recommend_model(hw, "ollama")
# 4B * 0.5 * 1.1 = 2.2 GB; available = 4 * 0.9 = 3.6 → fits
assert result == "qwen3.5:4b"
# available = 4 * 0.9 = 3.6 GB → ≤8 tier → qwen3:1.7b
assert result == "qwen3:1.7b"
def test_2gb_gpu_picks_qwen35_3b(self) -> None:
hw = HardwareInfo(
platform="linux",
ram_gb=8.0,
gpu=GpuInfo(vendor="nvidia", name="GTX 750", vram_gb=2.0, count=1),
)
result = recommend_model(hw, "ollama")
# 3B * 0.5 * 1.1 = 1.65 GB; available = 2 * 0.9 = 1.8 → fits
assert result == "qwen3.5:2b"
def test_multi_gpu_picks_larger_model(self) -> None:
def test_multi_gpu_picks_qwen3_14b(self) -> None:
hw = HardwareInfo(
platform="linux",
ram_gb=256.0,
gpu=GpuInfo(vendor="nvidia", name="A100", vram_gb=80.0, count=2),
)
result = recommend_model(hw, "vllm")
# available = 80 * 2 * 0.9 = 144 GB
# 397B * 0.5 * 1.1 = 218.35 → too big
# 122B * 0.5 * 1.1 = 67.1 → fits
assert result == "qwen3.5:122b"
# available = 80 * 2 * 0.9 = 144 GB → >64 → qwen3:14b (tier fallback)
assert result == "qwen3:14b"
def test_huge_vram_picks_397b(self) -> None:
def test_huge_vram_falls_back_to_scan(self) -> None:
hw = HardwareInfo(
platform="linux",
ram_gb=512.0,
gpu=GpuInfo(vendor="nvidia", name="H100", vram_gb=80.0, count=4),
)
result = recommend_model(hw, "vllm")
# available = 80 * 4 * 0.9 = 288 GB
# 397B * 0.5 * 1.1 = 218.35 → fits
assert result == "qwen3.5:397b"
class TestRecommendModelCpuOnly:
"""CPU-only model recommendation."""
def test_cpu_only_16gb_ram(self) -> None:
hw = HardwareInfo(platform="linux", ram_gb=16.0, gpu=None)
result = recommend_model(hw, "llamacpp")
# available = (16 - 4) * 0.8 = 9.6 GB
# 27B * 0.5 * 1.1 = 14.85 → too big
# 9B * 0.5 * 1.1 = 4.95 → fits
assert result == "qwen3.5:9b"
def test_cpu_only_8gb_ram(self) -> None:
hw = HardwareInfo(platform="linux", ram_gb=8.0, gpu=None)
result = recommend_model(hw, "llamacpp")
# available = (8 - 4) * 0.8 = 3.2 GB
# 8B * 0.5 * 1.1 = 4.4 → too big
# 4B * 0.5 * 1.1 = 2.2 → fits
assert result == "qwen3.5:4b"
def test_cpu_only_4gb_ram(self) -> None:
hw = HardwareInfo(platform="linux", ram_gb=4.0, gpu=None)
result = recommend_model(hw, "llamacpp")
# available = (4 - 4) * 0.8 = 0 → nothing fits
assert result == ""
# available = 288 GB → tier gives qwen3:14b, but scan finds larger
# Tier fallback is qwen3:14b — it's valid for vllm
assert result == "qwen3:14b"
class TestRecommendModelEdgeCases:
@@ -106,16 +96,18 @@ class TestRecommendModelEdgeCases:
hw = HardwareInfo(platform="linux", ram_gb=0.0, gpu=None)
assert recommend_model(hw, "ollama") == ""
def test_engine_filter(self) -> None:
"""397b is not supported on ollama, only vllm/sglang."""
hw = HardwareInfo(
platform="linux",
ram_gb=512.0,
gpu=GpuInfo(vendor="nvidia", name="H100", vram_gb=80.0, count=4),
)
# With ollama, 397b is excluded (only vllm, sglang)
result = recommend_model(hw, "ollama")
assert result == "qwen3.5:122b"
def test_4gb_ram_picks_qwen3_1_7b(self) -> None:
hw = HardwareInfo(platform="linux", ram_gb=6.0, gpu=None)
result = recommend_model(hw, "llamacpp")
# available = (6 - 4) * 0.8 = 1.6 GB → ≤8 tier → qwen3:1.7b
# 1.7 * 0.5 * 1.1 = 0.935 → fits in 1.6
assert result == "qwen3:1.7b"
def test_very_low_ram_returns_empty(self) -> None:
hw = HardwareInfo(platform="linux", ram_gb=4.0, gpu=None)
result = recommend_model(hw, "llamacpp")
# available = (4 - 4) * 0.8 = 0 → nothing
assert result == ""
class TestRecommendModelMlx:
@@ -128,7 +120,9 @@ class TestRecommendModelMlx:
gpu=GpuInfo(vendor="apple", name="Apple M1", vram_gb=8.0, count=1),
)
result = recommend_model(hw, "mlx")
assert result == "qwen3.5:9b"
# available = 8 * 0.9 = 7.2 GB → ≤8 tier → qwen3:1.7b
# But qwen3:1.7b supports mlx → good
assert result == "qwen3:1.7b"
def test_apple_silicon_16gb_mlx(self) -> None:
hw = HardwareInfo(
@@ -137,10 +131,8 @@ class TestRecommendModelMlx:
gpu=GpuInfo(vendor="apple", name="Apple M2", vram_gb=16.0, count=1),
)
result = recommend_model(hw, "mlx")
# available = 16 * 0.9 = 14.4 GB
# 27B * 0.5 * 1.1 = 14.85 → too big
# 9B * 0.5 * 1.1 = 4.95 → fits
assert result == "qwen3.5:9b"
# available = 16 * 0.9 = 14.4 GB → ≤16 tier → qwen3:4b
assert result == "qwen3:4b"
def test_apple_silicon_32gb_mlx(self) -> None:
hw = HardwareInfo(
@@ -149,7 +141,8 @@ class TestRecommendModelMlx:
gpu=GpuInfo(vendor="apple", name="Apple M2 Pro", vram_gb=32.0, count=1),
)
result = recommend_model(hw, "mlx")
assert result == "qwen3.5:27b"
# available = 32 * 0.9 = 28.8 GB → ≤32 tier → qwen3:8b
assert result == "qwen3:8b"
def test_apple_silicon_64gb_mlx(self) -> None:
hw = HardwareInfo(
@@ -158,4 +151,6 @@ class TestRecommendModelMlx:
gpu=GpuInfo(vendor="apple", name="Apple M2 Max", vram_gb=64.0, count=1),
)
result = recommend_model(hw, "mlx")
assert result == "qwen3.5:27b"
# available = 64 * 0.9 = 57.6 GB → ≤64 tier → qwen3:14b
# qwen3:14b doesn't support mlx → fallback scan finds qwen3:8b
assert result == "qwen3:8b"