mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-27 21:05:34 +00:00
Andrew Park
e6c47e9532
The orchestrator SFT pipeline was living entirely in the working tree and never got committed. One script in that lineage (clean_sft_data.py) has already been lost with no way to recover it, so track the rest before the same thing happens again: run_sft_fsdp.py full-parameter FSDP trainer (launched via accelerate) sft_tokenize.py conversation -> tokens + assistant-only masking render_sft_data.py record -> chat-template rendering make_splits.py train / holdout / overfit splits chat_orchestrator.py REPL for eyeballing a trained checkpoint's routing format_eval_sample.py, upload_to_braintrust.py scripts/train/ fsdp4.yaml, fsdp8.yaml, eval sbatch toolorchestra/tracing.py was also untracked despite being imported by unified.py and rollout.py, which left the branch broken for a fresh checkout. Add it along with the rest of the ToolOrchestra package split. Also here: mmlu_pro / supergpqa scorers, expert pricing for the OpenRouter Qwen builds (estimates, flagged in-file), an expanded calculator/web_search tool surface, and the reject-sampling clean gate that run_sft_fsdp reads via --require-clean. GRPO is left at its upstream state -- the RL stage hasn't been run yet, so it stays out of this change. Paths in the sbatch are env-driven rather than hardcoded to one cluster home.
251 lines
8.6 KiB
TOML
251 lines
8.6 KiB
TOML
[build-system]
|
||
requires = ["hatchling", "hatch-vcs"]
|
||
build-backend = "hatchling.build"
|
||
|
||
[project]
|
||
name = "OpenJarvis"
|
||
dynamic = ["version"]
|
||
description = "OpenJarvis — modular AI assistant backend with composable intelligence primitives"
|
||
readme = "README.md"
|
||
# Upper bound: numpy 2.2.x (pinned transitively via datasets/pandas) ships no
|
||
# cp314 Windows wheel, so under Python 3.14 uv would compile numpy from source
|
||
# (Meson) and fail on Windows boxes without a C toolchain (#350). Cap to the
|
||
# range that has prebuilt wheels; matches the classifiers (3.10–3.13).
|
||
requires-python = ">=3.10,<3.14"
|
||
license = {text = "Apache-2.0"}
|
||
authors = [
|
||
{name = "Open Jarvis Contributors"},
|
||
]
|
||
classifiers = [
|
||
"Development Status :: 3 - Alpha",
|
||
"Intended Audience :: Developers",
|
||
"License :: OSI Approved :: Apache Software License",
|
||
"Programming Language :: Python :: 3",
|
||
"Programming Language :: Python :: 3.10",
|
||
"Programming Language :: Python :: 3.11",
|
||
"Programming Language :: Python :: 3.12",
|
||
"Programming Language :: Python :: 3.13",
|
||
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
||
]
|
||
dependencies = [
|
||
"braintrust>=0.0.150",
|
||
"click>=8",
|
||
"datasets>=4.5.0",
|
||
"ddgs>=9.11.4",
|
||
"httpx>=0.27",
|
||
"openai>=1.30",
|
||
"nvidia-ml-py>=12.560.30",
|
||
"posthog>=3.0",
|
||
"python-telegram-bot>=22.6",
|
||
"rich>=13",
|
||
"tomli>=2.0; python_version < '3.11'",
|
||
"tomlkit>=0.12",
|
||
"websockets>=15.0.1",
|
||
]
|
||
|
||
[project.optional-dependencies]
|
||
dev = [
|
||
"maturin>=1.12.6",
|
||
"pytest>=8",
|
||
"pytest-asyncio>=0.24",
|
||
"pytest-cov>=5",
|
||
"pytest-xdist>=3",
|
||
"respx>=0.22",
|
||
"ruff>=0.4",
|
||
"pre-commit>=3.0",
|
||
]
|
||
inference-mlx = ["mlx-lm>=0.31.1; sys_platform == 'darwin'"]
|
||
inference-vllm = ["vllm>=0.16.0"]
|
||
inference-cloud = [
|
||
"openai>=1.30",
|
||
"anthropic>=0.30",
|
||
]
|
||
inference-google = [
|
||
"google-genai>=1.0",
|
||
]
|
||
inference-litellm = ["litellm>=1.40"]
|
||
inference-gemma = ["pygemma>=0.1.3"]
|
||
tools-search = [
|
||
"tavily-python>=0.3",
|
||
"ddgs>=9.11.4",
|
||
]
|
||
memory-faiss = [
|
||
"faiss-cpu>=1.7",
|
||
"sentence-transformers>=2.2",
|
||
"numpy>=1.24",
|
||
]
|
||
memory-colbert = [
|
||
"colbert-ai>=0.2",
|
||
"torch>=2.0",
|
||
]
|
||
memory-pdf = ["pdfplumber>=0.10"]
|
||
memory-bm25 = ["rank-bm25>=0.2.2"]
|
||
server = [
|
||
"fastapi>=0.110",
|
||
"uvicorn>=0.30",
|
||
"pydantic>=2.0",
|
||
"python-multipart>=0.0.9",
|
||
]
|
||
desktop = [
|
||
"fastapi>=0.110",
|
||
"uvicorn>=0.30",
|
||
"pydantic>=2.0",
|
||
"python-multipart>=0.0.9",
|
||
"faster-whisper>=1.0",
|
||
]
|
||
openhands = ["openhands-sdk>=1.0; python_version >= '3.12'"]
|
||
gpu-metrics = ["pynvml>=12.0"]
|
||
energy-amd = ["amdsmi>=6.1"]
|
||
energy-apple = ["zeus-ml[apple]"]
|
||
energy-all = ["pynvml>=12.0", "amdsmi>=6.1", "zeus-ml[apple]"]
|
||
orchestrator-training = ["torch>=2.0", "transformers>=4.40"]
|
||
learning-dspy = ["dspy>=2.6"]
|
||
learning-gepa = ["gepa>=0.1"]
|
||
channel-telegram = ["python-telegram-bot>=21.0"]
|
||
channel-discord = ["discord.py>=2.3"]
|
||
channel-slack = ["slack-sdk>=3.27"]
|
||
channel-line = ["line-bot-sdk>=3.0"]
|
||
channel-viber = ["viberbot>=1.0"]
|
||
channel-messenger = ["pymessenger>=0.0.7"]
|
||
channel-reddit = ["praw>=7.0"]
|
||
channel-mastodon = ["Mastodon.py>=1.8"]
|
||
channel-xmpp = ["slixmpp>=1.8"]
|
||
channel-rocketchat = ["rocketchat-API>=1.30"]
|
||
channel-zulip = ["zulip>=0.9"]
|
||
channel-twitter = ["httpx>=0.27"]
|
||
channel-twitch = ["twitchio>=2.6"]
|
||
channel-nostr = ["pynostr>=0.6"]
|
||
channel-twilio = ["twilio>=9.0"]
|
||
channel-gmail = [
|
||
"google-api-python-client>=2.0",
|
||
"google-auth-oauthlib>=1.0",
|
||
"google-auth-httplib2>=0.2",
|
||
]
|
||
browser = ["playwright>=1.40"]
|
||
media = ["openai>=1.30"]
|
||
mining-pearl-vllm = ["docker>=7.0", "httpx>=0.27"]
|
||
pdf = ["pdfplumber>=0.10"]
|
||
scheduler = ["croniter>=2.0"]
|
||
security-signing = ["cryptography>=43"]
|
||
sandbox-wasm = ["wasmtime>=25"]
|
||
sandbox-docker = ["docker>=7.0"]
|
||
dashboard = ["textual>=0.80"]
|
||
speech = ["faster-whisper>=1.0"]
|
||
speech-deepgram = ["deepgram-sdk>=3.0"]
|
||
eval-wandb = ["wandb>=0.17"]
|
||
eval-sheets = ["gspread>=6.0", "google-auth>=2.0"]
|
||
mining-pearl-cpu = [
|
||
# Pearl Python packages (py-pearl-mining, miner-base, pearl-gateway) are
|
||
# not on PyPI yet. They are installed at first `mine init` via the
|
||
# build-from-pin path in src/openjarvis/mining/_install.py:build_from_pin().
|
||
# When Pearl publishes wheels, replace the line below with version pins.
|
||
#
|
||
# The extra remains as a feature-flag namespace so users can run
|
||
# `uv sync --extra mining-pearl-cpu` to opt-in (it succeeds with no
|
||
# additional resolution today; the actual install happens on demand).
|
||
]
|
||
framework-comparison = ["polars>=1.0"]
|
||
docs = [
|
||
"mkdocs>=1.6",
|
||
"mkdocs-material>=9.5",
|
||
"mkdocstrings[python]>=0.25",
|
||
"mkdocs-gen-files>=0.5",
|
||
"mkdocs-literate-nav>=0.6",
|
||
]
|
||
|
||
[project.urls]
|
||
Homepage = "https://github.com/open-jarvis/OpenJarvis"
|
||
Documentation = "https://open-jarvis.github.io/OpenJarvis/"
|
||
Repository = "https://github.com/open-jarvis/OpenJarvis"
|
||
Issues = "https://github.com/open-jarvis/OpenJarvis/issues"
|
||
|
||
[project.scripts]
|
||
jarvis = "openjarvis.cli:main"
|
||
openjarvis-eval = "openjarvis.evals.cli:main"
|
||
|
||
# Version is derived from git tags by hatch-vcs (see #526). For source/editable
|
||
# checkouts this yields the true `git describe` version (e.g. 1.0.3.dev109+g<sha>)
|
||
# rather than a stale static string. CI release builds override this with
|
||
# SETUPTOOLS_SCM_PRETEND_VERSION so the published version equals the pushed tag.
|
||
#
|
||
# setuptools_scm cannot bump custom `.devN` tags (only `.dev0`), so the autotag
|
||
# `vX.Y.Z.devN` tags are deliberately EXCLUDED from version derivation here; the
|
||
# base is taken from the latest plain release tag (vX.Y.Z) and the dev distance
|
||
# is computed from commit count since that release.
|
||
[tool.hatch.version]
|
||
source = "vcs"
|
||
|
||
[tool.hatch.version.raw-options]
|
||
tag_regex = '^v(?P<version>[0-9]+\.[0-9]+\.[0-9]+)$'
|
||
git_describe_command = [
|
||
"git", "describe", "--dirty", "--tags", "--long",
|
||
"--match", "v[0-9]*", "--exclude", "*dev*", "--exclude", "*rc*", "--exclude", "desktop-*",
|
||
]
|
||
# Builds without a git checkout (e.g. the `COPY src/ src/` Docker stages, which
|
||
# never include .git) can't run `git describe`. Without a fallback that would
|
||
# hard-fail the build. Mirror the runtime sentinel in src/openjarvis/__init__.py.
|
||
# Such builds can inject the real version via SETUPTOOLS_SCM_PRETEND_VERSION.
|
||
fallback_version = "0.0.0+unknown"
|
||
|
||
[tool.uv.sources]
|
||
openjarvis-rust = { path = "rust/crates/openjarvis-python" }
|
||
|
||
[tool.hatch.build.targets.wheel]
|
||
packages = ["src/openjarvis"]
|
||
|
||
[tool.hatch.build.targets.wheel.force-include]
|
||
"src/openjarvis/agents/claude_code_runner" = "_node_modules/claude_code_runner"
|
||
"src/openjarvis/channels/whatsapp_baileys_bridge" = "_node_modules/whatsapp_baileys_bridge"
|
||
"scripts/install" = "_install_scripts"
|
||
"deploy/windows" = "_deploy/windows"
|
||
|
||
[tool.pytest.ini_options]
|
||
testpaths = ["tests"]
|
||
markers = [
|
||
"amd: requires AMD GPU",
|
||
"apple: requires Apple Silicon",
|
||
"cloud: requires cloud API keys",
|
||
"docker: requires a working Docker daemon (no GPU required)",
|
||
"live: requires running inference engine",
|
||
"macos15: requires macOS 15+ (Sequoia) for Apple FM",
|
||
"live_channel: requires real channel credentials (env vars)",
|
||
"nvidia: requires NVIDIA GPU",
|
||
"slow: long-running test",
|
||
"hub: downloads real datasets from the HuggingFace Hub at runtime; excluded from the default CI lane (run with -m hub)",
|
||
"live_external: requires HERMES_AGENT_PATH and OPENCLAW_PATH; spawns real foreign-framework subprocesses",
|
||
"modal: requires Modal token + network; runs real swebench harness on Modal",
|
||
]
|
||
|
||
[tool.ruff]
|
||
target-version = "py310"
|
||
src = ["src", "tests"]
|
||
|
||
[tool.ruff.lint]
|
||
select = ["E", "F", "I", "W"]
|
||
|
||
[tool.ruff.lint.per-file-ignores]
|
||
"src/openjarvis/evals/datasets/*.py" = ["E501"]
|
||
"src/openjarvis/evals/scorers/*.py" = ["E501"]
|
||
# hybrid/ is research code with long prompt strings and paradigm-specific
|
||
# config dicts — same relaxation as evals research code above. The ``**`` glob
|
||
# also covers subpackages (e.g. hybrid/skillorchestra/), which the prior
|
||
# ``hybrid/*.py`` glob missed.
|
||
"src/openjarvis/agents/hybrid/**/*.py" = ["E501"]
|
||
# research_loop.py carries long prompt strings too (documented in CLAUDE.md);
|
||
# the ignore was missing here.
|
||
"src/openjarvis/agents/research_loop.py" = ["E501"]
|
||
|
||
[dependency-groups]
|
||
dev = [
|
||
"maturin>=1.12.6",
|
||
]
|
||
# openjarvis_rust is the native PyO3 extension, built from the local Rust
|
||
# workspace. It lives in a uv dependency group (PEP 735) — not the published
|
||
# `desktop` extra — so `uv sync --group desktop-native` builds it from source
|
||
# for the desktop app, while `pip install openjarvis[desktop]` from PyPI does
|
||
# NOT try to resolve openjarvis-rust from PyPI, where it isn't published
|
||
# (dependency groups are excluded from wheel metadata). See #584 / #615.
|
||
desktop-native = [
|
||
"openjarvis-rust",
|
||
]
|