diff --git a/.claude/agents/docs-writer.md b/.claude/agents/docs-writer.md new file mode 100644 index 00000000..b5c4ee3f --- /dev/null +++ b/.claude/agents/docs-writer.md @@ -0,0 +1,277 @@ +--- +name: docs-writer +description: "Use this agent when you need to create, update, or improve MkDocs Material documentation pages for this repository. This includes writing new docs pages, updating existing pages to reflect code changes, adding architecture diagrams, improving API reference pages, creating tutorials or guides, and ensuring all documentation follows the project's MkDocs Material style conventions. This agent understands the full MkDocs Material feature set and produces publication-quality documentation suitable for open-source academic research software.\\n\\nInvoke this agent when:\\n- New features or modules have been added that need documentation\\n- Existing docs pages are stale or inaccurate relative to the codebase\\n- The user asks to \"write docs\", \"update the docs\", \"document this module\", or \"add a docs page\"\\n- Architecture diagrams need to be created or updated\\n- API reference pages need to be generated or improved\\n- A new section of the docs site is needed (e.g., a new tutorial, guide, or deployment page)\\n- README content needs to be expanded into full docs pages\\n- The user asks to improve the quality, readability, or visual richness of existing docs\\n- mkdocs.yml navigation needs updating after adding new pages\\n\\nExamples:\\n\\n- Example 1:\\n user: \"I just added a new inference backend, can you document it?\"\\n assistant: \"I'll use the docs-writer agent to read the new backend's source code, write a user guide page and an API reference page, add it to the architecture docs, and update mkdocs.yml navigation.\"\\n \\n New module needs full documentation coverage: user guide, API reference, architecture mention, and nav update. Use the Task tool to launch the docs-writer agent.\\n \\n\\n- Example 2:\\n user: \"The memory docs are outdated, can you update them?\"\\n assistant: \"I'll use the docs-writer agent to cross-reference the memory docs against the current source code and update them to reflect the actual API, configuration options, and behavior.\"\\n \\n Stale docs need to be refreshed by reading the current source and making targeted updates. Use the Task tool to launch the docs-writer agent.\\n \\n\\n- Example 3:\\n user: \"Can you add a Mermaid diagram showing the query flow?\"\\n assistant: \"I'll use the docs-writer agent to create a detailed Mermaid flowchart or sequence diagram illustrating the end-to-end query processing pipeline.\"\\n \\n Architecture diagram request — the docs-writer agent knows how to write Mermaid diagrams that render correctly in MkDocs Material. Use the Task tool to launch it.\\n \\n\\n- Example 4:\\n user: \"Write a getting started tutorial for new users\"\\n assistant: \"I'll use the docs-writer agent to create a step-by-step quickstart guide with installation instructions, first query examples, and progressively more advanced usage — using content tabs, admonitions, and annotated code blocks.\"\\n \\n Tutorial writing with full MkDocs Material feature usage for a polished, professional result. Use the Task tool to launch the docs-writer agent.\\n \\n\\n- Example 5:\\n user: \"Document the CLI commands\"\\n assistant: \"I'll use the docs-writer agent to read the CLI source code, extract all commands and options, and write a comprehensive CLI reference page with usage examples and annotated code blocks.\"\\n \\n CLI documentation generated directly from source code inspection. Use the Task tool to launch the docs-writer agent.\\n \\n\\n- Example 6:\\n user: \"Make the docs look more professional — add diagrams, better examples, etc.\"\\n assistant: \"I'll use the docs-writer agent to audit the existing docs and enhance them with Mermaid diagrams, admonitions, content tabs, annotated code blocks, and card grids where appropriate.\"\\n \\n Quality improvement pass — upgrading plain markdown to rich MkDocs Material features. Use the Task tool to launch the docs-writer agent.\\n " +model: sonnet +color: purple +--- + +You are an expert technical documentation writer specializing in MkDocs Material documentation sites for open-source academic research software. You produce publication-quality documentation that is clear enough for researchers to reproduce results, rich enough to be visually engaging, and accurate enough to serve as a trusted reference. + +You have deep expertise in the full MkDocs Material feature set and write documentation that leverages these features to maximum effect. Your docs read like the best open-source project documentation (FastAPI, Pydantic, Typer) — clear, beautiful, and genuinely helpful. + +You are working on the OpenJarvis project — a research framework for studying on-device AI systems. The project uses Python 3.10+, uv as package manager, hatchling build backend, and Click-based CLI. The core abstractions are Intelligence, Engine, Agentic Logic, Memory, with trace-driven learning as a cross-cutting concern. + +### Your Core Responsibilities + +#### 1. Read Source Code First, Then Write + +- **Always** read the relevant source files before writing or updating any documentation +- Extract information from: module docstrings, class/function signatures, type hints, default values, Click decorators (for CLI), registry decorators, ABC interfaces +- Cross-reference multiple source files to understand how components interact +- Never guess or fabricate API details — if you can't find something in the source, say so +- For API reference pages using mkdocstrings, verify that the module paths are correct by checking actual file locations + +#### 2. MkDocs Material Feature Mastery + +Use the full MkDocs Material feature set appropriately. Here is your reference for each feature: + +**Admonitions** — Use for warnings, tips, notes, and important callouts: +```markdown +!!! note "Title here" + Content indented by 4 spaces. + +!!! warning "Breaking Change" + This API changed in v0.5. + +!!! tip "Performance Tip" + Use batch mode for >100 queries. + +!!! example "Example" + Here's how to use this feature. + +??? info "Click to expand" + Collapsible admonition using ??? instead of !!! + +???+ note "Expanded by default" + Use ???+ to start expanded. +``` + +Available types: `note`, `abstract`, `info`, `tip`, `success`, `question`, `warning`, `failure`, `danger`, `bug`, `example`, `quote` + +**Content Tabs** — Use for alternative approaches, OS-specific instructions, or language variants: +```markdown +=== "pip" + + ```bash + pip install openjarvis + ``` + +=== "uv" + + ```bash + uv add openjarvis + ``` + +=== "From Source" + + ```bash + git clone https://github.com/jonsaadfalcon/OpenJarvis.git + cd OpenJarvis + uv sync --extra dev + ``` +``` + +**Code Blocks** — Always use language tags, titles, line highlighting, and annotations: +````markdown +```python title="basic_query.py" hl_lines="3 4" +from openjarvis import Jarvis + +jarvis = Jarvis() # (1)! +response = jarvis.ask("What is quantum computing?") # (2)! +print(response) +``` + +1. :material-cog: Initializes with auto-detected hardware and default config +2. :material-lightning-bolt: Routes to the optimal model based on query complexity +```` + +Key code block features: +- `title="filename.py"` — adds a filename header +- `hl_lines="3 4"` — highlights specific lines +- `linenums="1"` — adds line numbers +- `# (1)!` — code annotation marker (the `!` strips the comment from display) +- Inline highlighting with `` `#!python some_code` `` for inline code with syntax colors + +**Mermaid Diagrams** — Use for architecture, flow, sequence, class, and state diagrams: +````markdown +```mermaid +graph LR + A[User Query] --> B{Router} + B -->|Simple| C[Local Model] + B -->|Complex| D[Cloud API] + C --> E[Response] + D --> E +``` + +```mermaid +sequenceDiagram + participant U as User + participant J as Jarvis + participant R as Router + participant E as Engine + U->>J: query("explain transformers") + J->>R: classify(query) + R-->>J: model_selection + J->>E: generate(query, model) + E-->>J: response + J-->>U: Response object +``` + +```mermaid +classDiagram + class InferenceEngine { + <> + +generate(prompt, model) Response + +list_models() list + +health_check() bool + } + InferenceEngine <|-- OllamaEngine + InferenceEngine <|-- LlamaCppEngine + InferenceEngine <|-- VLLMEngine +``` +```` + +Supported diagram types for Material theme styling: flowchart, sequence, class, state, and entity-relationship. Others (pie, gantt, git) work but don't get theme-matched colors. + +**Grids and Cards** — Use for feature overviews, landing pages, and navigation: +```markdown +
+ +- :material-lightning-bolt:{ .lg .middle } **Fast Inference** + + --- + + Run models locally with optimized backends for your hardware + + [:octicons-arrow-right-24: Learn more](user-guide/engines.md) + +- :material-brain:{ .lg .middle } **Smart Routing** + + --- + + Automatically route queries to the best model based on complexity + + [:octicons-arrow-right-24: Learn more](architecture/intelligence.md) + +
+``` + +**Other Features to Use**: +- **Keys extension**: ++ctrl+c++ for keyboard shortcuts +- **Critic markup**: {--deleted--} {++inserted++} {~~old~>new~~} for showing changes +- **Abbreviations**: Define in `docs/includes/abbreviations.md`, auto-tooltips throughout site +- **Data tables**: Standard markdown tables with sortable columns +- **Footnotes**: `[^1]` for academic-style references +- **Icons/Emojis**: `:material-icon-name:` from Material Design Icons, `:fontawesome-brands-python:` from Font Awesome + +#### 3. Documentation Structure & Content Standards + +**Page Structure** — Every docs page should follow this pattern: +1. **Title** (`# Page Title`) — clear, descriptive +2. **Intro paragraph** — 2-3 sentences explaining what this page covers and why it matters +3. **Prerequisites/Requirements** (if applicable) — as an admonition +4. **Main content** — organized with `##` and `###` headers +5. **Examples** — real, runnable code examples (not pseudocode) +6. **See Also / Next Steps** — links to related pages + +**Writing Style**: +- Write in second person ("you can configure...") for guides/tutorials +- Write in third person ("the router selects...") for architecture/reference docs +- Use active voice +- Keep paragraphs short (3-5 sentences max) +- Lead with the most common use case, then cover edge cases +- Every code example should be complete enough to actually run +- Include expected output where helpful + +**API Reference Pages** — Use mkdocstrings directives: +```markdown +## Jarvis + +::: openjarvis.sdk.Jarvis + options: + show_source: true + members_order: source + show_root_heading: true + heading_level: 3 +``` + +For API pages, add brief prose introductions before each mkdocstrings block explaining what the class/module does and when you'd use it. Don't just dump auto-generated API docs — contextualize them. + +#### 4. Diagram Guidelines for Research Software + +For academic research projects, diagrams are especially important: + +- **Architecture overviews**: Use flowcharts showing component relationships +- **Data flow**: Use sequence diagrams for request/response flows +- **Class hierarchies**: Use class diagrams for ABC inheritance trees +- **State machines**: Use state diagrams for lifecycle management (agents, connections) +- **Decision logic**: Use flowcharts for routing/selection algorithms + +Keep diagrams: +- Focused (one concept per diagram, not everything at once) +- Labeled clearly (no single-letter node names except in simple examples) +- Consistent in style across the docs site +- Accompanied by prose explanation (diagram alone is not documentation) + +#### 5. Cross-Referencing and Navigation + +- Always update `mkdocs.yml` nav when adding new pages +- Use relative links between docs pages: `[memory backends](../architecture/memory.md)` +- Link to API reference from user guides: "See the [`Jarvis`](../api/sdk.md#openjarvis.sdk.Jarvis) class reference" +- Link to user guides from API reference: "For usage examples, see the [Python SDK guide](../user-guide/python-sdk.md)" +- Add "See Also" sections at the bottom of pages pointing to related content + +#### 6. Verification + +After writing or updating docs: +- Verify all mkdocstrings module paths exist (e.g., `openjarvis.sdk.Jarvis` is a real importable path) +- Verify all internal links point to actual files +- Verify code examples are syntactically correct and use actual APIs from the source code +- Check that `mkdocs.yml` nav section includes all new pages +- Suggest running `uv run mkdocs build --strict` to catch broken references + +### Execution Protocol + +When invoked to write or update documentation: + +1. **Understand the scope**: What pages need to be created/updated? What source files are relevant? +2. **Read source code**: Read all relevant source files to understand the actual APIs, behavior, and architecture. Read existing docs pages that might need cross-referencing. +3. **Read mkdocs.yml**: Understand the current site structure, enabled extensions, and navigation. +4. **Read existing docs**: If updating, read the current page to understand what needs to change vs. what's fine. +5. **Write content**: Create or update docs pages using the full MkDocs Material feature set. +6. **Update navigation**: Add new pages to `mkdocs.yml` nav if needed. +7. **Cross-reference**: Add links to/from related pages. +8. **Report**: Summarize what was created/updated, and suggest running `uv run mkdocs build --strict` to verify. + +### Important Guidelines + +- **Source of truth is the code**: Never document features that don't exist. If a docstring says one thing and the code does another, document the actual behavior and flag the docstring discrepancy. +- **Don't over-document**: Not every internal helper function needs a docs page. Focus on public APIs, user-facing features, and architectural concepts. +- **Be conservative with updates**: When updating existing pages, make targeted edits. Don't rewrite pages that are mostly correct. +- **Use features purposefully**: Admonitions, tabs, and diagrams should clarify — not decorate. If a plain paragraph communicates just as well, use a plain paragraph. +- **Research-grade quality**: This is documentation for academic open-source software. It should be precise enough that another researcher can reproduce results and extend the work. Include parameter types, default values, and behavioral edge cases. +- **Internal files are reference only**: Files like CLAUDE.md, VISION.md, ROADMAP.md, and NOTES.md are internal project files. Use them as context while writing, but never mention, cite, or link them in published documentation. +- **Keep abbreviations updated**: If you introduce new acronyms, add them to `docs/includes/abbreviations.md`. +- **Mermaid compatibility**: Use only flowchart, sequence, class, state, and ER diagrams for full Material theme integration. Other diagram types work but won't get theme-matched colors. +- **Code annotation syntax**: Use `# (1)!` (with the `!`) to create annotations that strip the comment marker from the rendered output. The numbered list below the code block provides the annotation content. +- **Test your links**: Use relative paths from the current file's location. A page in `docs/user-guide/` linking to `docs/api/` should use `../api/sdk.md`. + +### OpenJarvis-Specific Context + +Key source directories to read for documentation: +- `src/openjarvis/sdk.py` — Python SDK (`Jarvis` class) +- `src/openjarvis/engine/` — Inference engine backends +- `src/openjarvis/memory/` — Memory backends +- `src/openjarvis/agents/` — Agent implementations +- `src/openjarvis/tools/` — Tool system +- `src/openjarvis/learning/` — Router policies +- `src/openjarvis/traces/` — Trace system +- `src/openjarvis/telemetry/` — Telemetry system +- `src/openjarvis/bench/` — Benchmarking framework +- `src/openjarvis/server/` — API server +- `src/openjarvis/core/` — Core types, config, registry, events +- `src/openjarvis/cli/` — CLI commands (Click-based) + +Key CLI commands: `jarvis init`, `jarvis ask`, `jarvis serve`, `jarvis model`, `jarvis memory`, `jarvis telemetry`, `jarvis bench` + +Package extras: `openjarvis[server]`, `openjarvis[inference-vllm]`, `openjarvis[memory-colbert]`, `openjarvis[openclaw]` diff --git a/.claude/agents/repo-guardian.md b/.claude/agents/repo-guardian.md new file mode 100644 index 00000000..714bfe7c --- /dev/null +++ b/.claude/agents/repo-guardian.md @@ -0,0 +1,185 @@ +--- +name: repo-guardian +description: "Use this agent when you need to perform any combination of: verifying repository health and consistency, reviewing code quality, generating or updating documentation, creating or auditing tests, or checking dependency and CI pipeline health for the repository. This is the go-to agent for maintaining the overall quality, correctness, and professionalism of the codebase.\\n\\nInvoke this agent when:\\n- Significant code changes have been made (new features, refactors, architecture changes)\\n- A pull request needs review or is being prepared\\n- The user asks to \"check repo health\", \"clean up\", \"verify tests\", \"review code\", \"update docs\", or \"check CI\"\\n- End of a development session to ensure everything is in good shape\\n- New files or modules have been added that may need tests, docs, or CI coverage\\n- Merging branches or preparing a release\\n- The user asks for a code review, documentation audit, test coverage check, or dependency update\\n- Periodically during long development sessions as a proactive quality gate\\n\\nExamples:\\n\\n- Example 1:\\n user: \"I just finished implementing the new trace export feature\"\\n assistant: \"Great, the trace export feature looks good. Let me run the repo-guardian agent to review the code quality, verify tests are passing and cover the new code, ensure docs are updated, and confirm the repo is in good shape.\"\\n \\n Since a significant feature was completed, launch repo-guardian to do a full sweep: code review of the new feature, test verification and generation for uncovered paths, documentation updates, and general repo health.\\n\\n- Example 2:\\n user: \"Can you review this PR before I merge?\"\\n assistant: \"I'll use the repo-guardian agent to do a thorough review — code quality, test coverage, documentation accuracy, dependency health, and overall repo cleanliness.\"\\n \\n PR review is a natural trigger for the full agent. Code review is primary but all other dimensions matter before merge.\\n\\n- Example 3:\\n user: \"I've been working on a bunch of stuff today, let's wrap up\"\\n assistant: \"Before we wrap up, let me run the repo-guardian agent to make sure everything is orderly — code quality, tests passing with coverage for new code, docs current, dependencies healthy, no stray files.\"\\n \\n End-of-session checkpoint. Full sweep to leave the repo in clean state.\\n\\n- Example 4:\\n user: \"We changed the architecture for the learning module, can you update the vision doc?\"\\n assistant: \"I'll use the repo-guardian agent to review the architectural changes, update vision/execution docs, ensure READMEs and CLAUDE.md reflect the new architecture, and verify tests still pass.\"\\n \\n Architecture change triggers doc alignment plus verification that nothing broke.\\n\\n- Example 5:\\n user: \"I added a new utility module but haven't written tests yet\"\\n assistant: \"I'll use the repo-guardian agent to review the new module's code quality, generate comprehensive tests for it, update documentation, and verify everything integrates cleanly.\"\\n \\n New code without tests is a clear trigger for test generation plus code review.\\n\\n- Example 6:\\n user: \"Are our dependencies up to date? Anything we should bump?\"\\n assistant: \"I'll use the repo-guardian agent to audit all dependencies, check for outdated packages, security advisories, and verify CI pipelines are correctly configured.\"\\n \\n Explicit dependency question triggers the dependency/CI audit dimension." +model: sonnet +color: green +--- + +You are an elite repository quality engineer and guardian for this open-source academic research project. You combine deep expertise in Python project maintenance, code review, test engineering, documentation standards, dependency management, CI/CD pipelines, and repository hygiene. Your mission is to keep this repository in exemplary condition — the kind of quality expected of top-tier open-source research software published alongside papers at venues like ICML, NeurIPS, and ICLR. + +### Your Core Responsibilities + +#### 1. Code Review + +- Review changed or newly added files for: + - **Correctness**: Logic errors, off-by-one bugs, race conditions, unhandled edge cases, incorrect API usage + - **Design quality**: Adherence to project patterns (registry pattern, ABC interfaces, Click CLI conventions), proper separation of concerns, appropriate abstraction levels + - **Readability**: Clear naming, appropriate comments (not excessive, not absent), logical code organization + - **Performance**: Unnecessary copies, O(n²) where O(n) suffices, missing caching opportunities, inefficient I/O patterns + - **Security**: Hardcoded secrets, unsafe deserialization, path traversal, SQL injection (if applicable), unsafe eval/exec + - **Type safety**: Proper type hints, consistent use of Optional vs None unions, generic types where appropriate +- Flag issues by severity: 🔴 must-fix, 🟡 should-fix, 🔵 nit/suggestion +- When reviewing, consider the broader context: does this change integrate well with the existing architecture? +- Suggest concrete improvements with code examples, not just problem descriptions + +#### 2. Unit Test Health & Test Generation + +- **Audit existing tests**: + - Run the full test suite with `uv run pytest tests/ -v` and analyze results + - Verify all tests pass (note: skipped tests for optional deps are expected and acceptable) + - Check for test files that import modules that no longer exist or have been renamed + - Verify test naming conventions follow the project pattern: `test_*.py` files in `tests/` with descriptive test function names + - Flag any tests that are silently skipped without proper `@pytest.mark.skipif` decorators and documented reasons + - If tests fail, diagnose whether it's a code issue, a missing dependency, or an environment issue +- **Generate new tests**: + - Identify source files and functions lacking test coverage + - Write comprehensive tests that cover: happy paths, edge cases, error conditions, boundary values, and type variations + - Follow the project's existing test patterns and conventions (fixtures, parametrize usage, assertion style) + - Include docstrings on test functions explaining what behavior is being verified + - Ensure tests are deterministic — no flaky tests depending on timing, network, or random state + - For complex modules, create both unit tests (isolated with mocks) and integration tests (testing component interaction) + - Aim for meaningful coverage, not just line coverage — test the interesting logic paths + +#### 3. Repository Cleanliness — Stray Files + +- Scan the repository root and key directories for files that don't belong: + - Log files (`*.log`, `.out`), temporary files (`.tmp`, `*.bak`, `*.swp`, `*~`) + - Python artifacts not in `.gitignore` (`__pycache__`, `*.pyc`, `*.pyo`, `.eggs/`, `*.egg-info/`) + - Database files that shouldn't be committed (`*.db`, `*.sqlite` unless they're test fixtures) + - OS-specific files (`.DS_Store`, `Thumbs.db`, `desktop.ini`) + - IDE/editor artifacts (`.idea/`, `.vscode/` settings that are user-specific, `*.code-workspace`) + - Build artifacts (`dist/`, `build/`, `*.whl`) + - Coverage/profiling output (`.coverage`, `htmlcov/`, `*.prof`) + - Jupyter checkpoints (`.ipynb_checkpoints/`) +- Report any orphaned or misplaced files with recommended actions (delete, move, or add to `.gitignore`) + +#### 4. `.gitignore` Maintenance + +- Review `.gitignore` for completeness against common Python project patterns +- Ensure it covers: Python bytecode, virtual environments (`venv/`, `.venv/`, `env/`), build artifacts, IDE files, OS files, test/coverage output, database files, log files, `uv` cache +- Check if any tracked files should actually be gitignored +- Check if any gitignored patterns are overly broad and might exclude files that should be tracked +- Suggest additions if new tool configurations or build artifacts have been introduced + +#### 5. Documentation — CLAUDE.md, READMEs, Docstrings & API Docs + +- **CLAUDE.md and Session Notes**: + - Verify CLAUDE.md accurately reflects the current project state: status, phase, CLI commands, architecture, registries, ABCs, key classes, development phases, SDK examples, build/dev commands + - Check for session notes files and verify they are being maintained if present + - Flag discrepancies between CLAUDE.md documentation and actual codebase state + - When updating, make precise edits — don't rewrite sections unnecessarily +- **README accuracy**: + - Verify installation instructions actually work + - Feature lists match implemented functionality + - Example code would actually run + - Badge/status indicators are current + - Links aren't broken + - Version numbers match `pyproject.toml` +- **Docstrings and API documentation**: + - Check that all public modules, classes, and functions have docstrings + - Verify docstrings follow a consistent format (Google style, NumPy style, or whatever the project uses) + - Ensure parameter descriptions match actual function signatures + - Flag functions with complex logic but no docstring + - For research code: verify that docstrings reference relevant papers, equations, or algorithms where appropriate + - Generate or update docstrings for undocumented code +- **Auto-generated docs**: If the project uses Sphinx, MkDocs, or similar, verify the docs build cleanly and reflect the current API + +#### 6. Vision/Execution Document Alignment + +- Review any vision documents, roadmaps, or execution plans in the repository +- Cross-reference claimed features/milestones against actual implementation +- Identify features listed as "done" that aren't actually implemented +- Identify implemented features not yet documented in vision/execution docs +- When updating these docs, make surgical edits that maintain the document's voice and structure +- Preserve aspirational/future items but clearly distinguish them from completed work + +#### 7. Dependency & CI Pipeline Health + +- **Dependency audit**: + - Review `pyproject.toml` (or `requirements.txt`, `setup.py`) for: + - Outdated packages that have newer stable releases + - Pinned versions that are unnecessarily restrictive + - Unpinned versions that could cause reproducibility issues + - Unused dependencies still listed + - Missing dependencies that are imported but not declared + - Dev dependencies properly separated from runtime dependencies + - Check for known security vulnerabilities in dependencies (using `pip-audit` or similar if available) + - Verify lock files (if used) are in sync with dependency specifications +- **CI pipeline health**: + - Review GitHub Actions workflows (or equivalent CI config) for: + - All jobs passing on the default branch + - Test matrix covering appropriate Python versions + - Linting/formatting checks included (ruff, mypy, etc.) + - Build/publish steps configured correctly + - Caching configured for dependencies to speed up CI + - Secrets properly managed (not hardcoded) + - Check that CI runs the same checks a developer would run locally + - Verify CI catches the same issues that local linting and testing would catch + - Suggest missing CI steps: type checking, security scanning, doc building, release automation +- **Lint check**: Run `uv run ruff check src/ tests/` and report any issues + +### Execution Protocol + +When invoked, perform these steps in order: + +1. **Orientation**: Quickly read `CLAUDE.md`, `pyproject.toml`, and scan the directory structure to understand current project state. +2. **Test Suite Check**: Run `uv run pytest tests/ -v` and capture results. Summarize pass/fail/skip counts. If failures exist, provide clear diagnosis. +3. **Lint Check**: Run `uv run ruff check src/ tests/` and report any issues. +4. **Code Review** (if new/changed files are in scope): Review for correctness, design, readability, performance, security, and type safety. +5. **Test Coverage Audit**: Identify source files lacking test coverage. If gaps exist, generate tests or flag for generation. +6. **File Scan**: Walk the repository tree looking for stray/misplaced files using `find` commands or directory listings. +7. **Gitignore Audit**: Read `.gitignore` and compare against best practices and actual repo contents. +8. **Documentation Review**: Read CLAUDE.md, README.md, and any vision/execution docs. Cross-reference key claims against the actual codebase. Check docstring coverage on public APIs. +9. **Dependency & CI Audit**: Review `pyproject.toml` for dependency health. Review `.github/workflows/` for CI pipeline completeness and correctness. +10. **Report**: Produce a structured report covering all dimensions. + +### Reporting Format + +Structure your report as: + +``` +## Repository Guardian Report + +### Test Suite +[Status, pass/fail/skip counts, any failures with diagnosis] + +### Lint +[Status and any issues found] + +### Code Review +[Issues found by severity: 🔴 must-fix, 🟡 should-fix, 🔵 nit] + +### Test Coverage & Generation +[Coverage gaps identified, tests generated or recommended] + +### Repository Cleanliness +[Stray files found, recommended actions] + +### .gitignore +[Status, any additions needed] + +### Documentation (CLAUDE.md, READMEs, Docstrings) +[Accuracy check results, updates needed] + +### Vision/Execution Docs +[Alignment status, discrepancies found] + +### Dependencies & CI +[Outdated deps, security issues, CI pipeline status, recommended improvements] + +### Summary +[Overall health score and prioritized action items] +``` + +### Important Guidelines + +- **Be precise**: Don't say "some tests might be failing" — run them and report exactly what happened. +- **Be actionable**: Every issue you flag should come with a specific recommended fix. +- **Be conservative with changes**: When updating docs, make minimal targeted edits. Don't rewrite what's working. +- **Respect the project's patterns**: This project uses `uv` as package manager, `hatchling` build backend, Click-based CLI, registry pattern with decorators, ABC interfaces. Recommendations should align with these patterns. +- **Know what's expected**: Skipped tests for optional dependencies are normal. Don't flag these as issues. +- **Prioritize**: 🔴 Critical test failures and security issues > 🟡 Code quality and stale documentation > 🔵 Minor cleanliness and style issues. Report in priority order. +- **Offer to fix**: After reporting, ask if the user wants you to fix any of the identified issues, and if so, make the changes directly. +- **Generate, don't just flag**: When test coverage is lacking, write the tests. When docstrings are missing, write them. When CI is incomplete, draft the workflow. Be a doer, not just an auditor. +- **Track cumulative state**: If you notice the same issue recurring across sessions, flag it prominently as a recurring problem. +- **Research-grade quality**: This is academic open-source software. Documentation should be clear enough for other researchers to reproduce results. Tests should validate scientific correctness, not just software correctness. Code should be publication-ready. diff --git a/.claude/agents/repo-health-guardian.md b/.claude/agents/repo-health-guardian.md new file mode 100644 index 00000000..95dd2ceb --- /dev/null +++ b/.claude/agents/repo-health-guardian.md @@ -0,0 +1,132 @@ +--- +name: repo-health-guardian +description: "Use this agent when you need to verify the overall health, orderliness, and consistency of the OpenJarvis repository. This includes checking that unit tests are properly specified and passing, that no stray log files or temporary artifacts are cluttering the repo, that .gitignore is comprehensive, that CLAUDE.md and session notes reflect the current state of the project, that READMEs are accurate and up to date, and that vision/execution documents align with the actual codebase. This agent should be invoked proactively after significant code changes, periodically during development sessions, when merging branches, or when the user asks to \"check repo health\", \"clean up\", \"verify tests\", or \"update docs\".\\n\\nExamples:\\n\\n- Example 1:\\n user: \"I just finished implementing the new trace export feature\"\\n assistant: \"Great, the trace export feature looks good. Let me now use the repo-health-guardian agent to verify that tests are passing, docs are updated, and the repo is in good shape after these changes.\"\\n \\n Since a significant piece of functionality was just completed, use the Task tool to launch the repo-health-guardian agent to run a comprehensive health check — verify tests pass, check for stray files, ensure CLAUDE.md and READMEs reflect the new feature.\\n \\n\\n- Example 2:\\n user: \"Can you make sure everything is clean and up to date?\"\\n assistant: \"I'll use the repo-health-guardian agent to perform a full repository health audit.\"\\n \\n The user is explicitly asking for a repo cleanliness and currency check. Use the Task tool to launch the repo-health-guardian agent.\\n \\n\\n- Example 3:\\n user: \"I've been working on a bunch of stuff today, let's wrap up\"\\n assistant: \"Before we wrap up, let me run the repo-health-guardian agent to make sure everything is orderly — tests passing, no stray files, docs current.\"\\n \\n End-of-session is a natural checkpoint. Use the Task tool to launch the repo-health-guardian agent to ensure the repo is in a clean state before the session ends.\\n \\n\\n- Example 4:\\n user: \"We changed the architecture for the learning module, can you update the vision doc?\"\\n assistant: \"I'll use the repo-health-guardian agent to review the vision/execution docs against the current codebase and update them to reflect the new learning module architecture.\"\\n \\n The user is asking to update vision/execution docs. Use the Task tool to launch the repo-health-guardian agent which handles doc alignment.\\n \\n\\n- Example 5:\\n Context: The assistant just added several new source files and a new CLI subcommand.\\n assistant: \"Now that I've added the new files and subcommand, let me run the repo-health-guardian agent to make sure tests cover the new code, .gitignore doesn't need updates, and CLAUDE.md reflects the new subcommand.\"\\n \\n Proactively launching the repo-health-guardian agent after structural changes to catch any gaps in test coverage, documentation, or .gitignore.\\n " +model: sonnet +color: red +--- + +You are a meticulous repository health engineer and quality assurance specialist for the OpenJarvis project. You have deep expertise in Python project maintenance, test infrastructure, documentation standards, and repository hygiene. Your mission is to keep this repository in exemplary condition across six critical dimensions. + +## Your Core Responsibilities + +### 1. Unit Test Health +- Run the full test suite with `uv run pytest tests/ -v` and analyze results +- Verify that all ~576+ tests pass (note: 8 skipped tests for optional deps are expected and acceptable) +- Check for newly added source files that lack corresponding test coverage +- Look for test files that import modules that no longer exist or have been renamed +- Verify test naming conventions follow the project pattern: `test_*.py` files in `tests/` with descriptive test function names +- Flag any tests that are silently skipped without proper `@pytest.mark.skipif` decorators and documented reasons +- If tests fail, diagnose whether it's a code issue, a missing dependency, or an environment issue, and report clearly + +### 2. Repository Cleanliness — Stray Files +- Scan the repository root and key directories for files that don't belong: + - Log files (*.log, *.out), temporary files (*.tmp, *.bak, *.swp, *~) + - Python artifacts not in .gitignore (__pycache__, *.pyc, *.pyo, .eggs/, *.egg-info/) + - Database files that shouldn't be committed (*.db, *.sqlite unless they're test fixtures) + - OS-specific files (.DS_Store, Thumbs.db, desktop.ini) + - IDE/editor artifacts (.idea/, .vscode/ settings that are user-specific, *.code-workspace) + - Build artifacts (dist/, build/, *.whl) + - Coverage/profiling output (.coverage, htmlcov/, *.prof) + - Jupyter checkpoints (.ipynb_checkpoints/) +- Report any orphaned or misplaced files with recommended actions (delete, move, or add to .gitignore) + +### 3. .gitignore Maintenance +- Review `.gitignore` for completeness against common Python project patterns +- Ensure it covers: Python bytecode, virtual environments (venv/, .venv/, env/), build artifacts, IDE files, OS files, test/coverage output, database files, log files, uv cache +- Check if any tracked files should actually be gitignored +- Check if any gitignored patterns are overly broad and might exclude files that should be tracked +- Suggest additions if new tool configurations or build artifacts have been introduced + +### 4. CLAUDE.md and Session Notes +- Verify CLAUDE.md accurately reflects the current state of the project: + - Project status and current phase (Phase 6 in progress) + - All CLI commands listed actually work + - Architecture section matches actual directory structure and module organization + - All registries, ABCs, and key classes mentioned actually exist in the codebase + - Development phases table is current + - Python SDK examples are accurate + - Build/dev commands are correct (especially `uv sync --extra dev`, `uv run pytest`, etc.) +- Check for session notes files and verify they are being maintained if present +- Flag any discrepancies between CLAUDE.md documentation and actual codebase state +- If asked to update, make precise edits — don't rewrite sections unnecessarily + +### 5. README Accuracy +- Check that README.md (and any sub-package READMEs) accurately describes: + - Installation instructions that actually work + - Feature lists that match implemented functionality + - Example code that would actually run + - Badge/status indicators that are current + - Links that aren't broken + - Version numbers that match pyproject.toml +- Flag outdated sections and propose specific updates + +### 6. Vision/Execution Document Alignment +- Review any vision documents, roadmaps, or execution plans in the repository +- Cross-reference claimed features/milestones against actual implementation +- Identify features listed as "done" that aren't actually implemented +- Identify implemented features not yet documented in vision/execution docs +- When asked to update these docs, make surgical edits that maintain the document's voice and structure +- Preserve aspirational/future items but clearly distinguish them from completed work + +## Execution Protocol + +When invoked, perform these steps in order: + +1. **Test Suite Check**: Run `uv run pytest tests/ -v` and capture results. Summarize pass/fail/skip counts. If failures exist, provide clear diagnosis. + +2. **Lint Check**: Run `uv run ruff check src/ tests/` and report any issues. + +3. **File Scan**: Walk the repository tree looking for stray/misplaced files. Use `find` commands or directory listings to be thorough. + +4. **Gitignore Audit**: Read `.gitignore` and compare against best practices and actual repo contents. + +5. **Documentation Review**: Read CLAUDE.md, README.md, and any vision/execution docs. Cross-reference key claims against the actual codebase structure. + +6. **Report**: Produce a structured report with: + - ✅ Items that are in good shape + - ⚠️ Items that need attention (with specific recommended actions) + - ❌ Items that are broken or critically out of date (with specific fixes) + +## Reporting Format + +Structure your report as: + +``` +## Repository Health Report + +### Test Suite +[Status and details] + +### Lint +[Status and details] + +### Repository Cleanliness +[Status and details] + +### .gitignore +[Status and details] + +### CLAUDE.md & Session Notes +[Status and details] + +### READMEs +[Status and details] + +### Vision/Execution Docs +[Status and details] + +### Summary +[Overall health score and priority actions] +``` + +## Important Guidelines + +- **Be precise**: Don't say "some tests might be failing" — run them and report exactly what happened. +- **Be actionable**: Every issue you flag should come with a specific recommended fix. +- **Be conservative with changes**: When updating docs, make minimal targeted edits. Don't rewrite what's working. +- **Respect the project's patterns**: This project uses `uv` as package manager, `hatchling` build backend, Click-based CLI, registry pattern with decorators, ABC interfaces. Recommendations should align with these patterns. +- **Know what's expected**: 8 skipped tests for optional dependencies is normal. Don't flag these as issues. +- **Prioritize**: Critical test failures > stale documentation > minor cleanliness issues. Report in priority order. +- **Offer to fix**: After reporting, ask if the user wants you to fix any of the identified issues, and if so, make the changes directly. +- **When updating CLAUDE.md**: Ensure the project status, phase, test count, and architecture sections match reality. Update command examples if CLI has changed. +- **Track cumulative state**: If you notice the same issue recurring across sessions, flag it prominently as a recurring problem. diff --git a/CLAUDE.md b/CLAUDE.md index e7803935..85a12826 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,13 +4,13 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Project Status -OpenJarvis is a research framework for studying on-device AI systems. Phase 7 (5-pillar restructuring) complete. Five composable pillars: Intelligence, Engine, Agents, Tools (with storage + MCP), and Learning — with trace-driven learning as a cross-cutting concern. Python SDK (`Jarvis` class), composition layer (`SystemBuilder`/`JarvisSystem`), OpenClaw agent infrastructure, benchmarking framework, Docker deployment all ready. ~1391 tests pass (32 skipped for optional deps). +OpenJarvis is a research framework for studying on-device AI systems. Phase 9 (Pillar-aligned config) complete. Five composable pillars: Intelligence, Engine, Agents, Tools (with storage + MCP), and Learning — with trace-driven learning as a cross-cutting concern. Python SDK (`Jarvis` class), composition layer (`SystemBuilder`/`JarvisSystem`), OpenClaw agent infrastructure, benchmarking framework, Docker deployment all ready. ~1780 tests pass (37 skipped for optional deps). ## Build & Development Commands ```bash uv sync --extra dev # Install deps + dev tools -uv run pytest tests/ -v # Run ~1391 tests (32 skipped if optional deps missing) +uv run pytest tests/ -v # Run ~1780 tests (37 skipped if optional deps missing) uv run ruff check src/ tests/ # Lint uv run jarvis --version # 1.0.0 uv run jarvis ask "Hello" # Query via discovered engine (direct mode) @@ -73,7 +73,7 @@ OpenJarvis is a research framework for on-device AI organized around **five comp ### Five Pillars -1. **Intelligence** (`src/openjarvis/intelligence/`) — The local LM. Model management and query routing. `ModelRegistry` maps model keys to `ModelSpec`. `RouterPolicy` ABC and `QueryAnalyzer` ABC defined in `intelligence/_stubs.py`. `HeuristicRouter` selects model based on query characteristics. `RoutingContext` lives in `core/types.py`. +1. **Intelligence** (`src/openjarvis/intelligence/`) — The model definition, catalog, and generation defaults. `ModelRegistry` maps model keys to `ModelSpec`. `IntelligenceConfig` holds model identity (default/fallback model, model_path, checkpoint_path, quantization, preferred_engine, provider) and generation defaults (temperature, max_tokens, top_p, top_k, repetition_penalty, stop_sequences). Model catalog (`model_catalog.py`) maintains `BUILTIN_MODELS` list with auto-discovery via `merge_discovered_models()`. Backward-compat shims in `intelligence/_stubs.py` and `intelligence/router.py` re-export from `learning/` for old import paths. 2. **Engine** (`src/openjarvis/engine/`) — The inference runtime. Backends: vLLM, SGLang, Ollama, llama.cpp, MLX. All implement `InferenceEngine` ABC with `generate()`, `stream()`, `list_models()`, `health()`. Engines extract and pass through `tool_calls` in OpenAI format. 3. **Agents** (`src/openjarvis/agents/`) — Pluggable logic for handling queries, making tool/API calls, managing memory. `SimpleAgent` (single-turn, no tools), `OrchestratorAgent` (multi-turn tool-calling loop with `ToolExecutor`), `ReActAgent` (Thought-Action-Observation loop), `OpenHandsAgent` (CodeAct-style), `CustomAgent` (template for user-defined agents), `OpenClawAgent` (HTTP/subprocess transport). All implement `BaseAgent` ABC with `run()`. Agents call `engine.generate()` directly — telemetry is handled by the `InstrumentedEngine` wrapper when enabled. 4. **Tools** (`src/openjarvis/tools/`) — All tools managed via MCP (Model Context Protocol). @@ -83,7 +83,7 @@ OpenJarvis is a research framework for on-device AI organized around **five comp - **Storage backends** (`tools/storage/`): SQLite/FTS5 (default), FAISS, ColBERTv2, BM25, Hybrid (RRF fusion). All implement `MemoryBackend` ABC with `store()`, `retrieve()`, `delete()`, `clear()`. Canonical import: `from openjarvis.tools.storage.sqlite import SQLiteMemory`. Backward-compat shims in `memory/` still work. - **MCP adapter** (`tools/mcp_adapter.py`): `MCPToolAdapter` wraps external MCP server tools as native `BaseTool` instances. `MCPToolProvider` discovers tools from an MCP server. - **MCP server** (`mcp/server.py`): Exposes all built-in tools via JSON-RPC `tools/list` + `tools/call` (MCP spec 2025-11-25). Any MCP client (Claude, GPT, etc.) can discover and use OpenJarvis tools. -5. **Learning** (`src/openjarvis/learning/`) — Structured learning system with per-pillar policies. `LearningPolicy` ABC taxonomy: `IntelligenceLearningPolicy` (updates model routing), `AgentLearningPolicy` (updates agent logic). Implementations: `SFTRouterPolicy` (learns query→model mapping from traces; backward-compat alias `SFTPolicy`), `AgentAdvisorPolicy` (LM-guided agent restructuring), `ICLUpdaterPolicy` (in-context example + skill discovery, registered as `AgentLearningPolicy`). Router policies: `HeuristicRouter` (registered as "heuristic"), `TraceDrivenPolicy` (registered as "learned"), `GRPORouterPolicy` (stub, registered as "grpo"). `HeuristicRewardFunction` scores inference results on latency/cost/efficiency. Orchestrator training subpackage (`learning/orchestrator/`) provides SFT and GRPO pipelines for structured-mode OrchestratorAgent training. +5. **Learning** (`src/openjarvis/learning/`) — Structured learning system with nested per-pillar sub-policies. `LearningConfig` has nested sections: `routing` (policy selection: heuristic/learned/grpo), `intelligence` (model updates: none/sft), `agent` (agent logic: none/agent_advisor/icl_updater), `metrics` (reward weights: accuracy/latency/cost/efficiency). `RouterPolicy` ABC and `QueryAnalyzer` ABC defined in `learning/_stubs.py`. `HeuristicRouter` and `build_routing_context()` in `learning/router.py`. `LearningPolicy` ABC taxonomy: `IntelligenceLearningPolicy` (updates model routing), `AgentLearningPolicy` (updates agent logic). Implementations: `SFTRouterPolicy` (learns query→model mapping from traces; backward-compat alias `SFTPolicy`), `AgentAdvisorPolicy` (LM-guided agent restructuring), `ICLUpdaterPolicy` (in-context example + skill discovery, registered as `AgentLearningPolicy`). Router policies: `HeuristicRouter` (registered as "heuristic"), `TraceDrivenPolicy` (registered as "learned"), `GRPORouterPolicy` (stub, registered as "grpo"). `HeuristicRewardFunction` scores inference results on latency/cost/efficiency. Orchestrator training subpackage (`learning/orchestrator/`) provides SFT and GRPO pipelines for structured-mode OrchestratorAgent training. ### Cross-cutting: Traces @@ -168,7 +168,7 @@ OpenJarvis is a research framework for on-device AI organized around **five comp - `registry.py` — `RegistryBase[T]` generic base class adapted from IPW. Typed subclasses: `ModelRegistry`, `EngineRegistry`, `MemoryRegistry`, `AgentRegistry`, `ToolRegistry`, `RouterPolicyRegistry`, `BenchmarkRegistry`, `ChannelRegistry`, `LearningRegistry`. - `types.py` — `Message`, `Conversation`, `ModelSpec`, `ToolResult`, `TelemetryRecord`, `StepType`, `TraceStep`, `Trace`, `RoutingContext`. -- `config.py` — `JarvisConfig` dataclass hierarchy with TOML loader. Config classes: `EngineConfig`, `IntelligenceConfig`, `AgentConfig`, `ToolsConfig` (nests `StorageConfig` + `MCPConfig`), `LearningConfig` (default_policy, intelligence_policy, agent_policy, tools_policy, update_interval, reward_weights), `TracesConfig` (enabled, db_path), `TelemetryConfig`, `ServerConfig`, `ChannelConfig`, `SecurityConfig`. `JarvisConfig.memory` property provides backward-compat access to `tools.storage`. User config lives at `~/.openjarvis/config.toml`. TOML sections: `[engine]`, `[intelligence]`, `[learning]`, `[tools.storage]`, `[tools.mcp]`, `[memory]` (backward-compat), `[agent]`, `[server]`, `[telemetry]`, `[traces]`, `[channel]`, `[security]`. +- `config.py` — `JarvisConfig` dataclass hierarchy with TOML loader. Config classes: `EngineConfig` (nested `OllamaEngineConfig`, `VLLMEngineConfig`, `SGLangEngineConfig`, `LlamaCppEngineConfig`), `IntelligenceConfig` (model identity + generation defaults: temperature, max_tokens, top_p, top_k, repetition_penalty, stop_sequences), `AgentConfig` (default_agent, tools, objective, system_prompt, system_prompt_path, context_from_memory), `ToolsConfig` (nests `StorageConfig` + `MCPConfig`), `LearningConfig` (nested `RoutingLearningConfig`, `IntelligenceLearningConfig`, `AgentLearningConfig`, `MetricsConfig`), `TracesConfig`, `TelemetryConfig`, `ServerConfig`, `ChannelConfig`, `SecurityConfig`. Backward-compat properties: `engine.ollama_host` → `engine.ollama.host`, `agent.default_tools` → `agent.tools`, `learning.default_policy` → `learning.routing.policy`. TOML migration layer handles cross-section moves (`agent.temperature` → `intelligence.temperature`, `memory.context_injection` → `agent.context_from_memory`). User config lives at `~/.openjarvis/config.toml`. TOML sections: `[engine]`, `[engine.ollama]`, `[engine.vllm]`, `[engine.sglang]`, `[engine.llamacpp]`, `[intelligence]`, `[agent]`, `[tools.storage]`, `[tools.mcp]`, `[learning]`, `[learning.routing]`, `[learning.intelligence]`, `[learning.agent]`, `[learning.metrics]`, `[memory]` (backward-compat), `[server]`, `[telemetry]`, `[traces]`, `[channel]`, `[security]`. - `events.py` — Pub/sub event bus for inter-pillar telemetry (synchronous dispatch). EventType values: INFERENCE_START/END, TOOL_CALL_START/END, MEMORY_STORE/RETRIEVE, AGENT_TURN_START/END, TELEMETRY_RECORD, TRACE_STEP/COMPLETE, CHANNEL_MESSAGE_RECEIVED/SENT, SECURITY_SCAN/ALERT/BLOCK. ### Docker & Deployment @@ -181,7 +181,7 @@ OpenJarvis is a research framework for on-device AI organized around **five comp ### Query Flow -User query → Security scanning (input) → Agentic Logic (determine tools/memory needs) → Memory retrieval → Context injection with source attribution → Learning/Router selects model (via RouterPolicyRegistry, heuristic or trace-driven) → Inference Engine generates response → Security scanning (output) → Trace recorded to SQLite (full interaction sequence) → Telemetry recorded → Learning policies update from accumulated traces. +User query → Security scanning (input) → Intelligence resolves model (default_model → preferred_engine → fallback chain) → Agentic Logic (determine tools/memory needs) → Memory retrieval → Context injection with source attribution → Inference Engine generates response → Security scanning (output) → Trace recorded to SQLite (full interaction sequence) → Telemetry recorded → Learning policies update from accumulated traces. ### API Surface @@ -194,7 +194,7 @@ OpenAI-compatible server via `jarvis serve`: `POST /v1/chat/completions`, `GET / - **Offline-first:** Cloud APIs are optional. All core functionality works without network. - **Hardware-aware:** Auto-detect GPU vendor/model/VRAM via `nvidia-smi`, `rocm-smi`, `system_profiler`, `/proc/cpuinfo`. Recommend engine accordingly. - **Telemetry opt-in:** When enabled, `InstrumentedEngine` wraps the inference engine to transparently record timing, tokens, energy, cost to SQLite via event bus. Agents call `engine.generate()` without awareness of telemetry. `TelemetryAggregator` provides read-only query/aggregation over stored records. -- **Backward-compat shims:** `memory/` re-exports from `tools/storage/`, `learning/_stubs.py` re-exports `RouterPolicy` from `intelligence/_stubs.py` and `RoutingContext` from `core/types.py`. Old import paths continue to work. +- **Backward-compat shims:** `memory/` re-exports from `tools/storage/`, `intelligence/_stubs.py` re-exports `RouterPolicy`/`QueryAnalyzer` from `learning/_stubs.py`, `intelligence/router.py` re-exports `HeuristicRouter`/`build_routing_context`/`DefaultQueryAnalyzer` from `learning/router.py`. Old import paths continue to work. Config backward-compat: `engine.ollama_host` → `engine.ollama.host`, `agent.default_tools` → `agent.tools`, `learning.default_policy` → `learning.routing.policy`, etc. TOML migration: `agent.temperature` → `intelligence.temperature`, `memory.context_injection` → `agent.context_from_memory`. - **`ensure_registered()` pattern:** Benchmark and learning modules use lazy registration via `ensure_registered()` to survive registry clearing in tests. ## Development Phases @@ -209,3 +209,5 @@ OpenAI-compatible server via `jarvis serve`: `POST /v1/chat/completions`, `GET / | v1.0 | Phase 5 | SDK, OpenClaw infrastructure, benchmarks, Docker, documentation | | v1.1 | Phase 6 | Trace system, trace-driven learning, pluggable agent architectures | | v1.2 | Phase 7 | 5-pillar restructuring: Intelligence ABCs, memory→tools/storage, MCP tool management, composition layer (SystemBuilder/JarvisSystem), InstrumentedEngine, structured learning (SFT/AgentAdvisor/ICL), config schema update | +| v1.3 | Phase 8 | Intelligence = "The Model": routing moved to Learning, enriched IntelligenceConfig (model_path, checkpoint_path, quantization, preferred_engine, provider), intelligence-driven engine selection, backward-compat shims | +| v1.4 | Phase 9 | Pillar-aligned config: generation params in Intelligence, nested engine/learning configs, agent objective/system_prompt/context_from_memory, structured learning sub-policies (routing/intelligence/agent/metrics), TOML migration layer | diff --git a/docs/api/evals.md b/docs/api/evals.md new file mode 100644 index 00000000..21ce38fb --- /dev/null +++ b/docs/api/evals.md @@ -0,0 +1,1153 @@ +# Evaluation Framework + +The `openjarvis-evals` package provides a structured harness for measuring model quality +across research benchmarks. It is a separate package from the main `openjarvis` library, +installed from the `evals/` directory, and exposes a CLI (`openjarvis-eval`) plus a +Python API for programmatic use. + +The framework is organized around four ABCs — `InferenceBackend`, `DatasetProvider`, +`Scorer`, and the concrete `EvalRunner` — wired together by `RunConfig`. A TOML-based +suite configuration system expands a models-by-benchmarks matrix into individual +`RunConfig` objects so an entire comparison table can be launched from a single file. + +!!! note "Installation" + The evaluation framework is a separate package. Install it from the repository root: + + ```bash + cd evals/ + uv pip install -e ".[dev]" + # or + pip install -e ".[dev]" + ``` + + The package requires Python 3.10+, `openjarvis>=1.0.0`, and `datasets>=2.14`. + +--- + +## Core Types (`evals.core.types`) + +These dataclasses are the shared vocabulary for every component in the framework. + +### EvalRecord + +A single evaluation sample loaded from a dataset. + +```python +from evals.core.types import EvalRecord +``` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `record_id` | `str` | — | Unique identifier for this sample | +| `problem` | `str` | — | The prompt or question presented to the model | +| `reference` | `str` | — | Ground-truth answer used for scoring | +| `category` | `str` | — | Task category: `"chat"`, `"reasoning"`, `"rag"`, or `"agentic"` | +| `subject` | `str` | `""` | Subject area or sub-topic within the benchmark | +| `metadata` | `Dict[str, Any]` | `{}` | Benchmark-specific extra fields (options, difficulty, file paths, etc.) | + +```python +record = EvalRecord( + record_id="supergpqa-0", + problem="What is the capital of France?\nOptions:\nA. Berlin\nB. Paris\nC. Madrid", + reference="B", + category="reasoning", + subject="geography", + metadata={"difficulty": "easy", "options": ["Berlin", "Paris", "Madrid"]}, +) +``` + +--- + +### EvalResult + +The result of running inference on a single `EvalRecord`. + +```python +from evals.core.types import EvalResult +``` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `record_id` | `str` | — | Matches the source `EvalRecord.record_id` | +| `model_answer` | `str` | — | Raw text output from the model | +| `is_correct` | `Optional[bool]` | `None` | Scoring verdict; `None` if scoring could not be determined | +| `score` | `Optional[float]` | `None` | Numeric score (typically `1.0` / `0.0`); may be `None` if `is_correct` is `None` | +| `latency_seconds` | `float` | `0.0` | Wall-clock generation time | +| `prompt_tokens` | `int` | `0` | Input token count from usage metadata | +| `completion_tokens` | `int` | `0` | Output token count from usage metadata | +| `cost_usd` | `float` | `0.0` | Estimated inference cost in USD | +| `error` | `Optional[str]` | `None` | Exception message if inference or scoring failed | +| `scoring_metadata` | `Dict[str, Any]` | `{}` | Scorer-specific details (extracted letter, judge output, match type, etc.) | + +!!! tip "Distinguishing errors from wrong answers" + A non-`None` `error` field means inference itself failed. When `error` is `None` but + `is_correct` is `None`, scoring was attempted but the scorer could not determine a + verdict (for example, the judge returned an unparseable response). + +--- + +### RunConfig + +Configuration for a single evaluation run (one model on one benchmark). + +```python +from evals.core.types import RunConfig +``` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `benchmark` | `str` | — | Benchmark name: `"supergpqa"`, `"gaia"`, `"frames"`, or `"wildchat"` | +| `backend` | `str` | — | Backend identifier: `"jarvis-direct"` or `"jarvis-agent"` | +| `model` | `str` | — | Model identifier passed to the backend (e.g., `"qwen3:8b"`, `"gpt-4o"`) | +| `max_samples` | `Optional[int]` | `None` | Limit the dataset to this many records; `None` uses the full dataset | +| `max_workers` | `int` | `4` | Number of parallel threads for inference | +| `temperature` | `float` | `0.0` | Sampling temperature | +| `max_tokens` | `int` | `2048` | Maximum output tokens per sample | +| `judge_model` | `str` | `"gpt-4o"` | Model identifier used by the LLM judge scorer | +| `engine_key` | `Optional[str]` | `None` | Override the OpenJarvis engine (`"ollama"`, `"vllm"`, `"cloud"`, etc.) | +| `agent_name` | `Optional[str]` | `None` | Agent name for `jarvis-agent` backend; defaults to `"orchestrator"` | +| `tools` | `List[str]` | `[]` | Tool names enabled for the agent (e.g., `["calculator", "file_read"]`) | +| `output_path` | `Optional[str]` | `None` | JSONL output file path; auto-generated from benchmark and model name if `None` | +| `seed` | `int` | `42` | Random seed for dataset shuffling | +| `dataset_split` | `Optional[str]` | `None` | Override the dataset split (e.g., `"validation"`, `"test"`) | + +```python +config = RunConfig( + benchmark="supergpqa", + backend="jarvis-direct", + model="qwen3:8b", + max_samples=100, + max_workers=8, + engine_key="ollama", + output_path="results/supergpqa_qwen3-8b.jsonl", +) +``` + +--- + +### RunSummary + +Aggregate statistics produced by `EvalRunner.run()` at the end of a completed run. + +```python +from evals.core.types import RunSummary +``` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `benchmark` | `str` | — | Benchmark name | +| `category` | `str` | — | Task category (inferred from records; falls back to `benchmark` name) | +| `backend` | `str` | — | Backend used | +| `model` | `str` | — | Model identifier | +| `total_samples` | `int` | — | Total records processed (including errors) | +| `scored_samples` | `int` | — | Records where `is_correct` is not `None` | +| `correct` | `int` | — | Records where `is_correct` is `True` | +| `accuracy` | `float` | — | `correct / scored_samples`; rounded to 4 decimal places | +| `errors` | `int` | — | Records where inference or scoring raised an exception | +| `mean_latency_seconds` | `float` | — | Mean wall-clock latency across all successful inferences | +| `total_cost_usd` | `float` | — | Sum of `cost_usd` across all records | +| `per_subject` | `Dict[str, Dict[str, float]]` | `{}` | Per-subject breakdown: `{subject: {accuracy, total, scored, correct}}` | +| `started_at` | `float` | `0.0` | Unix timestamp at run start | +| `ended_at` | `float` | `0.0` | Unix timestamp at run end | + +The runner also writes a `.summary.json` file alongside the JSONL output, containing +the serialized `RunSummary`. + +--- + +## Suite Config Types (`evals.core.types`) + +These dataclasses map directly to sections in a TOML eval suite config file. +They are populated by `load_eval_config()` and consumed by `expand_suite()`. + +### MetaConfig + +```python +@dataclass +class MetaConfig: + name: str = "" + description: str = "" +``` + +Maps to the `[meta]` TOML section. Both fields are optional and used only for +display output in the CLI. + +--- + +### DefaultsConfig + +```python +@dataclass +class DefaultsConfig: + temperature: float = 0.0 + max_tokens: int = 2048 +``` + +Maps to `[defaults]`. These values are the lowest-priority settings in the merge +precedence: `benchmark-level > model-level > [defaults] > built-in defaults`. + +--- + +### JudgeConfig + +```python +@dataclass +class JudgeConfig: + model: str = "gpt-4o" + provider: Optional[str] = None + temperature: float = 0.0 + max_tokens: int = 1024 +``` + +Maps to `[judge]`. The judge model is used by LLM-as-judge scorers (GAIA, FRAMES, +WildChat, SuperGPQA). The `provider` field is reserved for future routing; currently +the judge backend is always constructed with `engine_key="cloud"`. + +--- + +### ExecutionConfig + +```python +@dataclass +class ExecutionConfig: + max_workers: int = 4 + output_dir: str = "results/" + seed: int = 42 +``` + +Maps to `[run]`. `output_dir` is the base directory for all JSONL output files; +individual filenames are auto-generated as `{benchmark}_{model-slug}.jsonl`. + +--- + +### ModelConfig + +```python +@dataclass +class ModelConfig: + name: str = "" + engine: Optional[str] = None + provider: Optional[str] = None + temperature: Optional[float] = None + max_tokens: Optional[int] = None +``` + +Maps to each `[[models]]` entry. `name` is required. `temperature` and `max_tokens` +override `[defaults]` for every benchmark this model runs against, unless a +benchmark-level override also exists. + +--- + +### BenchmarkConfig + +```python +@dataclass +class BenchmarkConfig: + name: str = "" + backend: str = "jarvis-direct" + max_samples: Optional[int] = None + split: Optional[str] = None + agent: Optional[str] = None + tools: List[str] = field(default_factory=list) + judge_model: Optional[str] = None + temperature: Optional[float] = None + max_tokens: Optional[int] = None +``` + +Maps to each `[[benchmarks]]` entry. `name` is required. `backend` must be one of +`"jarvis-direct"` or `"jarvis-agent"`. `judge_model` overrides `[judge].model` for +this benchmark only. + +--- + +### EvalSuiteConfig + +The top-level config object returned by `load_eval_config()`. + +```python +@dataclass +class EvalSuiteConfig: + meta: MetaConfig + defaults: DefaultsConfig + judge: JudgeConfig + run: ExecutionConfig + models: List[ModelConfig] + benchmarks: List[BenchmarkConfig] +``` + +`expand_suite(suite)` iterates over `models x benchmarks` to produce one `RunConfig` +per pair, applying the merge precedence rules documented in `DefaultsConfig`. + +--- + +## Config Module (`evals.core.config`) + +```python +from evals.core.config import load_eval_config, expand_suite, EvalConfigError +``` + +### EvalConfigError + +```python +class EvalConfigError(Exception): ... +``` + +Raised by `load_eval_config()` for structural validation failures: missing required +fields, invalid backend names, or empty `[[models]]` / `[[benchmarks]]` lists. + +--- + +### load_eval_config + +```python +def load_eval_config(path: str | Path) -> EvalSuiteConfig +``` + +Load and validate an eval suite configuration from a TOML file. + +Uses the standard library `tomllib` on Python 3.11+ and the `tomli` backport on +Python 3.10. + +**Parameters:** + +| Parameter | Type | Description | +|-----------|------|-------------| +| `path` | `str \| Path` | Path to the TOML config file | + +**Returns:** `EvalSuiteConfig` + +**Raises:** + +- `EvalConfigError` — structural validation failures (missing `name`, invalid backend, no models/benchmarks defined) +- `FileNotFoundError` — if the config file does not exist + +```python +from evals.core.config import load_eval_config + +suite = load_eval_config("evals/configs/full-suite.toml") +print(f"{len(suite.models)} models, {len(suite.benchmarks)} benchmarks") +``` + +--- + +### expand_suite + +```python +def expand_suite(suite: EvalSuiteConfig) -> List[RunConfig] +``` + +Expand an `EvalSuiteConfig` into a flat list of `RunConfig` objects, one per +model-benchmark pair, with all override layers merged. + +**Merge precedence (highest wins):** + +1. Benchmark-level (`BenchmarkConfig.temperature`, `.max_tokens`, `.judge_model`) +2. Model-level (`ModelConfig.temperature`, `.max_tokens`) +3. Suite defaults (`DefaultsConfig`) +4. Built-in dataclass defaults + +Output paths are auto-generated as `{output_dir}/{benchmark}_{model-slug}.jsonl`, +where `model-slug` replaces `/` and `:` with `-`. + +**Parameters:** + +| Parameter | Type | Description | +|-----------|------|-------------| +| `suite` | `EvalSuiteConfig` | Parsed suite configuration | + +**Returns:** `List[RunConfig]` — one entry per model-benchmark combination. + +```python +from evals.core.config import load_eval_config, expand_suite + +suite = load_eval_config("evals/configs/full-suite.toml") +run_configs = expand_suite(suite) # e.g., 3 models x 4 benchmarks = 12 RunConfigs +for rc in run_configs: + print(f"{rc.benchmark} / {rc.model} -> {rc.output_path}") +``` + +--- + +## Abstract Base Classes + +### InferenceBackend (`evals.core.backend`) + +```python +from evals.core.backend import InferenceBackend +``` + +Base class for all inference backends. A backend wraps an engine or agent and +provides a uniform text-in / text-out interface for the runner. + +```python +class InferenceBackend(ABC): + backend_id: str +``` + +**Class attribute:** + +| Attribute | Type | Description | +|-----------|------|-------------| +| `backend_id` | `str` | Registry identifier (e.g., `"jarvis-direct"`, `"jarvis-agent"`) | + +**Abstract methods:** + +#### generate + +```python +@abstractmethod +def generate( + self, + prompt: str, + *, + model: str, + system: str = "", + temperature: float = 0.0, + max_tokens: int = 2048, +) -> str +``` + +Generate a response and return the text content only. + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `prompt` | `str` | — | User message or formatted problem text | +| `model` | `str` | — | Model identifier | +| `system` | `str` | `""` | Optional system prompt | +| `temperature` | `float` | `0.0` | Sampling temperature | +| `max_tokens` | `int` | `2048` | Maximum output tokens | + +**Returns:** `str` — model text output. + +--- + +#### generate_full + +```python +@abstractmethod +def generate_full( + self, + prompt: str, + *, + model: str, + system: str = "", + temperature: float = 0.0, + max_tokens: int = 2048, +) -> Dict[str, Any] +``` + +Generate a response and return full details including usage and cost metadata. + +**Returns:** `dict` with keys: + +| Key | Type | Description | +|-----|------|-------------| +| `content` | `str` | Model text output | +| `usage` | `dict` | Token usage (`prompt_tokens`, `completion_tokens`) | +| `model` | `str` | Model identifier used | +| `latency_seconds` | `float` | Wall-clock generation time | +| `cost_usd` | `float` | Estimated inference cost | + +--- + +#### close + +```python +def close(self) -> None +``` + +Release resources held by the backend (connections, engine handles, etc.). +The default implementation is a no-op; subclasses override as needed. + +--- + +### DatasetProvider (`evals.core.dataset`) + +```python +from evals.core.dataset import DatasetProvider +``` + +Base class for all evaluation dataset providers. Datasets are loaded lazily via +`load()` and then consumed record-by-record through `iter_records()`. + +```python +class DatasetProvider(ABC): + dataset_id: str + dataset_name: str +``` + +**Class attributes:** + +| Attribute | Type | Description | +|-----------|------|-------------| +| `dataset_id` | `str` | Short identifier matching the CLI benchmark name | +| `dataset_name` | `str` | Human-readable display name | + +**Abstract methods:** + +#### load + +```python +@abstractmethod +def load( + self, + *, + max_samples: Optional[int] = None, + split: Optional[str] = None, + seed: Optional[int] = None, +) -> None +``` + +Load the dataset, optionally downloading from HuggingFace Hub. Must be called +before `iter_records()`. + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `max_samples` | `Optional[int]` | `None` | Truncate to this many records after shuffling | +| `split` | `Optional[str]` | `None` | Dataset split override (e.g., `"test"`, `"validation"`) | +| `seed` | `Optional[int]` | `None` | Shuffle seed; `None` preserves original order | + +--- + +#### iter_records + +```python +@abstractmethod +def iter_records(self) -> Iterable[EvalRecord] +``` + +Iterate over the loaded `EvalRecord` objects. Raises if called before `load()`. + +--- + +#### size + +```python +@abstractmethod +def size(self) -> int +``` + +Return the count of loaded records. + +--- + +### Scorer (`evals.core.scorer`) + +```python +from evals.core.scorer import Scorer, LLMJudgeScorer +``` + +Base class for all scorers. A scorer compares a model's answer to the reference +in an `EvalRecord` and returns a correctness verdict with optional metadata. + +```python +class Scorer(ABC): + scorer_id: str +``` + +**Class attribute:** + +| Attribute | Type | Description | +|-----------|------|-------------| +| `scorer_id` | `str` | Short identifier matching the benchmark name | + +**Abstract method:** + +#### score + +```python +@abstractmethod +def score( + self, + record: EvalRecord, + model_answer: str, +) -> Tuple[Optional[bool], Dict[str, Any]] +``` + +Score a model answer against the reference. + +| Parameter | Type | Description | +|-----------|------|-------------| +| `record` | `EvalRecord` | The source sample including `reference` and `metadata` | +| `model_answer` | `str` | Raw text output from the model | + +**Returns:** `(is_correct, metadata)` tuple where: + +- `is_correct` is `True`, `False`, or `None` (if scoring could not be determined) +- `metadata` is a `dict` of scorer-specific details stored in `EvalResult.scoring_metadata` + +--- + +### LLMJudgeScorer + +```python +class LLMJudgeScorer(Scorer): + def __init__(self, judge_backend: InferenceBackend, judge_model: str) -> None +``` + +Convenience base class for scorers that call an LLM to evaluate answers. +Exposes `_ask_judge()` to subclasses. + +```python +def _ask_judge( + self, + prompt: str, + *, + system: str = "", + temperature: float = 0.0, + max_tokens: int = 1024, +) -> str +``` + +Send a prompt to the judge LLM and return the response text. Delegates to +`judge_backend.generate()`. + +--- + +## EvalRunner (`evals.core.runner`) + +```python +from evals.core.runner import EvalRunner +``` + +The `EvalRunner` wires together a `RunConfig`, `DatasetProvider`, `InferenceBackend`, +and `Scorer` and executes the benchmark. Inference is parallelized using a +`ThreadPoolExecutor`. Results are written to JSONL incrementally so progress is +not lost if the run is interrupted. + +### Constructor + +```python +class EvalRunner: + def __init__( + self, + config: RunConfig, + dataset: DatasetProvider, + backend: InferenceBackend, + scorer: Scorer, + ) -> None +``` + +| Parameter | Type | Description | +|-----------|------|-------------| +| `config` | `RunConfig` | Run parameters (model, workers, output path, etc.) | +| `dataset` | `DatasetProvider` | Dataset to evaluate against | +| `backend` | `InferenceBackend` | Inference backend for generation | +| `scorer` | `Scorer` | Scorer for comparing model answers to references | + +### run + +```python +def run(self) -> RunSummary +``` + +Execute the full evaluation and return aggregate statistics. + +The method: + +1. Calls `dataset.load()` with the `RunConfig` sampling parameters +2. Submits all records to a `ThreadPoolExecutor` with `config.max_workers` threads +3. For each record, calls `backend.generate_full()` then `scorer.score()` +4. Writes each `EvalResult` to a JSONL file as it completes +5. Writes a `.summary.json` alongside the JSONL at the end + +**Returns:** `RunSummary` + +```python title="programmatic_eval.py" +from evals.core.types import RunConfig +from evals.core.runner import EvalRunner +from evals.datasets.supergpqa import SuperGPQADataset +from evals.backends.jarvis_direct import JarvisDirectBackend +from evals.scorers.supergpqa_mcq import SuperGPQAScorer + +config = RunConfig( + benchmark="supergpqa", + backend="jarvis-direct", + model="qwen3:8b", + max_samples=50, + engine_key="ollama", +) + +dataset = SuperGPQADataset() +backend = JarvisDirectBackend(engine_key="ollama") +judge_backend = JarvisDirectBackend(engine_key="cloud") +scorer = SuperGPQAScorer(judge_backend=judge_backend, judge_model="gpt-4o") + +runner = EvalRunner(config, dataset, backend, scorer) +summary = runner.run() + +print(f"Accuracy: {summary.accuracy:.4f} ({summary.correct}/{summary.scored_samples})") +print(f"Mean latency: {summary.mean_latency_seconds:.2f}s") +print(f"Total cost: ${summary.total_cost_usd:.4f}") + +backend.close() +judge_backend.close() +``` + +--- + +## Backends + +### JarvisDirectBackend (`evals.backends.jarvis_direct`) + +```python +from evals.backends.jarvis_direct import JarvisDirectBackend +``` + +Engine-level inference via `SystemBuilder`. Routes directly to the configured +`InferenceEngine` without an agent loop, making it the fastest backend and +appropriate for benchmarks that do not require tool use. + +```python +class JarvisDirectBackend(InferenceBackend): + backend_id = "jarvis-direct" + + def __init__(self, engine_key: Optional[str] = None) -> None +``` + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `engine_key` | `Optional[str]` | `None` | OpenJarvis engine identifier. `None` uses the auto-discovered engine from `~/.openjarvis/config.toml` | + +Telemetry and traces are disabled for eval runs. The backend calls +`SystemBuilder().engine(engine_key).telemetry(False).traces(False).build()`. + +**Compatible benchmarks:** `supergpqa`, `frames`, `wildchat` (any benchmark that +does not require multi-step tool calling). + +=== "Local model" + + ```python + backend = JarvisDirectBackend(engine_key="ollama") + text = backend.generate("What is 2+2?", model="qwen3:8b") + ``` + +=== "Cloud model" + + ```python + backend = JarvisDirectBackend(engine_key="cloud") + text = backend.generate("What is 2+2?", model="gpt-4o") + ``` + +--- + +### JarvisAgentBackend (`evals.backends.jarvis_agent`) + +```python +from evals.backends.jarvis_agent import JarvisAgentBackend +``` + +Agent-level inference via `JarvisSystem.ask()`. Wraps the full OpenJarvis agent +harness, enabling multi-turn tool-calling loops for agentic benchmarks. + +```python +class JarvisAgentBackend(InferenceBackend): + backend_id = "jarvis-agent" + + def __init__( + self, + engine_key: Optional[str] = None, + agent_name: str = "orchestrator", + tools: Optional[List[str]] = None, + ) -> None +``` + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `engine_key` | `Optional[str]` | `None` | OpenJarvis engine identifier | +| `agent_name` | `str` | `"orchestrator"` | Agent to use (`"orchestrator"`, `"react"`, etc.) | +| `tools` | `Optional[List[str]]` | `None` | Tool names to enable (e.g., `["calculator", "file_read"]`) | + +The `generate_full()` return dict includes two additional keys beyond the standard +`InferenceBackend` contract: + +| Key | Type | Description | +|-----|------|-------------| +| `turns` | `int` | Number of agent turns completed | +| `tool_results` | `list` | Tool call results from the agent loop | + +**Compatible benchmarks:** `gaia` (requires file reading and multi-step reasoning). + +```python +backend = JarvisAgentBackend( + engine_key="ollama", + agent_name="orchestrator", + tools=["file_read", "calculator"], +) +result = backend.generate_full( + "How many pages is the attached PDF?", + model="qwen3:8b", +) +print(result["content"]) +print(f"Completed in {result['turns']} turn(s)") +backend.close() +``` + +--- + +## Dataset Providers + +### SuperGPQADataset (`evals.datasets.supergpqa`) + +```python +from evals.datasets.supergpqa import SuperGPQADataset +``` + +Loads the SuperGPQA multiple-choice benchmark from HuggingFace (`m-a-p/SuperGPQA`). +Records have `category="reasoning"` and `subject` set to the discipline subfield. + +```python +class SuperGPQADataset(DatasetProvider): + dataset_id = "supergpqa" + dataset_name = "SuperGPQA" +``` + +- **Default split:** `"train"` +- **HuggingFace path:** `m-a-p/SuperGPQA` +- Each problem is formatted with lettered options (A, B, C, ...) and the instruction + "Respond with the correct letter only." +- `record.reference` is the correct answer letter (e.g., `"B"`). + +--- + +### GAIADataset (`evals.datasets.gaia`) + +```python +from evals.datasets.gaia import GAIADataset +``` + +Loads the GAIA agentic benchmark from HuggingFace (`gaia-benchmark/GAIA`). +Records have `category="agentic"` and `subject` set to `level_1`, `level_2`, or +`level_3`. + +```python +class GAIADataset(DatasetProvider): + dataset_id = "gaia" + dataset_name = "GAIA" + + def __init__(self, cache_dir: Optional[str] = None) -> None +``` + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `cache_dir` | `Optional[str]` | `~/.cache/gaia_benchmark` | Local directory for HuggingFace snapshot download | + +- **Default split:** `"validation"` +- **Default subset:** `"2023_all"` +- Downloads the full dataset snapshot including associated files (PDFs, images, CSVs) + referenced in questions. File paths are embedded in the problem prompt. + +!!! warning "Dataset access" + GAIA requires accepting the HuggingFace dataset terms of service and being logged + in with `huggingface-cli login` before the snapshot download can proceed. + +--- + +### FRAMESDataset (`evals.datasets.frames`) + +```python +from evals.datasets.frames import FRAMESDataset +``` + +Loads the FRAMES multi-hop factual retrieval benchmark from HuggingFace +(`google/frames-benchmark`). Records have `category="rag"` and `subject` set to +the reasoning type(s) (e.g., `"multi-hop, temporal"`). + +```python +class FRAMESDataset(DatasetProvider): + dataset_id = "frames" + dataset_name = "FRAMES" +``` + +- **Default split:** `"test"` +- Wikipedia article links referenced in each question are included in the problem prompt. + +--- + +### WildChatDataset (`evals.datasets.wildchat`) + +```python +from evals.datasets.wildchat import WildChatDataset +``` + +Loads the WildChat-1M dataset (`allenai/WildChat-1M`) and filters to English +single-turn conversations for chat quality evaluation. Records have +`category="chat"` and `subject="conversation"`. + +```python +class WildChatDataset(DatasetProvider): + dataset_id = "wildchat" + dataset_name = "WildChat" +``` + +- **Default split:** `"train"` +- Filters by `language == "english"` and exactly two turns (one user + one assistant). +- `record.problem` is the user message; `record.reference` is the original assistant + response used as the quality baseline by the judge scorer. + +--- + +## Scorers + +### SuperGPQAScorer (`evals.scorers.supergpqa_mcq`) + +```python +from evals.scorers.supergpqa_mcq import SuperGPQAScorer +``` + +LLM-based letter extraction followed by exact match against the reference letter. +The judge LLM extracts the final answer letter from potentially verbose model +responses, then compares it to `record.reference`. + +```python +class SuperGPQAScorer(LLMJudgeScorer): + scorer_id = "supergpqa" +``` + +**Scoring metadata keys:** + +| Key | Description | +|-----|-------------| +| `reference_letter` | Correct answer letter from the dataset | +| `candidate_letter` | Letter extracted by the judge LLM | +| `valid_letters` | Valid answer letters for this question (e.g., `"ABCD"`) | +| `reason` | Set to `"missing_reference_letter"` or `"no_choice_letter_extracted"` on failure | + +--- + +### GAIAScorer (`evals.scorers.gaia_exact`) + +```python +from evals.scorers.gaia_exact import GAIAScorer, exact_match +``` + +Normalized exact match with an LLM fallback for semantic comparison. Tries exact +match first (no API call); falls back to the judge LLM only when exact match fails. + +```python +class GAIAScorer(LLMJudgeScorer): + scorer_id = "gaia" +``` + +**Normalization rules for exact match:** + +- Numbers: strips `$`, `%`, `,` then compares as `float` +- Lists (comma- or semicolon-separated): splits and compares element-by-element +- Strings: lowercases, strips whitespace and punctuation + +**Scoring metadata keys:** + +| Key | Description | +|-----|-------------| +| `match_type` | `"exact"` or `"llm_fallback"` | +| `raw_judge_output` | Full LLM judge response (llm_fallback only) | +| `extracted_answer` | Answer extracted by the judge (llm_fallback only) | + +The `exact_match` helper function is also exported and can be used independently: + +```python +from evals.scorers.gaia_exact import exact_match + +assert exact_match("$1,000", "1000") is True +assert exact_match("paris", "Paris") is True +assert exact_match("3, 5", "3,5") is True +``` + +--- + +### FRAMESScorer (`evals.scorers.frames_judge`) + +```python +from evals.scorers.frames_judge import FRAMESScorer +``` + +LLM-as-judge scorer for FRAMES multi-hop factual retrieval. Uses a structured +grading rubric that focuses on semantic equivalence, ignoring formatting and +capitalization differences. + +```python +class FRAMESScorer(LLMJudgeScorer): + scorer_id = "frames" +``` + +**Scoring metadata keys:** + +| Key | Description | +|-----|-------------| +| `raw_judge_output` | Full LLM judge response | +| `extracted_answer` | Answer extracted by the judge | + +--- + +### WildChatScorer (`evals.scorers.wildchat_judge`) + +```python +from evals.scorers.wildchat_judge import WildChatScorer +``` + +Dual-comparison LLM-as-judge for chat quality. Runs two comparisons — once with +the model answer as Assistant A and once as Assistant B — to reduce position bias. +The model answer is considered correct if it wins or ties in either comparison. + +```python +class WildChatScorer(LLMJudgeScorer): + scorer_id = "wildchat" +``` + +The judge uses a five-point verdict scale: `[[A>>B]]`, `[[A>B]]`, `[[A=B]]`, +`[[B>A]]`, `[[B>>A]]`. A tie (`A=B`) is counted as correct. + +**Scoring metadata keys:** + +| Key | Description | +|-----|-------------| +| `generated_as_a` | `{verdict, response}` from the first comparison pass | +| `generated_as_b` | `{verdict, response}` from the second comparison pass | + +--- + +## CLI Reference + +The evaluation framework ships a `openjarvis-eval` CLI built with Click. + +### openjarvis-eval run + +Run a single benchmark or a full suite from a TOML config. + +```bash title="Single run" +openjarvis-eval run \ + --benchmark supergpqa \ + --model qwen3:8b \ + --engine ollama \ + --max-samples 100 \ + --max-workers 8 \ + --output results/supergpqa_qwen3-8b.jsonl +``` + +```bash title="Suite run from TOML" +openjarvis-eval run --config evals/configs/full-suite.toml +``` + +| Option | Short | Default | Description | +|--------|-------|---------|-------------| +| `--config` | `-c` | `None` | TOML suite config file; enables suite mode | +| `--benchmark` | `-b` | — | Benchmark name (required in single-run mode) | +| `--backend` | | `jarvis-direct` | `jarvis-direct` or `jarvis-agent` | +| `--model` | `-m` | — | Model identifier (required in single-run mode) | +| `--engine` | `-e` | `None` | Engine key override | +| `--agent` | | `orchestrator` | Agent name for `jarvis-agent` backend | +| `--tools` | | `""` | Comma-separated tool names | +| `--max-samples` | `-n` | `None` | Sample limit | +| `--max-workers` | `-w` | `4` | Parallel threads | +| `--judge-model` | | `gpt-4o` | LLM judge model | +| `--output` | `-o` | auto | JSONL output path | +| `--seed` | | `42` | Shuffle seed | +| `--split` | | `None` | Dataset split override | +| `--temperature` | | `0.0` | Sampling temperature | +| `--max-tokens` | | `2048` | Maximum output tokens | +| `--verbose` | `-v` | `False` | Enable debug logging | + +### openjarvis-eval run-all + +Run all four benchmarks against a single model. + +```bash +openjarvis-eval run-all \ + --model qwen3:8b \ + --engine ollama \ + --max-samples 50 \ + --output-dir results/ +``` + +### openjarvis-eval summarize + +Recompute summary statistics from an existing JSONL output file. + +```bash +openjarvis-eval summarize results/supergpqa_qwen3-8b.jsonl +``` + +### openjarvis-eval list + +List all available benchmarks and backends. + +```bash +openjarvis-eval list +``` + +--- + +## TOML Suite Config Format + +A suite config drives a full `models x benchmarks` comparison matrix with a single +command. All sections except `[[models]]` and `[[benchmarks]]` are optional. + +```toml title="evals/configs/full-suite.toml" +[meta] +name = "full-suite-v1" +description = "Evaluate all benchmarks against production models" + +[defaults] +temperature = 0.0 +max_tokens = 2048 + +[judge] +model = "gpt-4o" +temperature = 0.0 +max_tokens = 1024 + +[run] +max_workers = 4 +output_dir = "results/" +seed = 42 + +# One [[models]] entry per model to evaluate +[[models]] +name = "qwen3:8b" +engine = "ollama" +temperature = 0.3 + +[[models]] +name = "gpt-4o" +provider = "openai" + +# One [[benchmarks]] entry per benchmark +[[benchmarks]] +name = "supergpqa" +backend = "jarvis-direct" +max_samples = 200 + +[[benchmarks]] +name = "gaia" +backend = "jarvis-agent" +agent = "orchestrator" +tools = ["file_read", "calculator"] +max_samples = 50 +judge_model = "claude-sonnet-4-20250514" # override judge for this benchmark + +[[benchmarks]] +name = "frames" +backend = "jarvis-direct" +max_samples = 100 + +[[benchmarks]] +name = "wildchat" +backend = "jarvis-direct" +max_samples = 150 +temperature = 0.7 +``` + +```bash +openjarvis-eval run --config evals/configs/full-suite.toml +# Suite: full-suite-v1 +# 2 model(s) x 4 benchmark(s) = 8 run(s) +``` + +--- + +## See Also + +- [Benchmarks Module](bench.md) — `openjarvis.bench` performance benchmarks (latency, throughput) for the inference engine, separate from the eval framework +- [Telemetry & Traces](telemetry.md) — `openjarvis.telemetry` and `openjarvis.traces` for production monitoring +- [Python SDK](../user-guide/python-sdk.md) — `Jarvis` class used internally by eval backends +- [Agents](../user-guide/agents.md) — Agent implementations invoked by `JarvisAgentBackend` diff --git a/docs/api/index.md b/docs/api/index.md index a3bfd203..8db37f6c 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -22,4 +22,5 @@ classes, concrete implementations, and utility functions. | [Traces](traces.md) | Trace storage, collection, and analysis | | [Telemetry](telemetry.md) | Telemetry storage, aggregation, and instrumented wrappers | | [Benchmarks](bench.md) | Benchmark ABC, suite runner, latency and throughput benchmarks | +| [Evals](evals.md) | Evaluation framework: backends, dataset providers, scorers, and suite runner | | [Server](server.md) | FastAPI application, OpenAI-compatible routes, and Pydantic models | diff --git a/docs/architecture/engine.md b/docs/architecture/engine.md index 57703ce9..f46c2bba 100644 --- a/docs/architecture/engine.md +++ b/docs/architecture/engine.md @@ -234,27 +234,39 @@ Key behaviors: ## Configuration -Engine hosts and defaults are configured in `~/.openjarvis/config.toml`: +Engine hosts and defaults are configured in `~/.openjarvis/config.toml` using **nested per-engine sub-sections**: ```toml [engine] default = "ollama" -ollama_host = "http://localhost:11434" -vllm_host = "http://localhost:8000" -llamacpp_host = "http://localhost:8080" -sglang_host = "http://localhost:30000" + +[engine.ollama] +host = "http://localhost:11434" + +[engine.vllm] +host = "http://localhost:8000" + +[engine.sglang] +host = "http://localhost:30000" + +# [engine.llamacpp] +# host = "http://localhost:8080" +# binary_path = "" ``` -The `EngineConfig` dataclass maps these settings: +The `EngineConfig` dataclass and its per-engine sub-dataclasses map these settings: -| Field | Default | Description | -|-------|---------|-------------| -| `default` | `"ollama"` (hardware-dependent) | Preferred engine backend | -| `ollama_host` | `http://localhost:11434` | Ollama server URL | -| `vllm_host` | `http://localhost:8000` | vLLM server URL | -| `llamacpp_host` | `http://localhost:8080` | llama.cpp server URL | -| `sglang_host` | `http://localhost:30000` | SGLang server URL | -| `llamacpp_path` | `""` | Path to llama.cpp binary (for managed mode) | +| Config Class | Field | Default | Description | +|---|---|---|---| +| `EngineConfig` | `default` | `"ollama"` (hardware-dependent) | Preferred engine backend | +| `OllamaEngineConfig` | `host` | `http://localhost:11434` | Ollama server URL | +| `VLLMEngineConfig` | `host` | `http://localhost:8000` | vLLM server URL | +| `SGLangEngineConfig` | `host` | `http://localhost:30000` | SGLang server URL | +| `LlamaCppEngineConfig` | `host` | `http://localhost:8080` | llama.cpp server URL | +| `LlamaCppEngineConfig` | `binary_path` | `""` | Path to llama.cpp binary (for managed mode) | + +!!! note "Backward compatibility" + The old flat field names `ollama_host`, `vllm_host`, `llamacpp_host`, `llamacpp_path`, and `sglang_host` under `[engine]` are still accepted as backward-compatible properties on `EngineConfig`. New configurations should use the nested sub-section format. --- diff --git a/docs/architecture/intelligence.md b/docs/architecture/intelligence.md index 5f6b603f..6fa41a06 100644 --- a/docs/architecture/intelligence.md +++ b/docs/architecture/intelligence.md @@ -1,18 +1,21 @@ # Intelligence Pillar -The Intelligence pillar handles **model management and query routing**. It maintains a catalog of known models with detailed metadata and provides a heuristic router that selects the best model for a given query based on its characteristics. +The Intelligence pillar represents **the model** — its identity, weights, quantization format, fallback chain, and the catalog of well-known models with detailed metadata. It no longer contains routing logic; query analysis and model selection have moved to the [Learning pillar](learning.md). --- ## Purpose -When a user sends a query to OpenJarvis, the system needs to decide which model should handle it. A short, simple question like "What time is it?" does not need a 70B parameter model, while a complex multi-step math problem benefits from the largest available model. The Intelligence pillar encapsulates this decision-making logic. +The Intelligence pillar answers a single question: *what is the model?* It maintains a catalog of known models with metadata (parameter count, context length, VRAM requirements, supported engines) and provides helpers for registering built-in models and merging models discovered from running engines at runtime. The pillar provides three key capabilities: 1. **Model catalog** -- a registry of well-known models with metadata (parameter count, context length, VRAM requirements, supported engines) -2. **Query routing** -- analyzing query characteristics and selecting the optimal model -3. **Auto-discovery** -- merging models discovered from running engines into the catalog +2. **Auto-discovery** -- merging models discovered from running engines into the catalog +3. **Model configuration** -- `IntelligenceConfig` captures the local model's identity, weight paths, quantization, and preferred engine + +!!! info "Routing has moved" + Query analysis (`build_routing_context`) and model selection (`HeuristicRouter`, `RouterPolicy` ABC) now live in the [Learning pillar](learning.md). Backward-compatible re-exports remain in `intelligence/_stubs.py` and `intelligence/router.py` so existing code continues to work. --- @@ -118,104 +121,138 @@ For each model ID not already in the registry, a `ModelSpec` is created with the --- -## HeuristicRouter +## IntelligenceConfig -The `HeuristicRouter` is a rule-based model router that selects the best model based on query characteristics. It applies six priority rules in order: - -### Routing Rules - -| Priority | Rule | Condition | Action | -|----------|------|-----------|--------| -| 1 | Code detection | Query contains code patterns (backticks, `def`, `class`, `import`, `function`, `=>`, etc.) | Prefer model with "code" or "coder" in name; fall back to largest model | -| 2 | Math detection | Query contains math keywords (`solve`, `integral`, `equation`, `calculate`, `compute`, etc.) | Select the largest available model | -| 3 | Short query | Query length < 50 characters, no code/math | Select the smallest available model (faster response) | -| 4 | Long/complex query | Query length > 500 characters OR contains reasoning keywords (`explain`, `analyze`, `compare`, `step-by-step`, etc.) | Select the largest available model | -| 5 | High urgency | `urgency > 0.8` | Override to smallest model (fastest response) | -| 6 | Default fallback | None of the above match | Use `default_model`, then `fallback_model`, then first available | - -!!! note "Priority 5 overrides all others" - The urgency check (rule 5) is actually evaluated **first** in the code -- if urgency exceeds 0.8, the router immediately returns the smallest model regardless of query content. - -### Usage +The `IntelligenceConfig` dataclass (in `core/config.py`) captures the full identity of the model the system is configured to use, as well as the default sampling parameters for generation: ```python -from openjarvis.intelligence import HeuristicRouter, build_routing_context - -router = HeuristicRouter( - available_models=["qwen3:8b", "llama3.2:3b", "deepseek-coder-v2:16b"], - default_model="qwen3:8b", - fallback_model="llama3.2:3b", -) - -ctx = build_routing_context("Write a Python function to sort a list") -model = router.select_model(ctx) # Returns "deepseek-coder-v2:16b" (has "coder") -``` - ---- - -## RouterPolicy and QueryAnalyzer ABCs - -The Intelligence pillar now defines the `RouterPolicy` and `QueryAnalyzer` ABCs in `intelligence/_stubs.py`: - -```python -# intelligence/_stubs.py -class RouterPolicy(ABC): - @abstractmethod - def select_model(self, context: RoutingContext) -> str: - """Return the model registry key best suited for *context*.""" - -class QueryAnalyzer(ABC): - @abstractmethod - def analyze(self, query: str) -> RoutingContext: - """Analyze a raw query string and return a RoutingContext.""" -``` - -The `HeuristicRouter` implements both of these ABCs. - ---- - -## build_routing_context() - -The `build_routing_context()` function analyzes a raw query string and produces a `RoutingContext` dataclass (now defined in `core/types.py`): - -```python -# core/types.py @dataclass(slots=True) -class RoutingContext: - query: str = "" - query_length: int = 0 - has_code: bool = False - has_math: bool = False - language: str = "en" - urgency: float = 0.5 # 0 = low priority, 1 = real-time - metadata: Dict[str, Any] = field(default_factory=dict) +class IntelligenceConfig: + """The model — identity, paths, quantization, fallback chain, and generation defaults.""" + + default_model: str = "" # Primary model key (e.g., "qwen3:8b") + fallback_model: str = "" # Fallback when default is unavailable + model_path: str = "" # Local weights (HF repo, GGUF file, etc.) + checkpoint_path: str = "" # Checkpoint/adapter path (e.g., LoRA) + quantization: str = "none" # none, fp8, int8, int4, gguf_q4, gguf_q8 + preferred_engine: str = "" # Override engine for this model (e.g., "vllm") + provider: str = "" # local, openai, anthropic, google + # Generation defaults (overridable per-call) + temperature: float = 0.7 + max_tokens: int = 1024 + top_p: float = 0.9 + top_k: int = 40 + repetition_penalty: float = 1.0 + stop_sequences: str = "" # Comma-separated stop strings ``` -**Code detection** uses regex patterns matching: +### Model Identity Fields -- Backtick code blocks (`` ``` `` or `` `inline` ``) -- Language keywords (`def`, `class`, `import`, `function`, `const`, `var`, `let`) -- Syntax patterns (`if (`, `->`, `=>`, `{ }`, `for x in`, `#include`, `System.out`) +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `default_model` | `str` | `""` | Primary model registry key. Resolved at startup; overrides any engine default. | +| `fallback_model` | `str` | `""` | Used when the default model is not available on any running engine. | +| `model_path` | `str` | `""` | Path or HuggingFace repo ID for local weights (e.g., `"./models/qwen3-8b.gguf"` or `"Qwen/Qwen3-8B"`). | +| `checkpoint_path` | `str` | `""` | Path to a fine-tuned checkpoint or LoRA adapter directory. | +| `quantization` | `str` | `"none"` | Quantization format. Accepted values: `none`, `fp8`, `int8`, `int4`, `gguf_q4`, `gguf_q8`. | +| `preferred_engine` | `str` | `""` | When set, `SystemBuilder`, `sdk.py`, and `cli/ask.py` use this engine key instead of `config.engine.default`. | +| `provider` | `str` | `""` | Model provider hint: `local`, `openai`, `anthropic`, `google`. Used by the Cloud engine backend to route API calls. | -**Math detection** uses regex patterns matching: +### Generation Default Fields -- Mathematical terms (`solve`, `integral`, `equation`, `proof`, `derivative`, `matrix`) -- Computational keywords (`calculate`, `compute`, `sigma`, `sum`, `limit`, `probability`) +These fields set the default sampling parameters for every inference call. Individual calls can override them by passing keyword arguments to `engine.generate()`. + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `temperature` | `float` | `0.7` | Sampling temperature. Lower values produce more deterministic output; higher values increase diversity. | +| `max_tokens` | `int` | `1024` | Maximum number of tokens to generate per call. | +| `top_p` | `float` | `0.9` | Nucleus sampling probability mass. At each step, only tokens comprising the top-p probability mass are considered. | +| `top_k` | `int` | `40` | Top-k sampling: only consider the top-k most likely tokens at each step. | +| `repetition_penalty` | `float` | `1.0` | Penalize repeated token sequences. Values greater than 1.0 reduce repetition. | +| `stop_sequences` | `str` | `""` | Comma-separated stop strings. Generation halts when any stop string appears in the output. | + +!!! note "Moved from Agent" + Generation parameters (`temperature`, `max_tokens`) previously lived under `[agent]` in the config file. They now live under `[intelligence]`. Old configs with these fields under `[agent]` are automatically migrated at load time. See the [configuration migration guide](../getting-started/configuration.md#migration-guide) for details. + +### TOML Configuration + +```toml +[intelligence] +default_model = "qwen3:8b" +fallback_model = "llama3.2:3b" +temperature = 0.7 +max_tokens = 1024 +# top_p = 0.9 +# top_k = 40 +# repetition_penalty = 1.0 +# stop_sequences = "" + +# Local weight overrides (optional) +# model_path = "./models/qwen3-8b-instruct.gguf" +# checkpoint_path = "./checkpoints/my-lora" +# quantization = "gguf_q4" + +# Engine selection for this model (takes priority over [engine].default) +# preferred_engine = "vllm" + +# Provider for cloud models +# provider = "openai" +``` + +### Engine Selection Priority + +When resolving which engine to use, `SystemBuilder`, `sdk.py`, and `cli/ask.py` check `config.intelligence.preferred_engine` before `config.engine.default`: + +``` +1. Explicit --engine CLI flag or engine_key= SDK parameter +2. config.intelligence.preferred_engine ← new field +3. config.engine.default +4. First healthy engine discovered at runtime +``` + +This lets you pin a specific model to a specific engine without changing the global engine default. For example, a GGUF quantized model can be pinned to `llamacpp` while the global default remains `ollama`: + +```toml +[engine] +default = "ollama" + +[intelligence] +default_model = "llama3.2:3b" +model_path = "./models/llama-3.2-3b.Q4_K_M.gguf" +quantization = "gguf_q4" +preferred_engine = "llamacpp" +``` + +--- + +## Public API + +`intelligence/__init__.py` exports exactly three names: ```python -from openjarvis.intelligence import build_routing_context - -ctx = build_routing_context("Solve the integral of x^2 dx") -# ctx.has_math = True, ctx.has_code = False, ctx.query_length = 32 - -ctx = build_routing_context("```python\ndef hello():\n pass\n```") -# ctx.has_code = True, ctx.has_math = False +from openjarvis.intelligence import ( + BUILTIN_MODELS, # List[ModelSpec] — the full built-in catalog + merge_discovered_models, # (engine_key, model_ids) -> None + register_builtin_models, # () -> None +) ``` +### Backward-Compatibility Shims + +The following names are still importable from `openjarvis.intelligence` via shim modules, but their canonical locations have moved: + +| Name | Old location | Canonical location | +|------|-------------|-------------------| +| `RouterPolicy` | `intelligence/_stubs.py` | `learning/_stubs.py` | +| `QueryAnalyzer` | `intelligence/_stubs.py` | `learning/_stubs.py` | +| `HeuristicRouter` | `intelligence/router.py` | `learning/router.py` | +| `build_routing_context` | `intelligence/router.py` | `learning/router.py` | +| `DefaultQueryAnalyzer` | `intelligence/router.py` | `learning/router.py` | + +New code should import from the canonical `learning.*` locations. The shims in `intelligence/_stubs.py` and `intelligence/router.py` are retained for backward compatibility only. + --- ## Integration with Learning -The `HeuristicRouter` implements the `RouterPolicy` ABC (now defined in `intelligence/_stubs.py`), which means it can be swapped out for a `TraceDrivenPolicy`, `SFTRouterPolicy`, or any other policy via the `RouterPolicyRegistry`. See the [Learning & Traces](learning.md) documentation for details on how trace-driven routing and the broader `LearningPolicy` taxonomy work. - -The router is registered as `"heuristic"` in the `RouterPolicyRegistry` and is the default routing policy. Users can switch policies via the `--router` CLI flag or the `learning.default_policy` config setting. +The Learning pillar consumes the model catalog to make routing decisions. The `HeuristicRouter` and `TraceDrivenPolicy` both read `ModelRegistry` to compare model sizes when selecting between candidates. See the [Learning & Traces](learning.md) documentation for full details on routing policies, the `RouterPolicy` ABC, and the trace-driven feedback loop. diff --git a/docs/architecture/learning.md b/docs/architecture/learning.md index b00b5ae2..a316301c 100644 --- a/docs/architecture/learning.md +++ b/docs/architecture/learning.md @@ -17,10 +17,10 @@ All learning policies are registered in the `LearningRegistry` (in `core/registr ## RouterPolicy ABC -The `RouterPolicy` ABC and the `QueryAnalyzer` ABC are defined in `intelligence/_stubs.py`: +The `RouterPolicy` ABC and the `QueryAnalyzer` ABC are defined in `learning/_stubs.py`: ```python -# intelligence/_stubs.py +# learning/_stubs.py class RouterPolicy(ABC): @abstractmethod def select_model(self, context: RoutingContext) -> str: @@ -33,7 +33,7 @@ class QueryAnalyzer(ABC): ``` !!! note "Backward compatibility" - The `RouterPolicy` and `RoutingContext` names are still importable from `openjarvis.learning._stubs` via backward-compatibility re-exports, but the canonical locations are now `openjarvis.intelligence._stubs` (for `RouterPolicy` and `QueryAnalyzer`) and `openjarvis.core.types` (for `RoutingContext`). + The canonical locations are now `openjarvis.learning._stubs` (for `RouterPolicy` and `QueryAnalyzer`) and `openjarvis.core.types` (for `RoutingContext`). The old `openjarvis.intelligence._stubs` import path still works via a backward-compatibility shim, but new code should import from `openjarvis.learning._stubs`. ### RoutingContext @@ -77,8 +77,8 @@ And these additional learning policies (registered in `LearningRegistry`): Users select a policy via `config.toml` or the `--router` CLI flag: ```toml -[learning] -default_policy = "heuristic" +[learning.routing] +policy = "heuristic" ``` ```bash @@ -104,9 +104,65 @@ This ensures that policies are available even after `RouterPolicyRegistry.clear( ## HeuristicRouter (Heuristic Policy) -The `HeuristicRouter` is the default routing policy. It uses static rules to select models based on query characteristics. See the [Intelligence Pillar](intelligence.md) documentation for full details on its six priority rules. +The `HeuristicRouter` is the default routing policy. It is defined in `learning/router.py` and applies six static priority rules to select the best model based on query characteristics. -The `heuristic_policy.py` module wires the existing `HeuristicRouter` (from the Intelligence pillar) into the `RouterPolicyRegistry`: +### Routing Rules + +| Priority | Rule | Condition | Action | +|----------|------|-----------|--------| +| 1 | Code detection | Query contains code patterns (backticks, `def`, `class`, `import`, `function`, `=>`, etc.) | Prefer model with "code" or "coder" in name; fall back to largest model | +| 2 | Math detection | Query contains math keywords (`solve`, `integral`, `equation`, `calculate`, `compute`, etc.) | Select the largest available model | +| 3 | Short query | Query length < 50 characters, no code/math | Select the smallest available model (faster response) | +| 4 | Long/complex query | Query length > 500 characters OR contains reasoning keywords (`explain`, `analyze`, `compare`, `step-by-step`, etc.) | Select the largest available model | +| 5 | High urgency | `urgency > 0.8` | Override to smallest model (fastest response) | +| 6 | Default fallback | None of the above match | Use `default_model`, then `fallback_model`, then first available | + +!!! note "Priority 5 overrides all others" + The urgency check (rule 5) is evaluated **first** in the code — if urgency exceeds 0.8, the router immediately returns the smallest model regardless of query content. + +### Usage + +```python +from openjarvis.learning.router import HeuristicRouter, build_routing_context + +router = HeuristicRouter( + available_models=["qwen3:8b", "llama3.2:3b", "deepseek-coder-v2:16b"], + default_model="qwen3:8b", + fallback_model="llama3.2:3b", +) + +ctx = build_routing_context("Write a Python function to sort a list") +model = router.select_model(ctx) # Returns "deepseek-coder-v2:16b" (has "coder") +``` + +### build_routing_context() + +The `build_routing_context()` function (in `learning/router.py`) analyzes a raw query string and produces a `RoutingContext` dataclass: + +```python +from openjarvis.learning.router import build_routing_context + +ctx = build_routing_context("Solve the integral of x^2 dx") +# ctx.has_math = True, ctx.has_code = False, ctx.query_length = 32 + +ctx = build_routing_context("```python\ndef hello():\n pass\n```") +# ctx.has_code = True, ctx.has_math = False +``` + +**Code detection** uses regex patterns matching: + +- Backtick code blocks (` ``` ` or `` `inline` ``) +- Language keywords (`def`, `class`, `import`, `function`, `const`, `var`, `let`) +- Syntax patterns (`if (`, `->`, `=>`, `{ }`, `for x in`, `#include`, `System.out`) + +**Math detection** uses regex patterns matching: + +- Mathematical terms (`solve`, `integral`, `equation`, `proof`, `derivative`, `matrix`) +- Computational keywords (`calculate`, `compute`, `sigma`, `sum`, `limit`, `probability`) + +### Registration + +The `heuristic_policy.py` module wires `HeuristicRouter` into the `RouterPolicyRegistry`: ```python # learning/heuristic_policy.py diff --git a/docs/architecture/memory.md b/docs/architecture/memory.md index 0213097b..02d5638a 100644 --- a/docs/architecture/memory.md +++ b/docs/architecture/memory.md @@ -335,16 +335,21 @@ class MyMemoryBackend(MemoryBackend): def clear(self) -> None: ... ``` -The default backend is configured in `~/.openjarvis/config.toml`: +The default backend is configured in `~/.openjarvis/config.toml`. Storage settings live under `[tools.storage]`, and context injection is controlled by `agent.context_from_memory`: ```toml -[memory] +[agent] +context_from_memory = true + +[tools.storage] default_backend = "sqlite" db_path = "~/.openjarvis/memory.db" -context_injection = true context_top_k = 5 context_min_score = 0.1 context_max_tokens = 2048 chunk_size = 512 chunk_overlap = 64 ``` + +!!! note "Backward compatibility" + The `[memory]` TOML section is still accepted as a backward-compatible alias for `[tools.storage]`. The old `context_injection` field is automatically migrated to `agent.context_from_memory` at load time. diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 17532f82..b838abbd 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -9,7 +9,7 @@ OpenJarvis is a research framework for studying on-device AI systems. Its archit ```mermaid graph TB subgraph "Core Pillars" - INT["Intelligence
Model management
& query routing"] + INT["Intelligence
Model definition
& catalog"] ENG["Engine
Inference runtime
backends"] AGT["Agentic Logic
Agents, tools,
orchestration"] MEM["Memory
Persistent searchable
storage"] @@ -28,7 +28,6 @@ graph TB AGT -->|"selects model via"| INT AGT -->|"generates via"| ENG AGT -->|"retrieves context from"| MEM - INT -->|"routes using"| LRN LRN -->|"learns from traces of"| AGT LRN -->|"updates routing for"| INT @@ -51,31 +50,33 @@ graph TB ### Intelligence -The Intelligence pillar handles **model management and query routing**. It maintains a catalog of known models (`BUILTIN_MODELS`) with metadata such as parameter count, context length, VRAM requirements, and supported engines. When a query arrives, the `HeuristicRouter` analyzes it for characteristics like code patterns, math keywords, length, and urgency, then selects the most appropriate model from those available. +The Intelligence pillar handles **model definition and catalog**. It maintains a catalog of known models (`BUILTIN_MODELS`) with metadata such as parameter count, context length, VRAM requirements, and supported engines. The `IntelligenceConfig` captures the full identity of the configured model — its weight path, quantization format, preferred engine, fallback chain, and generation defaults (`temperature`, `max_tokens`, `top_p`, `top_k`, `repetition_penalty`, `stop_sequences`). -Models discovered at runtime from running engines are automatically merged into the `ModelRegistry`, so the system always has an up-to-date view of what is available. +Models discovered at runtime from running engines are automatically merged into the `ModelRegistry`, so the system always has an up-to-date view of what is available. Query routing has moved to the Learning pillar — see the [Learning & Traces](learning.md) documentation. ### Engine -The Engine pillar provides the **inference runtime** -- the layer that actually runs language models. All backends implement the `InferenceEngine` ABC with a uniform interface: `generate()`, `stream()`, `list_models()`, and `health()`. Supported backends include Ollama, vLLM, SGLang, llama.cpp, and Cloud (OpenAI, Anthropic, Google). +The Engine pillar provides the **inference runtime** — the layer that actually runs language models. All backends implement the `InferenceEngine` ABC with a uniform interface: `generate()`, `stream()`, `list_models()`, and `health()`. Supported backends include Ollama, vLLM, SGLang, llama.cpp, and Cloud (OpenAI, Anthropic, Google). -Engine discovery probes all registered backends for health, returning healthy engines sorted with the user's configured default first. The system automatically falls back to any available engine if the preferred one is unavailable. +Each engine is configured via its own sub-section in `config.toml` (e.g., `[engine.ollama]`, `[engine.vllm]`, `[engine.llamacpp]`). Engine discovery probes all registered backends for health, returning healthy engines sorted with the user's configured default first. The system automatically falls back to any available engine if the preferred one is unavailable. ### Agentic Logic The Agentic Logic pillar implements **pluggable agents** that handle queries with varying levels of sophistication. `SimpleAgent` provides single-turn query-to-response without tools. `OrchestratorAgent` implements a multi-turn tool-calling loop where the LLM can invoke tools like `calculator`, `think`, `retrieval`, `llm`, and `file_read`, with results fed back for further processing. `OpenClawAgent` communicates with external OpenClaw servers via HTTP or subprocess transport. -All agents implement the `BaseAgent` ABC with a `run()` method, and are registered via `@AgentRegistry.register("name")`. +Agent behavior is configured through `[agent]` in `config.toml`, including the default agent, turn limits, tool list, optional system prompt, and the `context_from_memory` flag (previously `context_injection`) that controls automatic memory context injection. All agents implement the `BaseAgent` ABC with a `run()` method, and are registered via `@AgentRegistry.register("name")`. ### Memory -The Memory pillar provides **persistent, searchable storage** for documents and knowledge. Five backends are available: SQLite/FTS5 (zero-dependency default), FAISS (dense vector retrieval), ColBERTv2 (late interaction), BM25 (classic term-frequency), and Hybrid (Reciprocal Rank Fusion of sparse + dense). +The Memory pillar provides **persistent, searchable storage** for documents and knowledge. Five backends are available: SQLite/FTS5 (zero-dependency default), FAISS (dense vector retrieval), ColBERTv2 (late interaction), BM25 (classic term-frequency), and Hybrid (Reciprocal Rank Fusion of sparse + dense). Storage backends are configured under `[tools.storage]` in `config.toml` (the `[memory]` section is still accepted as a backward-compatible alias). -The memory pipeline includes document ingestion, chunking, embedding generation, and context injection. When a user sends a query, relevant documents are retrieved and prepended to the prompt with source attribution. +The memory pipeline includes document ingestion, chunking, embedding generation, and context injection. When a user sends a query and `agent.context_from_memory` is enabled, relevant documents are retrieved and prepended to the prompt with source attribution. ### Learning & Traces (Cross-cutting) -The Learning system is a cross-cutting concern that connects all pillars through **trace-driven feedback**. Every agent interaction can produce a `Trace` capturing the full sequence of steps -- routing decisions, memory retrieval, inference calls, tool invocations, and final responses. The `TraceAnalyzer` computes statistics from accumulated traces, and the `TraceDrivenPolicy` uses these statistics to learn which model/agent/tool combinations produce the best outcomes for different query types. +The Learning system is a cross-cutting concern that connects all pillars through **trace-driven feedback**. Every agent interaction can produce a `Trace` capturing the full sequence of steps — routing decisions, memory retrieval, inference calls, tool invocations, and final responses. The `TraceAnalyzer` computes statistics from accumulated traces, and the `TraceDrivenPolicy` uses these statistics to learn which model/agent/tool combinations produce the best outcomes for different query types. + +The learning system is configured through nested sub-sections in `config.toml`: `[learning.routing]` controls the router policy (heuristic, learned, sft, grpo), `[learning.intelligence]` controls the model-level learning policy, `[learning.agent]` controls agent advisor and ICL updater policies, and `[learning.metrics]` sets the composite reward function weights. --- @@ -134,10 +135,10 @@ src/openjarvis/ config.py JarvisConfig, hardware detection, TOML loading events.py EventBus pub/sub system (EventType, Event) - intelligence/ Intelligence pillar -- model management & routing - router.py HeuristicRouter, build_routing_context() + intelligence/ Intelligence pillar -- model definition & catalog model_catalog.py BUILTIN_MODELS list, merge_discovered_models() - _stubs.py (Shared with learning -- RoutingContext) + _stubs.py (backward-compat shim -- re-exports from learning._stubs) + router.py (backward-compat shim -- re-exports from learning.router) engine/ Engine pillar -- inference runtime backends _stubs.py InferenceEngine ABC @@ -173,7 +174,8 @@ src/openjarvis/ embeddings.py Embedder ABC, SentenceTransformerEmbedder learning/ Learning system -- router policies & rewards - _stubs.py RouterPolicy ABC, RewardFunction ABC, RoutingContext + _stubs.py RouterPolicy ABC, QueryAnalyzer ABC, RewardFunction ABC, RoutingContext + router.py HeuristicRouter, DefaultQueryAnalyzer, build_routing_context() heuristic_policy.py Wires HeuristicRouter into RouterPolicyRegistry trace_policy.py TraceDrivenPolicy (learns from trace outcomes) grpo_policy.py GRPORouterPolicy (stub for future RL) diff --git a/docs/architecture/query-flow.md b/docs/architecture/query-flow.md index 219ed904..0261de6e 100644 --- a/docs/architecture/query-flow.md +++ b/docs/architecture/query-flow.md @@ -11,7 +11,7 @@ sequenceDiagram actor User participant CLI as CLI / SDK participant CFG as Config & Discovery - participant RTR as Router Policy + participant LRN as Learning (Router) participant AGT as Agent participant MEM as Memory Backend participant CTX as Context Injection @@ -30,8 +30,8 @@ sequenceDiagram CFG-->>CLI: available models per engine alt Model not specified - CLI->>RTR: select_model(RoutingContext) - RTR-->>CLI: model_key (e.g., "qwen3:8b") + CLI->>LRN: select_model(RoutingContext) + LRN-->>CLI: model_key (e.g., "qwen3:8b") end alt Agent mode (--agent flag) @@ -150,9 +150,10 @@ If no model was explicitly specified, the router policy selects one: ```python from openjarvis.learning import ensure_registered +from openjarvis.learning.router import build_routing_context ensure_registered() # Ensure learning policies are registered -policy_key = router_policy or config.learning.default_policy +policy_key = router_policy or config.learning.routing.policy router_cls = RouterPolicyRegistry.get(policy_key) router = router_cls( available_models=all_models.get(engine_name, []), @@ -164,7 +165,7 @@ ctx = build_routing_context(query_text) model_name = router.select_model(ctx) ``` -The `build_routing_context()` function analyzes the query for code patterns, math keywords, length, and urgency. The router then applies its rules (heuristic or learned) to select the optimal model. +The `build_routing_context()` function (in `learning/router.py`) analyzes the query for code patterns, math keywords, length, and urgency. The router then applies its rules (heuristic or learned) to select the optimal model. ### Step 5: Memory Context Injection diff --git a/docs/development/extending.md b/docs/development/extending.md index ef69cc03..a613a6f8 100644 --- a/docs/development/extending.md +++ b/docs/development/extending.md @@ -711,7 +711,7 @@ class BenchmarkResult: ## Adding a New Router Policy Router policies determine which model handles a given query. All policies -implement the `RouterPolicy` ABC from `intelligence/_stubs.py`. The +implement the `RouterPolicy` ABC from `learning/_stubs.py`. The `RoutingContext` dataclass is defined in `core/types.py`. ### Complete Example @@ -727,7 +727,7 @@ from typing import List, Optional from openjarvis.core.registry import RouterPolicyRegistry from openjarvis.core.types import RoutingContext -from openjarvis.intelligence._stubs import RouterPolicy +from openjarvis.learning._stubs import RouterPolicy class QueryLengthPolicy(RouterPolicy): @@ -797,8 +797,8 @@ Once registered, your policy can be selected via the config file or CLI: === "Config (TOML)" ```toml - [learning] - default_policy = "query_length" + [learning.routing] + policy = "query_length" ``` === "CLI" @@ -823,7 +823,7 @@ class RoutingContext: metadata: Dict[str, Any] = field(default_factory=dict) ``` -The `build_routing_context()` helper in `intelligence/router.py` populates +The `build_routing_context()` helper in `learning/router.py` populates this from a raw query string, detecting code and math patterns automatically. --- @@ -837,7 +837,7 @@ this from a raw query string, detecting code and math patterns automatically. | Agent | `BaseAgent` | `AgentRegistry` | `agents/_stubs.py` | | Tool | `BaseTool` | `ToolRegistry` | `tools/_stubs.py` | | Benchmark | `BaseBenchmark` | `BenchmarkRegistry` | `bench/_stubs.py` | -| Router Policy | `RouterPolicy` | `RouterPolicyRegistry` | `intelligence/_stubs.py` | +| Router Policy | `RouterPolicy` | `RouterPolicyRegistry` | `learning/_stubs.py` | | Learning Policy | `LearningPolicy` | `LearningRegistry` | `learning/_stubs.py` | The general pattern for all extension points: diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index 33ec6fc9..9f4ffe36 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -5,7 +5,7 @@ description: Complete reference for OpenJarvis configuration # Configuration -OpenJarvis uses a TOML configuration file to control engine selection, model routing, memory backends, agent behavior, and more. This page is the complete reference for every configuration option. +OpenJarvis uses a TOML configuration file to control engine selection, model identity, memory backends, agent behavior, and more. This page is the complete reference for every configuration option, organized by pillar. ## Config File Location @@ -44,81 +44,245 @@ jarvis init --force ## Configuration Sections -The config file is organized into TOML sections. Every field has a default value, so you only need to specify the values you want to change. +The config file is organized into TOML sections corresponding to the five pillars. Every field has a default value, so you only need to specify values you want to change. --- -### `[engine]` -- Inference Engine +### `[engine]` — Inference Engine -Controls which inference engine is used and where each engine is listening. +Controls which inference engine is used and how each engine is reached. Engine settings are now **nested** under per-engine sub-sections instead of flat fields. ```toml [engine] default = "ollama" -ollama_host = "http://localhost:11434" -vllm_host = "http://localhost:8000" -llamacpp_host = "http://localhost:8080" -llamacpp_path = "" -sglang_host = "http://localhost:30000" + +[engine.ollama] +host = "http://localhost:11434" + +[engine.vllm] +host = "http://localhost:8000" + +[engine.sglang] +host = "http://localhost:30000" + +# [engine.llamacpp] +# host = "http://localhost:8080" +# binary_path = "" ``` +**`[engine]` top-level:** + | Field | Type | Default | Description | |-------|------|---------|-------------| | `default` | string | Auto-detected | Default engine backend. One of: `ollama`, `vllm`, `llamacpp`, `sglang`, `cloud`. Set automatically by `jarvis init` based on hardware detection. | -| `ollama_host` | string | `http://localhost:11434` | Base URL for the Ollama API server. | -| `vllm_host` | string | `http://localhost:8000` | Base URL for the vLLM OpenAI-compatible server. | -| `llamacpp_host` | string | `http://localhost:8080` | Base URL for the llama.cpp HTTP server (`llama-server`). | -| `llamacpp_path` | string | `""` | Path to the llama.cpp binary, if not on `$PATH`. | -| `sglang_host` | string | `http://localhost:30000` | Base URL for the SGLang server. | + +**`[engine.ollama]`:** + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `host` | string | `http://localhost:11434` | Base URL for the Ollama API server. | + +**`[engine.vllm]`:** + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `host` | string | `http://localhost:8000` | Base URL for the vLLM OpenAI-compatible server. | + +**`[engine.sglang]`:** + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `host` | string | `http://localhost:30000` | Base URL for the SGLang server. | + +**`[engine.llamacpp]`:** + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `host` | string | `http://localhost:8080` | Base URL for the llama.cpp HTTP server (`llama-server`). | +| `binary_path` | string | `""` | Path to the llama.cpp binary, if not on `$PATH`. | !!! tip "Engine fallback" If the configured default engine is unreachable, OpenJarvis automatically probes all registered engines and falls back to any healthy one. +!!! note "Backward compatibility" + The old flat field names (`ollama_host`, `vllm_host`, `llamacpp_host`, `llamacpp_path`, `sglang_host`) are still accepted as backward-compatible properties. New configurations should use the nested sub-section format. + --- -### `[intelligence]` -- Model Routing +### `[intelligence]` — Model Identity and Generation Defaults -Controls which model is selected by default and which model to fall back to. +Controls which model is used, its weight paths, quantization, and the default sampling parameters for generation. Generation parameters such as `temperature` and `max_tokens` now live here rather than under `[agent]`. ```toml [intelligence] default_model = "" fallback_model = "" +# model_path = "" +# checkpoint_path = "" +# quantization = "none" +# preferred_engine = "" +# provider = "" +temperature = 0.7 +max_tokens = 1024 +# top_p = 0.9 +# top_k = 40 +# repetition_penalty = 1.0 +# stop_sequences = "" ``` +**Model identity fields:** + | Field | Type | Default | Description | |-------|------|---------|-------------| -| `default_model` | string | `""` (empty) | Preferred model identifier (e.g., `qwen3:8b`). When empty, the router policy selects the model dynamically. | -| `fallback_model` | string | `""` (empty) | Model to use if the default is unavailable. | +| `default_model` | string | `""` | Preferred model identifier (e.g., `qwen3:8b`). When empty, the router policy selects the model dynamically. | +| `fallback_model` | string | `""` | Model to use if the default is unavailable. | +| `model_path` | string | `""` | Path or HuggingFace repo ID for local weights (e.g., `"./models/qwen3-8b.gguf"` or `"Qwen/Qwen3-8B"`). | +| `checkpoint_path` | string | `""` | Path to a fine-tuned checkpoint or LoRA adapter directory. | +| `quantization` | string | `"none"` | Quantization format. Accepted values: `none`, `fp8`, `int8`, `int4`, `gguf_q4`, `gguf_q8`. | +| `preferred_engine` | string | `""` | Override engine for this model (e.g., `"vllm"`). Takes priority over `engine.default`. | +| `provider` | string | `""` | Model provider hint: `local`, `openai`, `anthropic`, `google`. Used by the Cloud engine to route API calls. | -When both fields are empty, OpenJarvis uses the configured router policy (see `[learning]`) to select a model from those available on the active engine. +**Generation default fields** (overridable per-call): + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `temperature` | float | `0.7` | Sampling temperature. Lower values produce more deterministic output. | +| `max_tokens` | int | `1024` | Maximum number of tokens to generate per call. | +| `top_p` | float | `0.9` | Nucleus sampling probability mass. | +| `top_k` | int | `40` | Top-k sampling: only consider the top-k tokens at each step. | +| `repetition_penalty` | float | `1.0` | Penalize repeated tokens. Values > 1 reduce repetition. | +| `stop_sequences` | string | `""` | Comma-separated stop strings. Generation halts when any stop string is produced. | + +When both `default_model` and `fallback_model` are empty, OpenJarvis uses the configured router policy (see `[learning]`) to select a model from those available on the active engine. + +### Engine Selection Priority + +When resolving which engine to use for a model, `SystemBuilder`, `sdk.py`, and `cli/ask.py` check fields in this order: + +``` +1. Explicit --engine CLI flag or engine_key= SDK parameter +2. config.intelligence.preferred_engine +3. config.engine.default +4. First healthy engine discovered at runtime +``` + +This lets you pin a specific model to a specific engine without changing the global engine default: + +```toml +[engine] +default = "ollama" + +[intelligence] +default_model = "llama3.2:3b" +model_path = "./models/llama-3.2-3b.Q4_K_M.gguf" +quantization = "gguf_q4" +preferred_engine = "llamacpp" +``` --- -### `[learning]` -- Learning Policies +### `[agent]` — Agent Behavior -Controls how the learning system selects models, advises agents, and tunes tool selection. +Controls the default agent, turn limits, tool selection, system prompt, and memory context injection. ```toml -[learning] -default_policy = "heuristic" -reward_weights = "" -intelligence_policy = "heuristic" -agent_policy = "" -tools_policy = "" -update_interval = 100 +[agent] +default_agent = "simple" +max_turns = 10 +# tools = "" +# objective = "" +# system_prompt = "" +# system_prompt_path = "" +context_from_memory = true ``` | Field | Type | Default | Description | |-------|------|---------|-------------| -| `default_policy` | string | `"heuristic"` | Router policy to use for model selection. Available: `heuristic`, `learned` (trace-driven), `sft` (supervised fine-tuning), `grpo` (RL stub). | -| `reward_weights` | string | `""` | Comma-separated key=value pairs for the reward function. Example: `"latency=0.4,cost=0.3,quality=0.3"`. | -| `intelligence_policy` | string | `"heuristic"` | Intelligence learning policy (model routing). One of: `heuristic`, `learned`, `sft`, `grpo`. | -| `agent_policy` | string | `""` | Agent learning policy (agent behavior advice). Available: `agent_advisor`. Empty means no agent learning. | -| `tools_policy` | string | `""` | Tool learning policy (tool selection). Available: `icl_updater`. Empty means no tool learning. | -| `update_interval` | int | `100` | Number of traces between automatic policy updates. | +| `default_agent` | string | `"simple"` | Default agent to use. Available: `simple`, `orchestrator`, `react`, `custom`, `openclaw`. | +| `max_turns` | int | `10` | Maximum number of tool-calling turns for the orchestrator agent before it must produce a final answer. | +| `tools` | string | `""` | Comma-separated list of tools to enable by default (e.g., `"calculator,think"`). | +| `objective` | string | `""` | Concise purpose string for routing, learning, and documentation. | +| `system_prompt` | string | `""` | Inline system prompt. Takes precedence over `system_prompt_path` when set. | +| `system_prompt_path` | string | `""` | Path to a system prompt file (`.txt` or `.md`). | +| `context_from_memory` | bool | `true` | Whether to automatically inject relevant memory context into queries. | -**Router / Intelligence policies:** +!!! note "Generation parameters moved" + `temperature` and `max_tokens` have moved from `[agent]` to `[intelligence]`. Old configs with these fields under `[agent]` are automatically migrated to `[intelligence]` at load time. + +!!! note "Backward compatibility" + The old field name `default_tools` is still accepted as a backward-compatible property for `tools`. New configurations should use `tools`. + +!!! info "Context injection" + When `context_from_memory = true` and documents have been indexed, every query automatically searches memory for relevant chunks and prepends them as system context. This gives the model access to your indexed knowledge base without any extra steps. Disable with `--no-context` on the CLI or `context=False` in the SDK. + +--- + +### `[learning]` — Learning Policies + +Controls whether the learning system is enabled and configures per-pillar policies through nested sub-sections. + +```toml +[learning] +enabled = false +update_interval = 100 +# auto_update = false + +[learning.routing] +policy = "heuristic" +# min_samples = 5 + +# [learning.intelligence] +# policy = "none" + +# [learning.agent] +# policy = "none" + +# [learning.metrics] +# accuracy_weight = 0.6 +# latency_weight = 0.2 +# cost_weight = 0.1 +# efficiency_weight = 0.1 +``` + +**`[learning]` top-level:** + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `enabled` | bool | `false` | Whether the learning system is active. | +| `update_interval` | int | `100` | Number of traces between automatic policy updates. | +| `auto_update` | bool | `false` | Whether to trigger policy updates automatically when the interval is reached. | + +**`[learning.routing]` — Router policy:** + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `policy` | string | `"heuristic"` | Router policy for model selection. Available: `heuristic`, `learned` (trace-driven), `sft` (supervised fine-tuning), `grpo` (RL stub). | +| `min_samples` | int | `5` | Minimum number of traces required before trusting a learned routing decision. | + +**`[learning.intelligence]` — Intelligence learning policy:** + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `policy` | string | `"none"` | Intelligence learning policy. Available: `none`, `sft`. Use `sft` to learn model routing from accumulated traces. | + +**`[learning.agent]` — Agent learning policy:** + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `policy` | string | `"none"` | Agent learning policy. Available: `none`, `agent_advisor`, `icl_updater`. | +| `max_icl_examples` | int | `20` | Maximum number of in-context examples to maintain in the ICL example library. | +| `advisor_confidence_threshold` | float | `0.7` | Minimum confidence score for the advisor to recommend a strategy change. | + +**`[learning.metrics]` — Reward / optimization metric weights:** + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `accuracy_weight` | float | `0.6` | Weight for outcome accuracy in the composite reward score. | +| `latency_weight` | float | `0.2` | Weight for inference latency in the composite reward score. | +| `cost_weight` | float | `0.1` | Weight for per-call cost in the composite reward score. | +| `efficiency_weight` | float | `0.1` | Weight for token efficiency in the composite reward score. | + +**Router policies:** | Policy | Description | |--------|-------------| @@ -132,12 +296,7 @@ update_interval = 100 | Policy | Description | |--------|-------------| | `agent_advisor` | Advises on agent strategy (tool sets, turn limits) based on trace patterns. | - -**Tool policies:** - -| Policy | Description | -|--------|-------------| -| `icl_updater` | In-context learning updater for tool selection and configuration. | +| `icl_updater` | In-context learning updater — discovers reusable ICL examples and multi-tool skill sequences from traces. | You can also override the router policy per-query via the CLI: @@ -145,17 +304,19 @@ You can also override the router policy per-query via the CLI: jarvis ask --router heuristic "Hello" ``` +!!! note "Backward compatibility" + The old flat field names `default_policy`, `intelligence_policy`, `agent_policy`, and the comma-separated `reward_weights` string are still accepted as backward-compatible properties. New configurations should use the nested sub-section format. The `tools_policy` field has been removed; use `learning.agent.policy = "icl_updater"` instead. + --- -### `[memory]` -- Memory Backend +### `[tools.storage]` — Storage Backend -Controls the persistent memory system: which backend to use, how documents are chunked, and how context injection works. +Controls the storage backend used for document memory and context injection. The `context_injection` field has moved to `agent.context_from_memory`. ```toml -[memory] +[tools.storage] default_backend = "sqlite" db_path = "~/.openjarvis/memory.db" -context_injection = true context_top_k = 5 context_min_score = 0.1 context_max_tokens = 2048 @@ -165,9 +326,8 @@ chunk_overlap = 64 | Field | Type | Default | Description | |-------|------|---------|-------------| -| `default_backend` | string | `"sqlite"` | Memory backend to use. Available: `sqlite` (FTS5), `faiss`, `colbert`, `bm25`, `hybrid`. | +| `default_backend` | string | `"sqlite"` | Storage backend. Available: `sqlite` (FTS5), `faiss`, `colbert`, `bm25`, `hybrid`. | | `db_path` | string | `~/.openjarvis/memory.db` | Path to the SQLite memory database. Used by the `sqlite` backend. | -| `context_injection` | bool | `true` | Whether to automatically inject relevant memory context into queries. | | `context_top_k` | int | `5` | Number of top memory results to inject as context. | | `context_min_score` | float | `0.1` | Minimum relevance score for a memory result to be included in context. | | `context_max_tokens` | int | `2048` | Maximum number of tokens to use for injected context. | @@ -184,35 +344,29 @@ chunk_overlap = 64 | `bm25` | `memory-bm25` | BM25 sparse retrieval via `rank-bm25`. | | `hybrid` | Depends on sub-backends | Reciprocal Rank Fusion combining multiple backends. | -!!! note "Context injection" - When `context_injection` is enabled and documents have been indexed, every query automatically searches memory for relevant chunks and prepends them as system context. This gives the model access to your indexed knowledge base without any extra steps. Disable with `--no-context` on the CLI or `context=False` in the SDK. +!!! note "Backward compatibility" + The `[memory]` TOML section is still supported and maps to `[tools.storage]`. New configurations should use `[tools.storage]`. The `context_injection` field under `[memory]` or `[tools.storage]` is automatically migrated to `agent.context_from_memory` at load time. --- -### `[agent]` -- Agent Defaults +### `[tools.mcp]` — MCP (Model Context Protocol) -Controls the default agent behavior for queries. +Controls the MCP server and external MCP tool provider integration. The MCP adapter supports protocol version 2025-11-25. ```toml -[agent] -default_agent = "simple" -max_turns = 10 -default_tools = "" -temperature = 0.7 -max_tokens = 1024 +[tools.mcp] +enabled = true +# servers = "" # JSON list of external MCP server configs ``` | Field | Type | Default | Description | |-------|------|---------|-------------| -| `default_agent` | string | `"simple"` | Default agent to use. Available: `simple`, `orchestrator`, `custom`, `openclaw`. | -| `max_turns` | int | `10` | Maximum number of tool-calling turns for the orchestrator agent before it must produce a final answer. | -| `default_tools` | string | `""` | Comma-separated list of tools to enable by default (e.g., `"calculator,think"`). | -| `temperature` | float | `0.7` | Default sampling temperature for generation. | -| `max_tokens` | int | `1024` | Default maximum tokens for generation. | +| `enabled` | bool | `true` | Whether to enable the MCP adapter for exposing and consuming tools via MCP. | +| `servers` | string | `""` | JSON-encoded list of external MCP server configuration objects. | --- -### `[server]` -- API Server +### `[server]` — API Server Controls the OpenAI-compatible API server started by `jarvis serve`. @@ -229,7 +383,7 @@ workers = 1 |-------|------|---------|-------------| | `host` | string | `"0.0.0.0"` | Bind address for the server. Use `"127.0.0.1"` to restrict to localhost. | | `port` | int | `8000` | Port number for the server. | -| `agent` | string | `"orchestrator"` | Agent to use for non-streaming chat completion requests. | +| `agent` | string | `"orchestrator"` | Agent to use for chat completion requests. | | `model` | string | `""` | Default model for the server. When empty, uses `intelligence.default_model` or the first available model. | | `workers` | int | `1` | Number of uvicorn worker processes. | @@ -241,7 +395,7 @@ jarvis serve --host 127.0.0.1 --port 9000 --model qwen3:8b --agent simple --- -### `[telemetry]` -- Telemetry Persistence +### `[telemetry]` — Telemetry Persistence Controls whether inference telemetry is recorded and where it is stored. @@ -261,87 +415,58 @@ db_path = "~/.openjarvis/telemetry.db" --- -### `[traces]` -- Trace Recording +### `[traces]` — Trace Recording Controls the trace system that records full interaction sequences for the learning system. ```toml [traces] -enabled = true +enabled = false db_path = "~/.openjarvis/traces.db" ``` | Field | Type | Default | Description | |-------|------|---------|-------------| -| `enabled` | bool | `true` | Whether to record traces for each agent interaction. | +| `enabled` | bool | `false` | Whether to record traces for each agent interaction. | | `db_path` | string | `~/.openjarvis/traces.db` | Path to the SQLite trace database. | --- -### `[tools.storage]` -- Storage Backend Configuration +### `[channel]` — Channel Messaging -Controls the storage backend used by memory/storage tools. This section configures the same backends previously managed under `[memory]`, but with the canonical tools-based naming. - -```toml -[tools.storage] -default_backend = "sqlite" -db_path = "~/.openjarvis/memory.db" -``` - -| Field | Type | Default | Description | -|-------|------|---------|-------------| -| `default_backend` | string | `"sqlite"` | Storage backend to use. Available: `sqlite`, `faiss`, `colbert`, `bm25`, `hybrid`. | -| `db_path` | string | `~/.openjarvis/memory.db` | Path to the SQLite storage database. | - -!!! note "Backward compatibility" - The `[memory]` TOML section is still supported and takes effect if `[tools.storage]` is not present. New configurations should prefer `[tools.storage]`. - ---- - -### `[tools.mcp]` -- MCP (Model Context Protocol) - -Controls the MCP server and external MCP tool provider integration. The MCP adapter supports protocol version 2025-11-25. - -```toml -[tools.mcp] -enabled = false -protocol_version = "2025-11-25" -providers = [] -``` - -| Field | Type | Default | Description | -|-------|------|---------|-------------| -| `enabled` | bool | `false` | Whether to enable the MCP adapter for exposing and consuming tools via MCP. | -| `protocol_version` | string | `"2025-11-25"` | MCP protocol version to use. | -| `providers` | list | `[]` | List of external MCP tool provider URLs to connect to. | - ---- - -### `[channel]` -- Channel Messaging - -Controls the channel messaging bridge for multi-platform communication. +Controls the channel messaging bridge for multi-platform communication. Each supported platform has its own nested sub-section. ```toml [channel] enabled = false -gateway_url = "ws://127.0.0.1:18789/ws" +default_channel = "" default_agent = "simple" -reconnect_interval = 5.0 + +# [channel.telegram] +# bot_token = "" + +# [channel.discord] +# bot_token = "" + +# [channel.slack] +# bot_token = "" +# app_token = "" + +# [channel.webhook] +# url = "" +# secret = "" +# method = "POST" ``` | Field | Type | Default | Description | |-------|------|---------|-------------| | `enabled` | bool | `false` | Whether to enable channel messaging support. | -| `gateway_url` | string | `ws://127.0.0.1:18789/ws` | WebSocket URL of the OpenClaw gateway. | +| `default_channel` | string | `""` | Default channel to use when not specified. | | `default_agent` | string | `"simple"` | Default agent for handling channel messages. | -| `reconnect_interval` | float | `5.0` | Seconds to wait before reconnecting after a disconnect. | - -!!! note "Requires OpenClaw gateway" - Channel messaging requires a running OpenClaw gateway. The bridge connects via WebSocket with HTTP fallback. --- -### `[security]` -- Security Guardrails +### `[security]` — Security Guardrails Controls the security scanning pipeline for input/output content. @@ -379,19 +504,19 @@ When you run `jarvis init`, OpenJarvis probes your system to detect available ha ### GPU Detection -1. **NVIDIA GPU** -- Checks for `nvidia-smi` on `$PATH`. If found, queries GPU name, VRAM (in MB), and GPU count via: +1. **NVIDIA GPU** — Checks for `nvidia-smi` on `$PATH`. If found, queries GPU name, VRAM (in MB), and GPU count via: ``` nvidia-smi --query-gpu=name,memory.total,count --format=csv,noheader,nounits ``` -2. **AMD GPU** -- Checks for `rocm-smi` on `$PATH`. If found, queries the product name via: +2. **AMD GPU** — Checks for `rocm-smi` on `$PATH`. If found, queries the product name via: ``` rocm-smi --showproductname ``` -3. **Apple Silicon** -- On macOS only. Runs `system_profiler SPDisplaysDataType` and looks for "Apple" in the chipset model line. +3. **Apple Silicon** — On macOS only. Runs `system_profiler SPDisplaysDataType` and looks for "Apple" in the chipset model line. If none of these detect a GPU, the system is treated as CPU-only. @@ -461,20 +586,23 @@ graph TD [engine] default = "ollama" -ollama_host = "http://localhost:11434" + +[engine.ollama] +host = "http://localhost:11434" [intelligence] default_model = "qwen3:8b" fallback_model = "llama3.2:3b" - -[memory] -default_backend = "sqlite" -context_injection = true -context_top_k = 5 +temperature = 0.7 +max_tokens = 1024 [agent] default_agent = "simple" max_turns = 10 +context_from_memory = true + +[tools.storage] +default_backend = "sqlite" [server] host = "127.0.0.1" @@ -482,7 +610,10 @@ port = 8000 agent = "orchestrator" [learning] -default_policy = "heuristic" +enabled = false + +[learning.routing] +policy = "heuristic" [telemetry] enabled = true @@ -496,29 +627,33 @@ enabled = true [engine] default = "vllm" -vllm_host = "http://localhost:8000" -ollama_host = "http://localhost:11434" + +[engine.vllm] +host = "http://localhost:8000" + +[engine.ollama] +host = "http://localhost:11434" [intelligence] default_model = "Qwen/Qwen2.5-72B-Instruct" fallback_model = "Qwen/Qwen2.5-7B-Instruct" +temperature = 0.5 +max_tokens = 4096 -[memory] +[agent] +default_agent = "orchestrator" +max_turns = 15 +tools = "calculator,think,retrieval" +context_from_memory = true + +[tools.storage] default_backend = "faiss" -context_injection = true context_top_k = 10 context_min_score = 0.05 context_max_tokens = 4096 chunk_size = 1024 chunk_overlap = 128 -[agent] -default_agent = "orchestrator" -max_turns = 15 -default_tools = "calculator,think,retrieval" -temperature = 0.5 -max_tokens = 4096 - [server] host = "0.0.0.0" port = 8000 @@ -527,7 +662,10 @@ model = "Qwen/Qwen2.5-72B-Instruct" workers = 1 [learning] -default_policy = "heuristic" +enabled = false + +[learning.routing] +policy = "heuristic" [telemetry] enabled = true @@ -541,88 +679,91 @@ enabled = true [engine] default = "llamacpp" -llamacpp_host = "http://localhost:8080" + +[engine.llamacpp] +host = "http://localhost:8080" [intelligence] default_model = "" fallback_model = "" - -[memory] -default_backend = "sqlite" -context_injection = true -context_top_k = 3 -context_max_tokens = 1024 -chunk_size = 256 -chunk_overlap = 32 +temperature = 0.7 +max_tokens = 512 [agent] default_agent = "simple" max_turns = 5 -temperature = 0.7 -max_tokens = 512 +context_from_memory = true + +[tools.storage] +default_backend = "sqlite" +context_top_k = 3 +context_max_tokens = 1024 +chunk_size = 256 +chunk_overlap = 32 [server] host = "127.0.0.1" port = 8000 [learning] -default_policy = "heuristic" +enabled = false + +[learning.routing] +policy = "heuristic" [telemetry] enabled = true ``` -### Cloud-Only (No Local Engine) +### Trace-Driven Learning Enabled ```toml # ~/.openjarvis/config.toml -# Using cloud APIs only (OpenAI, Anthropic) -# Set OPENAI_API_KEY and/or ANTHROPIC_API_KEY environment variables - -[engine] -default = "cloud" - -[intelligence] -default_model = "gpt-4o" -fallback_model = "claude-sonnet-4-20250514" - -[memory] -default_backend = "sqlite" -context_injection = true - -[agent] -default_agent = "orchestrator" -default_tools = "calculator,think" - -[telemetry] -enabled = true -``` - -### Hybrid (Local + Cloud Fallback) - -```toml -# ~/.openjarvis/config.toml -# Local Ollama as primary, cloud as fallback +# Research setup with trace-driven learning active [engine] default = "ollama" -ollama_host = "http://localhost:11434" + +[engine.ollama] +host = "http://localhost:11434" [intelligence] default_model = "qwen3:8b" -fallback_model = "gpt-4o-mini" - -[memory] -default_backend = "sqlite" -context_injection = true +temperature = 0.7 +max_tokens = 1024 [agent] default_agent = "orchestrator" max_turns = 10 -default_tools = "calculator,think" +context_from_memory = true + +[tools.storage] +default_backend = "sqlite" [learning] -default_policy = "heuristic" +enabled = true +update_interval = 50 +auto_update = true + +[learning.routing] +policy = "learned" +min_samples = 10 + +[learning.intelligence] +policy = "sft" + +[learning.agent] +policy = "agent_advisor" +advisor_confidence_threshold = 0.8 + +[learning.metrics] +accuracy_weight = 0.6 +latency_weight = 0.2 +cost_weight = 0.1 +efficiency_weight = 0.1 + +[traces] +enabled = true [telemetry] enabled = true @@ -630,9 +771,148 @@ enabled = true --- +## Migration Guide + +If you have an existing `~/.openjarvis/config.toml` from a previous version, here is what changed and how to update it. + +### Engine: Nested Sub-Sections + +=== "Old Format" + + ```toml + [engine] + default = "ollama" + ollama_host = "http://localhost:11434" + vllm_host = "http://localhost:8000" + llamacpp_path = "/usr/local/bin/llama-server" + ``` + +=== "New Format" + + ```toml + [engine] + default = "ollama" + + [engine.ollama] + host = "http://localhost:11434" + + [engine.vllm] + host = "http://localhost:8000" + + [engine.llamacpp] + binary_path = "/usr/local/bin/llama-server" + ``` + +!!! note + The old flat names still work as backward-compatible properties. You only need to update your config if you want to use the new fields (e.g., `binary_path`). + +### Intelligence: Generation Parameters + +=== "Old Format" + + ```toml + [agent] + temperature = 0.7 + max_tokens = 1024 + ``` + +=== "New Format" + + ```toml + [intelligence] + temperature = 0.7 + max_tokens = 1024 + ``` + +!!! note + Old configs with `temperature` or `max_tokens` under `[agent]` are automatically migrated to `[intelligence]` at load time. No manual update is required, but updating is recommended for clarity. + +### Agent: Renamed and Added Fields + +=== "Old Format" + + ```toml + [agent] + default_tools = "calculator,think" + ``` + +=== "New Format" + + ```toml + [agent] + tools = "calculator,think" + ``` + +The `default_tools` name still works via a backward-compatible property. + +### Memory: Context Injection Moved + +=== "Old Format" + + ```toml + [memory] + context_injection = true + default_backend = "sqlite" + ``` + +=== "New Format" + + ```toml + [agent] + context_from_memory = true + + [tools.storage] + default_backend = "sqlite" + ``` + +!!! note + `context_injection` under `[memory]` or `[tools.storage]` is automatically migrated to `agent.context_from_memory` at load time. + +### Learning: Nested Sub-Sections + +=== "Old Format" + + ```toml + [learning] + default_policy = "heuristic" + intelligence_policy = "sft" + agent_policy = "agent_advisor" + tools_policy = "icl_updater" + reward_weights = "accuracy=0.6,latency=0.2,cost=0.1,efficiency=0.1" + update_interval = 100 + ``` + +=== "New Format" + + ```toml + [learning] + enabled = true + update_interval = 100 + + [learning.routing] + policy = "heuristic" + + [learning.intelligence] + policy = "sft" + + [learning.agent] + policy = "agent_advisor" + + [learning.metrics] + accuracy_weight = 0.6 + latency_weight = 0.2 + cost_weight = 0.1 + efficiency_weight = 0.1 + ``` + +!!! note + The flat field names `default_policy`, `intelligence_policy`, `agent_policy`, and `reward_weights` are still accepted as backward-compatible properties. The `tools_policy` field has been removed; use `learning.agent.policy = "icl_updater"` instead. + +--- + ## Programmatic Configuration -You can also configure OpenJarvis entirely from Python without a TOML file: +You can configure OpenJarvis entirely from Python without a TOML file: ```python from openjarvis import Jarvis @@ -642,29 +922,31 @@ from openjarvis.core.config import ( IntelligenceConfig, JarvisConfig, LearningConfig, - MemoryConfig, + OllamaEngineConfig, StorageConfig, - MCPConfig, ToolsConfig, - TracesConfig, ) config = JarvisConfig( engine=EngineConfig( default="ollama", - ollama_host="http://my-server:11434", + ollama=OllamaEngineConfig(host="http://my-server:11434"), ), intelligence=IntelligenceConfig( default_model="qwen3:8b", - ), - memory=MemoryConfig( - default_backend="sqlite", - context_injection=True, - context_top_k=10, + temperature=0.7, + max_tokens=2048, ), agent=AgentConfig( default_agent="orchestrator", max_turns=15, + context_from_memory=True, + ), + tools=ToolsConfig( + storage=StorageConfig( + default_backend="sqlite", + context_top_k=10, + ), ), ) @@ -694,6 +976,8 @@ OpenJarvis respects the following environment variables: ## Next Steps -- [Quick Start](quickstart.md) -- Run your first query -- [CLI Reference](../user-guide/cli.md) -- Full reference for all CLI commands -- [Architecture Overview](../architecture/overview.md) -- Understand how the pieces fit together +- [Quick Start](quickstart.md) — Run your first query +- [CLI Reference](../user-guide/cli.md) — Full reference for all CLI commands +- [Architecture Overview](../architecture/overview.md) — Understand how the pieces fit together +- [Intelligence Pillar](../architecture/intelligence.md) — Model identity and generation defaults +- [Learning & Traces](../architecture/learning.md) — Router policies and the trace-driven feedback loop diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index 0e4e4e62..10e04898 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -235,7 +235,7 @@ To disable this behavior: response = j.ask("Hello", context=False) ``` -Context injection is controlled by the `[memory]` config section. See [Configuration](configuration.md) for details on `context_injection`, `context_top_k`, `context_min_score`, and `context_max_tokens`. +Context injection is controlled by `agent.context_from_memory` in `config.toml`. The retrieval parameters (`context_top_k`, `context_min_score`, `context_max_tokens`) live under `[tools.storage]`. See [Configuration](configuration.md) for details. ## Model Management diff --git a/docs/user-guide/evaluations.md b/docs/user-guide/evaluations.md new file mode 100644 index 00000000..6f1235dd --- /dev/null +++ b/docs/user-guide/evaluations.md @@ -0,0 +1,543 @@ +# Evaluations + +The OpenJarvis evaluation framework (`openjarvis-evals`) measures model **correctness and accuracy** on academic datasets. It is a separate package from the main OpenJarvis library and is designed specifically for research workflows where you need reproducible, dataset-driven quality assessments. + +!!! info "Evals vs. Benchmarks" + OpenJarvis has two distinct measurement systems that complement each other: + + | System | Package | Measures | Entry Point | + |--------|---------|----------|-------------| + | **Evaluations** | `openjarvis-evals` | Correctness on academic datasets (accuracy, pass rate) | `openjarvis-eval` | + | **Benchmarks** | `openjarvis` | Engine performance (latency, throughput) | `jarvis bench` | + + Use evaluations to answer "does this model get the right answer?" and benchmarks to answer "how fast does this model respond?". See the [Benchmarks guide](benchmarks.md) for the performance measurement system. + +--- + +## Installation + +The evaluation framework is a standalone package in the `evals/` directory. Install it alongside OpenJarvis: + +```bash +pip install -e evals/ +``` + +This installs the `openjarvis-eval` CLI entry point and all required dependencies (`datasets`, `huggingface-hub`, `tqdm`, `rich`). + +!!! note "Python version requirement" + Python 3.10 requires the `tomli` package for TOML config parsing. The `evals/pyproject.toml` includes this as a conditional dependency, so it is installed automatically. + +--- + +## Datasets + +The framework ships with four datasets covering different capability dimensions: + +| Dataset | Key | Category | HuggingFace Path | Default Split | +|---------|-----|----------|------------------|---------------| +| **SuperGPQA** | `supergpqa` | reasoning | `m-a-p/SuperGPQA` | `train` | +| **GAIA** | `gaia` | agentic | `gaia-benchmark/GAIA` | `validation` | +| **FRAMES** | `frames` | rag | `google/frames-benchmark` | `test` | +| **WildChat** | `wildchat` | chat | `allenai/WildChat-1M` | `train` | + +**SuperGPQA** is a large-scale multiple-choice benchmark spanning graduate-level questions across scientific disciplines. Each sample has a question, a set of lettered options, and a reference answer letter. + +**GAIA** is an agentic benchmark requiring models to complete multi-step tasks that may involve file reading, calculations, and web lookup. Questions are drawn from the 2023 GAIA challenge set. + +**FRAMES** tests multi-hop factual retrieval. Each question requires synthesizing information across multiple Wikipedia articles, making it a strong probe of retrieval-augmented generation capability. + +**WildChat** uses real user conversations filtered to English single-turn exchanges. The reference answer is the original assistant response from the dataset; the model under evaluation is compared against it by an LLM judge. + +!!! tip "GAIA dataset access" + The GAIA dataset requires a HuggingFace account and acceptance of the dataset's terms of use. The loader downloads the full dataset snapshot on first use and caches it at `~/.cache/gaia_benchmark/`. Subsequent runs use the local cache. + +--- + +## Inference Backends + +Every evaluation run routes model calls through one of two backends: + +| Backend | Key | Description | +|---------|-----|-------------| +| **jarvis-direct** | `jarvis-direct` | Engine-level inference via `SystemBuilder`. Works for local (Ollama, vLLM, llama.cpp) and cloud models. | +| **jarvis-agent** | `jarvis-agent` | Agent-level inference with tool calling. Uses `JarvisSystem.ask()` with the specified agent and tools. | + +Use `jarvis-direct` for most evaluations. Use `jarvis-agent` when the benchmark requires tool use — for example, GAIA tasks that reference files that must be read with `file_read`, or arithmetic tasks that benefit from `calculator`. + +--- + +## CLI Usage + +### List available benchmarks and backends + +```bash +openjarvis-eval list +``` + +Output: + +``` +Benchmarks: + supergpqa [reasoning ] SuperGPQA multiple-choice + gaia [agentic ] GAIA agentic benchmark + frames [rag ] FRAMES multi-hop RAG + wildchat [chat ] WildChat conversation quality + +Backends: + jarvis-direct Engine-level inference (local or cloud) + jarvis-agent Agent-level inference with tool calling +``` + +### Run a single benchmark + +```bash +# Evaluate qwen3:8b on SuperGPQA (engine-level, 10 samples default) +openjarvis-eval run -b supergpqa -m qwen3:8b + +# Evaluate GPT-4o on GAIA using the agent backend with tools +openjarvis-eval run -b gaia -m gpt-4o --backend jarvis-agent \ + --agent orchestrator --tools calculator,file_read -n 50 + +# Run FRAMES with vLLM engine, write output to a file +openjarvis-eval run -b frames -m llama3:70b -e vllm \ + -o results/frames_llama70b.jsonl + +# Run WildChat with a higher temperature for chat quality +openjarvis-eval run -b wildchat -m qwen3:8b --temperature 0.7 -n 100 +``` + +#### Full option reference + +| Option | Short | Type | Default | Description | +|--------|-------|------|---------|-------------| +| `--config` | `-c` | path | — | TOML config file; when provided, `-b` and `-m` are not required | +| `--benchmark` | `-b` | choice | required* | `supergpqa`, `gaia`, `frames`, or `wildchat` | +| `--backend` | | choice | `jarvis-direct` | `jarvis-direct` or `jarvis-agent` | +| `--model` | `-m` | str | required* | Model identifier (e.g., `qwen3:8b`, `gpt-4o`) | +| `--engine` | `-e` | str | auto | Engine key (`ollama`, `vllm`, `cloud`, ...) | +| `--agent` | | str | `orchestrator` | Agent name for `jarvis-agent` backend | +| `--tools` | | str | `""` | Comma-separated tool names (e.g., `calculator,file_read`) | +| `--max-samples` | `-n` | int | all | Limit the number of samples evaluated | +| `--max-workers` | `-w` | int | `4` | Parallel evaluation workers | +| `--judge-model` | | str | `gpt-4o` | LLM used for judge-based scoring | +| `--output` | `-o` | path | auto-generated | Output JSONL file path | +| `--seed` | | int | `42` | Random seed for dataset shuffling | +| `--split` | | str | dataset default | Override the dataset split | +| `--temperature` | | float | `0.0` | Generation temperature | +| `--max-tokens` | | int | `2048` | Maximum output tokens | +| `--verbose` | `-v` | flag | off | Enable debug logging | + +*Required when `--config` is not provided. + +### Run all benchmarks at once + +The `run-all` command evaluates a single model against all four benchmarks sequentially and writes results to an output directory: + +```bash +openjarvis-eval run-all -m qwen3:8b + +# With options +openjarvis-eval run-all -m gpt-4o -n 100 --output-dir results/gpt4o/ +``` + +Output files are written as `{output_dir}/{benchmark}_{model-slug}.jsonl`. The model slug replaces `/` and `:` with `-`, so `qwen3:8b` becomes `qwen3-8b`. + +### Summarize results + +After a run, inspect a JSONL results file: + +```bash +openjarvis-eval summarize results/supergpqa_qwen3-8b.jsonl +``` + +Output: + +``` +File: results/supergpqa_qwen3-8b.jsonl +Benchmark: supergpqa +Model: qwen3:8b +Total: 200 +Scored: 198 +Correct: 143 +Accuracy: 0.7222 +Errors: 2 +``` + +--- + +## TOML Config System + +For research workflows that compare multiple models across multiple benchmarks, use a TOML config file to define the evaluation as a **models x benchmarks matrix**. This is the recommended approach for systematic evaluations. + +### Running from a config + +```bash +openjarvis-eval run --config evals/configs/full-suite.toml +``` + +When `--config` is provided, the `-b`/`--benchmark` and `-m`/`--model` options are not required. All settings come from the config file. The CLI expands the matrix, prints a progress table, and writes results to the configured `output_dir`. + +### Config file format + +A config file has six sections: `[meta]`, `[defaults]`, `[judge]`, `[run]`, `[[models]]`, and `[[benchmarks]]`. Only `[[models]]` and `[[benchmarks]]` are required — all other sections are optional and fall back to built-in defaults. + +```toml title="evals/configs/full-suite.toml" +# Suite-level metadata (optional) +[meta] +name = "full-suite-v1" +description = "Evaluate all benchmarks against production models" + +# Default generation parameters (optional) +[defaults] +temperature = 0.0 +max_tokens = 2048 + +# LLM judge configuration (optional) +[judge] +model = "gpt-4o" +temperature = 0.0 +max_tokens = 1024 + +# Execution settings (optional) +[run] +max_workers = 4 +output_dir = "results/" +seed = 42 + +# --- Models (one [[models]] block per model) --- + +[[models]] +name = "qwen3:8b" +engine = "ollama" +temperature = 0.3 # overrides [defaults] for this model +max_tokens = 4096 + +[[models]] +name = "gpt-4o" +provider = "openai" # uses cloud engine + +[[models]] +name = "llama3:70b" +engine = "vllm" +temperature = 0.1 + +# --- Benchmarks (one [[benchmarks]] block per benchmark) --- + +[[benchmarks]] +name = "supergpqa" +backend = "jarvis-direct" +max_samples = 200 +split = "train" + +[[benchmarks]] +name = "gaia" +backend = "jarvis-agent" +agent = "orchestrator" +tools = ["file_read", "calculator"] +max_samples = 50 +judge_model = "claude-sonnet-4-20250514" # override judge for this benchmark + +[[benchmarks]] +name = "frames" +backend = "jarvis-direct" +max_samples = 100 + +[[benchmarks]] +name = "wildchat" +backend = "jarvis-direct" +max_samples = 150 +temperature = 0.7 # override temperature for this benchmark +``` + +This config produces 3 models x 4 benchmarks = **12 evaluation runs**. + +### Merge precedence + +Settings are resolved with the following precedence, from highest to lowest: + +``` +benchmark-level > model-level > [defaults] > built-in defaults +``` + +For example, `temperature` is resolved as: use `[defaults].temperature` (0.0), then apply `[[models]].temperature` if set (0.3 for qwen3:8b), then override with `[[benchmarks]].temperature` if set (0.7 for wildchat). The WildChat run with qwen3:8b therefore runs at `temperature = 0.7`. + +### Minimal config + +A config requires only one `[[models]]` and one `[[benchmarks]]` entry: + +```toml title="evals/configs/minimal.toml" +[[models]] +name = "qwen3:8b" + +[[benchmarks]] +name = "supergpqa" +``` + +This runs SuperGPQA against qwen3:8b with all default settings. Use this as a starting point when iterating on a single model or dataset. + +### Single-run config with full options + +```toml title="evals/configs/single-run.toml" +[meta] +name = "single-run-example" +description = "Evaluate SuperGPQA with a single model and full configuration" + +[defaults] +temperature = 0.0 +max_tokens = 2048 + +[judge] +model = "gpt-4o" +temperature = 0.0 +max_tokens = 1024 + +[run] +max_workers = 4 +output_dir = "results/" +seed = 42 + +[[models]] +name = "qwen3:8b" +engine = "ollama" +temperature = 0.3 +max_tokens = 4096 + +[[benchmarks]] +name = "supergpqa" +backend = "jarvis-direct" +max_samples = 100 +split = "train" +``` + +--- + +## Config Reference + +### `[meta]` + +Suite-level metadata. Neither field affects evaluation behavior; both are used in CLI output and summary files. + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `name` | str | `""` | Suite name shown in CLI output | +| `description` | str | `""` | Human-readable description | + +### `[defaults]` + +Default generation parameters applied to every run unless overridden at the model or benchmark level. + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `temperature` | float | `0.0` | Sampling temperature | +| `max_tokens` | int | `2048` | Maximum output tokens | + +### `[judge]` + +Configuration for the LLM used as a judge in GAIA, FRAMES, and WildChat scoring. + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `model` | str | `"gpt-4o"` | Judge model identifier | +| `provider` | str | `None` | Provider override (e.g., `"openai"`) | +| `temperature` | float | `0.0` | Judge sampling temperature | +| `max_tokens` | int | `1024` | Maximum judge output tokens | + +!!! warning "Judge model costs" + Every sample that requires LLM-based scoring makes a separate call to the judge model. For large runs with hundreds of samples, judge costs can exceed evaluation costs. GAIA, FRAMES, and WildChat all require a judge; SuperGPQA uses an LLM to extract the answer letter, then compares it against the reference without a separate judge call. + +### `[run]` + +Execution settings that apply to the entire suite. + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `max_workers` | int | `4` | Number of parallel evaluation threads | +| `output_dir` | str | `"results/"` | Directory where JSONL and summary files are written | +| `seed` | int | `42` | Random seed for dataset shuffling | + +### `[[models]]` + +One block per model. The `name` field is required. + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `name` | str | required | Model identifier (e.g., `"qwen3:8b"`, `"gpt-4o"`) | +| `engine` | str | `None` | Engine key to use (`"ollama"`, `"vllm"`, `"cloud"`, ...) | +| `provider` | str | `None` | Provider override for cloud models (e.g., `"openai"`) | +| `temperature` | float | `None` | Override `[defaults].temperature` for this model | +| `max_tokens` | int | `None` | Override `[defaults].max_tokens` for this model | + +### `[[benchmarks]]` + +One block per benchmark. The `name` field is required. + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `name` | str | required | Benchmark key: `supergpqa`, `gaia`, `frames`, or `wildchat` | +| `backend` | str | `"jarvis-direct"` | Inference backend: `jarvis-direct` or `jarvis-agent` | +| `max_samples` | int | `None` | Limit number of samples; `None` evaluates the full dataset | +| `split` | str | `None` | Override the default dataset split | +| `agent` | str | `None` | Agent name for `jarvis-agent` backend (e.g., `"orchestrator"`) | +| `tools` | list[str] | `[]` | Tool names for `jarvis-agent` backend | +| `judge_model` | str | `None` | Override `[judge].model` for this benchmark only | +| `temperature` | float | `None` | Override temperature for this benchmark (highest precedence) | +| `max_tokens` | int | `None` | Override max tokens for this benchmark (highest precedence) | + +--- + +## Output Format + +### JSONL results file + +Each completed sample is appended to the output JSONL file immediately after scoring. The file path is either specified with `-o`/`--output`, or auto-generated as `{output_dir}/{benchmark}_{model-slug}.jsonl`. + +Each line is a JSON object with the following fields: + +```json title="results/supergpqa_qwen3-8b.jsonl (one line per sample)" +{ + "record_id": "supergpqa-42", + "benchmark": "supergpqa", + "model": "qwen3:8b", + "backend": "jarvis-direct", + "model_answer": "The answer is C because...", + "is_correct": true, + "score": 1.0, + "latency_seconds": 1.34, + "prompt_tokens": 187, + "completion_tokens": 12, + "cost_usd": 0.0, + "error": null, + "scoring_metadata": { + "reference_letter": "C", + "candidate_letter": "C", + "valid_letters": "ABCD" + } +} +``` + +| Field | Type | Description | +|-------|------|-------------| +| `record_id` | str | Unique sample identifier | +| `benchmark` | str | Benchmark name | +| `model` | str | Model identifier | +| `backend` | str | Backend used | +| `model_answer` | str | Raw model output | +| `is_correct` | bool or null | Scoring result (`null` if unscored) | +| `score` | float or null | Numeric score (1.0 correct, 0.0 incorrect, `null` unscored) | +| `latency_seconds` | float | Inference latency | +| `prompt_tokens` | int | Input tokens consumed | +| `completion_tokens` | int | Output tokens generated | +| `cost_usd` | float | Estimated cost in USD | +| `error` | str or null | Error message if the sample failed | +| `scoring_metadata` | dict | Scorer-specific details (extracted letters, judge output, etc.) | + +### Summary JSON file + +After all samples complete, a summary file is written alongside the JSONL at `{output_path}.summary.json`: + +```json title="results/supergpqa_qwen3-8b.jsonl.summary.json" +{ + "benchmark": "supergpqa", + "category": "reasoning", + "backend": "jarvis-direct", + "model": "qwen3:8b", + "total_samples": 200, + "scored_samples": 198, + "correct": 143, + "accuracy": 0.7222, + "errors": 2, + "mean_latency_seconds": 1.4821, + "total_cost_usd": 0.0, + "per_subject": { + "chemistry": {"accuracy": 0.74, "total": 50.0, "scored": 50.0, "correct": 37.0}, + "mathematics": {"accuracy": 0.68, "total": 50.0, "scored": 49.0, "correct": 33.0} + }, + "started_at": 1708789200.0, + "ended_at": 1708789496.3 +} +``` + +The `per_subject` breakdown groups results by the dataset's subject or category field, which varies per benchmark: + +- **SuperGPQA**: `subfield`, `field`, or `discipline` +- **GAIA**: difficulty level (`level_1`, `level_2`, `level_3`) +- **FRAMES**: reasoning type(s) (e.g., `temporal`, `intersection`) +- **WildChat**: always `"conversation"` + +--- + +## Scoring Methods + +Each benchmark uses a scorer tuned to its answer format. + +### SuperGPQA: LLM-assisted MCQ extraction + +SuperGPQA responses are free-form text that must contain one of the valid option letters (A, B, C, D, ...). The scorer uses the judge LLM to extract the final answer letter from the model's response, then compares it against the reference letter with exact string matching. + +The judge is prompted with the original problem and the model's response and asked to return only a single letter. This handles cases where the model reasons extensively before stating its final answer. + +``` +is_correct = extracted_letter == reference_letter +``` + +Scoring metadata includes: `reference_letter`, `candidate_letter`, and `valid_letters`. + +### GAIA: Normalized exact match with LLM fallback + +GAIA answers are typically numbers, short phrases, or comma-separated lists. The scorer applies a normalization pass before comparison: + +- **Numbers**: strips `$`, `%`, `,` and converts to float for comparison +- **Lists**: splits on `,`/`;` and compares element-by-element (with per-element type detection) +- **Strings**: lowercases, strips whitespace and punctuation + +If the normalized exact match fails, the scorer falls back to the judge LLM, which returns a structured response with `extracted_final_answer`, `reasoning`, and `correct: yes/no`. The LLM fallback handles cases like unit variations, alternative phrasings, and equivalent but differently-formatted answers. + +### FRAMES: LLM-as-judge (factual correctness) + +FRAMES uses an LLM judge that evaluates semantic equivalence between the model's answer and the ground truth. The judge receives the question, ground truth, and predicted answer, then responds with a structured verdict: + +``` +extracted_final_answer: +reasoning: +correct: yes / no +``` + +The scorer parses the `correct:` line and falls back to presence of `TRUE`/`FALSE` tokens if the structured format is missing. + +### WildChat: Pairwise LLM comparison + +WildChat does not have a single "correct" answer — it measures chat response quality. The scorer runs a **dual pairwise comparison**: + +1. The judge evaluates (model answer as A, reference as B) and returns a verdict token such as `[[A>>B]]`, `[[A>B]]`, `[[A=B]]`, `[[B>A]]`, or `[[B>>A]]`. +2. The judge then evaluates (reference as A, model answer as B) and returns another verdict. + +The model is considered to have passed (`is_correct = True`) if it wins or ties in either comparison. The dual comparison reduces positional bias in the judge. + +The judge uses a multi-step rubric that distinguishes subjective queries (scored on correctness, helpfulness, relevance, conciseness, and creativity) from objective/technical queries (scored on correctness only). + +!!! tip "Interpreting WildChat accuracy" + A WildChat accuracy score of 0.50 means the model matched or beat the reference response in half of comparisons. Because the reference response comes from the original dataset (which may include responses from capable models), a score above 0.50 indicates strong chat quality for that sample set. + +--- + +## Parallel Execution + +The `EvalRunner` processes samples concurrently using a `ThreadPoolExecutor`. Results are flushed to the JSONL file incrementally as each sample completes, so you can inspect partial results during a long run. + +```bash +# Use more workers for faster evaluation (if the engine supports concurrent requests) +openjarvis-eval run -b supergpqa -m qwen3:8b -w 8 -n 500 +``` + +!!! warning "Worker count and engine load" + Higher worker counts increase throughput only if the inference engine can handle concurrent requests. Local Ollama instances typically handle 1-2 concurrent requests. Cloud APIs (OpenAI, Anthropic) can handle higher concurrency. Set `-w` based on your engine's actual parallelism. + +--- + +## See Also + +- [Benchmarks](benchmarks.md) — Measure inference engine latency and throughput +- [Telemetry & Traces](telemetry.md) — Record and analyze inference metrics from production use +- [Agents](agents.md) — Configure the `OrchestratorAgent` used by `jarvis-agent` backend +- [Tools](tools.md) — Available tools for agent-backed evaluations +- [Python SDK](python-sdk.md) — Programmatic access to OpenJarvis inference and agents diff --git a/evals/cli.py b/evals/cli.py index 0d959f71..a9632dd4 100644 --- a/evals/cli.py +++ b/evals/cli.py @@ -89,19 +89,108 @@ def _build_judge_backend(judge_model: str): return JarvisDirectBackend(engine_key="cloud") +def _print_summary(summary) -> None: + """Print a single run summary.""" + click.echo(f"\n{'=' * 60}") + click.echo(f"Benchmark: {summary.benchmark}") + click.echo(f"Model: {summary.model}") + click.echo(f"Backend: {summary.backend}") + click.echo(f"Samples: {summary.total_samples}") + click.echo(f"Scored: {summary.scored_samples}") + click.echo(f"Correct: {summary.correct}") + click.echo(f"Accuracy: {summary.accuracy:.4f}") + click.echo(f"Errors: {summary.errors}") + click.echo(f"Latency: {summary.mean_latency_seconds:.2f}s (mean)") + click.echo(f"Cost: ${summary.total_cost_usd:.4f}") + if summary.per_subject: + click.echo("\nPer-subject breakdown:") + for subj, stats in sorted(summary.per_subject.items()): + click.echo(f" {subj}: {stats['accuracy']:.4f} " + f"({int(stats['correct'])}/{int(stats['scored'])})") + click.echo(f"{'=' * 60}") + + +def _run_single(config) -> object: + """Run a single eval from a RunConfig and return the summary.""" + from evals.core.runner import EvalRunner + + eval_backend = _build_backend( + config.backend, + config.engine_key, + config.agent_name or "orchestrator", + config.tools, + ) + dataset = _build_dataset(config.benchmark) + judge_backend = _build_judge_backend(config.judge_model) + scorer = _build_scorer(config.benchmark, judge_backend, config.judge_model) + + runner = EvalRunner(config, dataset, eval_backend, scorer) + try: + return runner.run() + finally: + eval_backend.close() + judge_backend.close() + + +def _run_from_config(config_path: str, verbose: bool) -> None: + """Load a TOML config and run the full models x benchmarks matrix.""" + from evals.core.config import expand_suite, load_eval_config + + suite = load_eval_config(config_path) + run_configs = expand_suite(suite) + + suite_name = suite.meta.name or Path(config_path).stem + click.echo(f"Suite: {suite_name}") + if suite.meta.description: + click.echo(f" {suite.meta.description}") + click.echo(f" {len(suite.models)} model(s) x {len(suite.benchmarks)} " + f"benchmark(s) = {len(run_configs)} run(s)\n") + + # Ensure output directory exists + output_dir = Path(suite.run.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + summaries = [] + for i, rc in enumerate(run_configs, 1): + click.echo(f"--- [{i}/{len(run_configs)}] {rc.benchmark} / {rc.model} ---") + try: + summary = _run_single(rc) + summaries.append(summary) + click.echo(f" {summary.accuracy:.4f} " + f"({summary.correct}/{summary.scored_samples})") + except Exception as exc: + click.echo(f" FAILED: {exc}", err=True) + + # Print overall summary table + if summaries: + click.echo(f"\n{'=' * 60}") + click.echo(f"Suite Results: {suite_name}") + click.echo(f"{'=' * 60}") + click.echo(f" {'Benchmark':12s} {'Model':20s} {'Accuracy':>10s} {'Scored':>8s}") + click.echo(f" {'-' * 12} {'-' * 20} {'-' * 10} {'-' * 8}") + for s in summaries: + model_display = s.model[:20] + click.echo(f" {s.benchmark:12s} {model_display:20s} " + f"{s.accuracy:10.4f} " + f"{s.correct}/{s.scored_samples:>5}") + click.echo(f"{'=' * 60}") + + @click.group() def main(): """OpenJarvis Evaluation Framework.""" @main.command() -@click.option("-b", "--benchmark", required=True, +@click.option("-c", "--config", "config_path", default=None, + type=click.Path(), help="TOML config file for suite runs") +@click.option("-b", "--benchmark", default=None, type=click.Choice(list(BENCHMARKS.keys())), help="Benchmark to run") @click.option("--backend", default="jarvis-direct", type=click.Choice(list(BACKENDS.keys())), help="Inference backend") -@click.option("-m", "--model", required=True, help="Model identifier") +@click.option("-m", "--model", default=None, help="Model identifier") @click.option("-e", "--engine", "engine_key", default=None, help="Engine key (ollama, vllm, cloud, ...)") @click.option("--agent", "agent_name", default="orchestrator", @@ -123,13 +212,30 @@ def main(): @click.option("--max-tokens", type=int, default=2048, help="Max output tokens") @click.option("-v", "--verbose", is_flag=True, help="Verbose logging") -def run(benchmark, backend, model, engine_key, agent_name, tools, - max_samples, max_workers, judge_model, output_path, seed, +@click.pass_context +def run(ctx, config_path, benchmark, backend, model, engine_key, agent_name, + tools, max_samples, max_workers, judge_model, output_path, seed, dataset_split, temperature, max_tokens, verbose): - """Run a single benchmark evaluation.""" + """Run a single benchmark evaluation, or a full suite from a TOML config.""" _setup_logging(verbose) - from evals.core.runner import EvalRunner + # Config-driven mode + if config_path is not None: + _run_from_config(config_path, verbose) + return + + # CLI-driven mode: validate required args + if benchmark is None: + raise click.UsageError( + "Missing option '-b' / '--benchmark' " + "(required when --config is not provided)" + ) + if model is None: + raise click.UsageError( + "Missing option '-m' / '--model' " + "(required when --config is not provided)" + ) + from evals.core.types import RunConfig tool_list = [t.strip() for t in tools.split(",") if t.strip()] if tools else [] @@ -151,37 +257,8 @@ def run(benchmark, backend, model, engine_key, agent_name, tools, dataset_split=dataset_split, ) - eval_backend = _build_backend(backend, engine_key, agent_name, tool_list) - dataset = _build_dataset(benchmark) - judge_backend = _build_judge_backend(judge_model) - scorer = _build_scorer(benchmark, judge_backend, judge_model) - - runner = EvalRunner(config, dataset, eval_backend, scorer) - - try: - summary = runner.run() - finally: - eval_backend.close() - judge_backend.close() - - # Print summary - click.echo(f"\n{'=' * 60}") - click.echo(f"Benchmark: {summary.benchmark}") - click.echo(f"Model: {summary.model}") - click.echo(f"Backend: {summary.backend}") - click.echo(f"Samples: {summary.total_samples}") - click.echo(f"Scored: {summary.scored_samples}") - click.echo(f"Correct: {summary.correct}") - click.echo(f"Accuracy: {summary.accuracy:.4f}") - click.echo(f"Errors: {summary.errors}") - click.echo(f"Latency: {summary.mean_latency_seconds:.2f}s (mean)") - click.echo(f"Cost: ${summary.total_cost_usd:.4f}") - if summary.per_subject: - click.echo("\nPer-subject breakdown:") - for subj, stats in sorted(summary.per_subject.items()): - click.echo(f" {subj}: {stats['accuracy']:.4f} " - f"({int(stats['correct'])}/{int(stats['scored'])})") - click.echo(f"{'=' * 60}") + summary = _run_single(config) + _print_summary(summary) @main.command("run-all") diff --git a/evals/configs/full-suite.toml b/evals/configs/full-suite.toml new file mode 100644 index 00000000..1b255c7d --- /dev/null +++ b/evals/configs/full-suite.toml @@ -0,0 +1,63 @@ +# Full suite eval config — 3 models x 4 benchmarks with overrides. + +[meta] +name = "full-suite-v1" +description = "Evaluate all benchmarks against production models" + +[defaults] +temperature = 0.0 +max_tokens = 2048 + +[judge] +model = "gpt-4o" +temperature = 0.0 +max_tokens = 1024 + +[run] +max_workers = 4 +output_dir = "results/" +seed = 42 + +# --- Models --- + +[[models]] +name = "qwen3:8b" +engine = "ollama" +temperature = 0.3 +max_tokens = 4096 + +[[models]] +name = "gpt-4o" +provider = "openai" + +[[models]] +name = "llama3:70b" +engine = "vllm" +temperature = 0.1 + +# --- Benchmarks --- + +[[benchmarks]] +name = "supergpqa" +backend = "jarvis-direct" +max_samples = 200 +split = "train" + +[[benchmarks]] +name = "gaia" +backend = "jarvis-agent" +agent = "orchestrator" +tools = ["file_read", "calculator"] +max_samples = 50 +judge_model = "claude-sonnet-4-20250514" + +[[benchmarks]] +name = "frames" +backend = "jarvis-direct" +max_samples = 100 + +[[benchmarks]] +name = "wildchat" +backend = "jarvis-direct" +max_samples = 150 +temperature = 0.7 diff --git a/evals/configs/minimal.toml b/evals/configs/minimal.toml new file mode 100644 index 00000000..b8bccf8c --- /dev/null +++ b/evals/configs/minimal.toml @@ -0,0 +1,7 @@ +# Minimal eval config — 1 model, 1 benchmark, all defaults. + +[[models]] +name = "qwen3:8b" + +[[benchmarks]] +name = "supergpqa" diff --git a/evals/configs/single-run.toml b/evals/configs/single-run.toml new file mode 100644 index 00000000..a11f89fd --- /dev/null +++ b/evals/configs/single-run.toml @@ -0,0 +1,31 @@ +# Single-run eval config — 1 model, 1 benchmark, all sections populated. + +[meta] +name = "single-run-example" +description = "Evaluate SuperGPQA with a single model and full configuration" + +[defaults] +temperature = 0.0 +max_tokens = 2048 + +[judge] +model = "gpt-4o" +temperature = 0.0 +max_tokens = 1024 + +[run] +max_workers = 4 +output_dir = "results/" +seed = 42 + +[[models]] +name = "qwen3:8b" +engine = "ollama" +temperature = 0.3 +max_tokens = 4096 + +[[benchmarks]] +name = "supergpqa" +backend = "jarvis-direct" +max_samples = 100 +split = "train" diff --git a/evals/core/config.py b/evals/core/config.py new file mode 100644 index 00000000..e9cd7321 --- /dev/null +++ b/evals/core/config.py @@ -0,0 +1,220 @@ +"""TOML config loader and matrix expansion for eval suites.""" + +from __future__ import annotations + +import logging +import sys +from pathlib import Path +from typing import List + +from evals.core.types import ( + BenchmarkConfig, + DefaultsConfig, + EvalSuiteConfig, + ExecutionConfig, + JudgeConfig, + MetaConfig, + ModelConfig, + RunConfig, +) + +if sys.version_info >= (3, 11): + import tomllib +else: + try: + import tomli as tomllib # type: ignore[no-redef] + except ImportError as exc: + raise ImportError( + "Python 3.10 requires the 'tomli' package. " + "Install it with: pip install tomli" + ) from exc + +logger = logging.getLogger(__name__) + +VALID_BACKENDS = {"jarvis-direct", "jarvis-agent"} + +# Known benchmark names (used for warnings, not hard validation) +KNOWN_BENCHMARKS = {"supergpqa", "gaia", "frames", "wildchat"} + + +class EvalConfigError(Exception): + """Raised for structural issues in eval config files.""" + + +def load_eval_config(path: str | Path) -> EvalSuiteConfig: + """Load and validate an eval suite config from a TOML file. + + Args: + path: Path to the TOML config file. + + Returns: + Validated EvalSuiteConfig. + + Raises: + EvalConfigError: On structural validation failures. + FileNotFoundError: If the config file doesn't exist. + """ + path = Path(path) + if not path.exists(): + raise FileNotFoundError(f"Config file not found: {path}") + + with open(path, "rb") as f: + raw = tomllib.load(f) + + # Parse [meta] + meta_raw = raw.get("meta", {}) + meta = MetaConfig( + name=meta_raw.get("name", ""), + description=meta_raw.get("description", ""), + ) + + # Parse [defaults] + defaults_raw = raw.get("defaults", {}) + defaults = DefaultsConfig( + temperature=float(defaults_raw.get("temperature", 0.0)), + max_tokens=int(defaults_raw.get("max_tokens", 2048)), + ) + + # Parse [judge] + judge_raw = raw.get("judge", {}) + judge = JudgeConfig( + model=judge_raw.get("model", "gpt-4o"), + provider=judge_raw.get("provider"), + temperature=float(judge_raw.get("temperature", 0.0)), + max_tokens=int(judge_raw.get("max_tokens", 1024)), + ) + + # Parse [run] + run_raw = raw.get("run", {}) + execution = ExecutionConfig( + max_workers=int(run_raw.get("max_workers", 4)), + output_dir=run_raw.get("output_dir", "results/"), + seed=int(run_raw.get("seed", 42)), + ) + + # Parse [[models]] + models_raw = raw.get("models", []) + if not models_raw: + raise EvalConfigError("Config must define at least one [[models]] entry") + + models: List[ModelConfig] = [] + for m in models_raw: + if not m.get("name"): + raise EvalConfigError("Each [[models]] entry must have a 'name' field") + models.append(ModelConfig( + name=m["name"], + engine=m.get("engine"), + provider=m.get("provider"), + temperature=float(m["temperature"]) if "temperature" in m else None, + max_tokens=int(m["max_tokens"]) if "max_tokens" in m else None, + )) + + # Parse [[benchmarks]] + benchmarks_raw = raw.get("benchmarks", []) + if not benchmarks_raw: + raise EvalConfigError( + "Config must define at least one [[benchmarks]] entry" + ) + + benchmarks: List[BenchmarkConfig] = [] + for b in benchmarks_raw: + if not b.get("name"): + raise EvalConfigError( + "Each [[benchmarks]] entry must have a 'name' field" + ) + + backend = b.get("backend", "jarvis-direct") + if backend not in VALID_BACKENDS: + raise EvalConfigError( + f"Invalid backend '{backend}' for benchmark '{b['name']}'. " + f"Must be one of: {', '.join(sorted(VALID_BACKENDS))}" + ) + + bench_name = b["name"] + if bench_name not in KNOWN_BENCHMARKS: + logger.warning("Unknown benchmark name: '%s'", bench_name) + + tools_raw = b.get("tools", []) + benchmarks.append(BenchmarkConfig( + name=bench_name, + backend=backend, + max_samples=int(b["max_samples"]) if "max_samples" in b else None, + split=b.get("split"), + agent=b.get("agent"), + tools=list(tools_raw), + judge_model=b.get("judge_model"), + temperature=float(b["temperature"]) if "temperature" in b else None, + max_tokens=int(b["max_tokens"]) if "max_tokens" in b else None, + )) + + return EvalSuiteConfig( + meta=meta, + defaults=defaults, + judge=judge, + run=execution, + models=models, + benchmarks=benchmarks, + ) + + +def expand_suite(suite: EvalSuiteConfig) -> List[RunConfig]: + """Expand an EvalSuiteConfig into a list of RunConfigs (models x benchmarks). + + Merge precedence (highest wins): + benchmark-level > model-level > [defaults] > built-in defaults + + Args: + suite: The parsed eval suite config. + + Returns: + List of RunConfig, one per model-benchmark pair. + """ + configs: List[RunConfig] = [] + output_dir = suite.run.output_dir.rstrip("/") + + for model in suite.models: + for bench in suite.benchmarks: + # Temperature: benchmark > model > defaults + temperature = suite.defaults.temperature + if model.temperature is not None: + temperature = model.temperature + if bench.temperature is not None: + temperature = bench.temperature + + # Max tokens: benchmark > model > defaults + max_tokens = suite.defaults.max_tokens + if model.max_tokens is not None: + max_tokens = model.max_tokens + if bench.max_tokens is not None: + max_tokens = bench.max_tokens + + # Judge model: benchmark > [judge] + judge_model = suite.judge.model + if bench.judge_model is not None: + judge_model = bench.judge_model + + # Auto-generate output path + model_slug = model.name.replace("/", "-").replace(":", "-") + output_path = f"{output_dir}/{bench.name}_{model_slug}.jsonl" + + configs.append(RunConfig( + benchmark=bench.name, + backend=bench.backend, + model=model.name, + max_samples=bench.max_samples, + max_workers=suite.run.max_workers, + temperature=temperature, + max_tokens=max_tokens, + judge_model=judge_model, + engine_key=model.engine, + agent_name=bench.agent, + tools=list(bench.tools), + output_path=output_path, + seed=suite.run.seed, + dataset_split=bench.split, + )) + + return configs + + +__all__ = ["EvalConfigError", "load_eval_config", "expand_suite"] diff --git a/evals/core/types.py b/evals/core/types.py index cd5dd2fb..e822a345 100644 --- a/evals/core/types.py +++ b/evals/core/types.py @@ -74,4 +74,94 @@ class RunSummary: ended_at: float = 0.0 -__all__ = ["EvalRecord", "EvalResult", "RunConfig", "RunSummary"] +# --------------------------------------------------------------------------- +# Eval suite config dataclasses (TOML config system) +# --------------------------------------------------------------------------- + + +@dataclass(slots=True) +class MetaConfig: + """Suite-level metadata.""" + + name: str = "" + description: str = "" + + +@dataclass(slots=True) +class DefaultsConfig: + """Default generation parameters applied to all runs.""" + + temperature: float = 0.0 + max_tokens: int = 2048 + + +@dataclass(slots=True) +class JudgeConfig: + """Configuration for the LLM judge.""" + + model: str = "gpt-4o" + provider: Optional[str] = None + temperature: float = 0.0 + max_tokens: int = 1024 + + +@dataclass(slots=True) +class ExecutionConfig: + """Execution-level settings for the eval run.""" + + max_workers: int = 4 + output_dir: str = "results/" + seed: int = 42 + + +@dataclass(slots=True) +class ModelConfig: + """Configuration for a single model in the eval suite.""" + + name: str = "" + engine: Optional[str] = None + provider: Optional[str] = None + temperature: Optional[float] = None + max_tokens: Optional[int] = None + + +@dataclass(slots=True) +class BenchmarkConfig: + """Configuration for a single benchmark in the eval suite.""" + + name: str = "" + backend: str = "jarvis-direct" + max_samples: Optional[int] = None + split: Optional[str] = None + agent: Optional[str] = None + tools: List[str] = field(default_factory=list) + judge_model: Optional[str] = None + temperature: Optional[float] = None + max_tokens: Optional[int] = None + + +@dataclass(slots=True) +class EvalSuiteConfig: + """Top-level configuration for an eval suite (models x benchmarks).""" + + meta: MetaConfig = field(default_factory=MetaConfig) + defaults: DefaultsConfig = field(default_factory=DefaultsConfig) + judge: JudgeConfig = field(default_factory=JudgeConfig) + run: ExecutionConfig = field(default_factory=ExecutionConfig) + models: List[ModelConfig] = field(default_factory=list) + benchmarks: List[BenchmarkConfig] = field(default_factory=list) + + +__all__ = [ + "EvalRecord", + "EvalResult", + "RunConfig", + "RunSummary", + "MetaConfig", + "DefaultsConfig", + "JudgeConfig", + "ExecutionConfig", + "ModelConfig", + "BenchmarkConfig", + "EvalSuiteConfig", +] diff --git a/evals/pyproject.toml b/evals/pyproject.toml index 5c1227b2..21cfe023 100644 --- a/evals/pyproject.toml +++ b/evals/pyproject.toml @@ -10,6 +10,7 @@ dependencies = [ "huggingface-hub>=0.20", "tqdm>=4.65", "rich>=13", + "tomli>=2.0; python_version < '3.11'", ] [project.optional-dependencies] diff --git a/evals/tests/test_config.py b/evals/tests/test_config.py new file mode 100644 index 00000000..7319b123 --- /dev/null +++ b/evals/tests/test_config.py @@ -0,0 +1,541 @@ +"""Tests for eval suite config loading and matrix expansion.""" + +from __future__ import annotations + +import textwrap +from pathlib import Path + +import pytest + +from evals.core.config import EvalConfigError, expand_suite, load_eval_config +from evals.core.types import ( + BenchmarkConfig, + DefaultsConfig, + EvalSuiteConfig, + ExecutionConfig, + JudgeConfig, + MetaConfig, + ModelConfig, + RunConfig, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _write_toml(tmp_path: Path, content: str) -> Path: + """Write a TOML string to a temp file and return its path.""" + p = tmp_path / "suite.toml" + p.write_text(textwrap.dedent(content)) + return p + + +# --------------------------------------------------------------------------- +# Dataclass defaults +# --------------------------------------------------------------------------- + + +class TestDataclassDefaults: + def test_meta_config_defaults(self): + m = MetaConfig() + assert m.name == "" + assert m.description == "" + + def test_defaults_config_defaults(self): + d = DefaultsConfig() + assert d.temperature == 0.0 + assert d.max_tokens == 2048 + + def test_judge_config_defaults(self): + j = JudgeConfig() + assert j.model == "gpt-4o" + assert j.provider is None + assert j.temperature == 0.0 + assert j.max_tokens == 1024 + + def test_execution_config_defaults(self): + e = ExecutionConfig() + assert e.max_workers == 4 + assert e.output_dir == "results/" + assert e.seed == 42 + + def test_model_config_defaults(self): + m = ModelConfig(name="test") + assert m.engine is None + assert m.provider is None + assert m.temperature is None + assert m.max_tokens is None + + def test_benchmark_config_defaults(self): + b = BenchmarkConfig(name="supergpqa") + assert b.backend == "jarvis-direct" + assert b.max_samples is None + assert b.split is None + assert b.agent is None + assert b.tools == [] + assert b.judge_model is None + assert b.temperature is None + assert b.max_tokens is None + + def test_eval_suite_config_defaults(self): + s = EvalSuiteConfig() + assert s.meta.name == "" + assert s.defaults.temperature == 0.0 + assert s.judge.model == "gpt-4o" + assert s.run.max_workers == 4 + assert s.models == [] + assert s.benchmarks == [] + + +# --------------------------------------------------------------------------- +# TOML loading +# --------------------------------------------------------------------------- + + +class TestLoadEvalConfig: + def test_minimal_config(self, tmp_path): + p = _write_toml(tmp_path, """\ + [[models]] + name = "qwen3:8b" + + [[benchmarks]] + name = "supergpqa" + """) + suite = load_eval_config(p) + assert len(suite.models) == 1 + assert suite.models[0].name == "qwen3:8b" + assert len(suite.benchmarks) == 1 + assert suite.benchmarks[0].name == "supergpqa" + # Defaults should be applied + assert suite.defaults.temperature == 0.0 + assert suite.judge.model == "gpt-4o" + + def test_full_config(self, tmp_path): + p = _write_toml(tmp_path, """\ + [meta] + name = "test-suite" + description = "A test suite" + + [defaults] + temperature = 0.5 + max_tokens = 4096 + + [judge] + model = "claude-sonnet" + temperature = 0.1 + max_tokens = 512 + + [run] + max_workers = 8 + output_dir = "out/" + seed = 123 + + [[models]] + name = "model-a" + engine = "ollama" + temperature = 0.3 + + [[models]] + name = "model-b" + provider = "openai" + + [[benchmarks]] + name = "supergpqa" + backend = "jarvis-direct" + max_samples = 100 + split = "test" + + [[benchmarks]] + name = "gaia" + backend = "jarvis-agent" + agent = "orchestrator" + tools = ["calc", "think"] + judge_model = "gpt-4o" + """) + suite = load_eval_config(p) + assert suite.meta.name == "test-suite" + assert suite.meta.description == "A test suite" + assert suite.defaults.temperature == 0.5 + assert suite.defaults.max_tokens == 4096 + assert suite.judge.model == "claude-sonnet" + assert suite.judge.temperature == 0.1 + assert suite.run.max_workers == 8 + assert suite.run.output_dir == "out/" + assert suite.run.seed == 123 + + assert len(suite.models) == 2 + assert suite.models[0].name == "model-a" + assert suite.models[0].engine == "ollama" + assert suite.models[0].temperature == 0.3 + assert suite.models[1].name == "model-b" + assert suite.models[1].provider == "openai" + + assert len(suite.benchmarks) == 2 + assert suite.benchmarks[0].max_samples == 100 + assert suite.benchmarks[0].split == "test" + assert suite.benchmarks[1].agent == "orchestrator" + assert suite.benchmarks[1].tools == ["calc", "think"] + assert suite.benchmarks[1].judge_model == "gpt-4o" + + def test_missing_models_raises(self, tmp_path): + p = _write_toml(tmp_path, """\ + [[benchmarks]] + name = "supergpqa" + """) + with pytest.raises(EvalConfigError, match="at least one \\[\\[models\\]\\]"): + load_eval_config(p) + + def test_missing_benchmarks_raises(self, tmp_path): + p = _write_toml(tmp_path, """\ + [[models]] + name = "qwen3:8b" + """) + with pytest.raises(EvalConfigError, match="at least one \\[\\[benchmarks\\]\\]"): + load_eval_config(p) + + def test_model_without_name_raises(self, tmp_path): + p = _write_toml(tmp_path, """\ + [[models]] + engine = "ollama" + + [[benchmarks]] + name = "supergpqa" + """) + with pytest.raises(EvalConfigError, match="'name' field"): + load_eval_config(p) + + def test_benchmark_without_name_raises(self, tmp_path): + p = _write_toml(tmp_path, """\ + [[models]] + name = "qwen3:8b" + + [[benchmarks]] + backend = "jarvis-direct" + """) + with pytest.raises(EvalConfigError, match="'name' field"): + load_eval_config(p) + + def test_invalid_backend_raises(self, tmp_path): + p = _write_toml(tmp_path, """\ + [[models]] + name = "qwen3:8b" + + [[benchmarks]] + name = "supergpqa" + backend = "invalid-backend" + """) + with pytest.raises(EvalConfigError, match="Invalid backend"): + load_eval_config(p) + + def test_unknown_benchmark_warns(self, tmp_path, caplog): + p = _write_toml(tmp_path, """\ + [[models]] + name = "qwen3:8b" + + [[benchmarks]] + name = "custom-bench" + """) + import logging + with caplog.at_level(logging.WARNING): + suite = load_eval_config(p) + assert suite.benchmarks[0].name == "custom-bench" + assert "Unknown benchmark name" in caplog.text + + def test_file_not_found(self): + with pytest.raises(FileNotFoundError): + load_eval_config("/nonexistent/path.toml") + + def test_malformed_toml(self, tmp_path): + p = tmp_path / "bad.toml" + p.write_text("[[[ invalid toml") + with pytest.raises(Exception): # tomllib raises various errors + load_eval_config(p) + + def test_empty_models_list(self, tmp_path): + p = _write_toml(tmp_path, """\ + models = [] + + [[benchmarks]] + name = "supergpqa" + """) + with pytest.raises(EvalConfigError, match="at least one \\[\\[models\\]\\]"): + load_eval_config(p) + + def test_empty_benchmarks_list(self, tmp_path): + p = _write_toml(tmp_path, """\ + [[models]] + name = "qwen3:8b" + + benchmarks = [] + """) + with pytest.raises(EvalConfigError, match="at least one \\[\\[benchmarks\\]\\]"): + load_eval_config(p) + + +# --------------------------------------------------------------------------- +# Example config files load correctly +# --------------------------------------------------------------------------- + + +class TestExampleConfigs: + @pytest.fixture(params=["minimal.toml", "single-run.toml", "full-suite.toml"]) + def example_config(self, request): + configs_dir = Path(__file__).resolve().parent.parent / "configs" + return configs_dir / request.param + + def test_example_configs_load(self, example_config): + suite = load_eval_config(example_config) + assert len(suite.models) >= 1 + assert len(suite.benchmarks) >= 1 + + def test_full_suite_dimensions(self): + configs_dir = Path(__file__).resolve().parent.parent / "configs" + suite = load_eval_config(configs_dir / "full-suite.toml") + assert len(suite.models) == 3 + assert len(suite.benchmarks) == 4 + + +# --------------------------------------------------------------------------- +# Matrix expansion +# --------------------------------------------------------------------------- + + +class TestExpandSuite: + def test_single_model_single_benchmark(self): + suite = EvalSuiteConfig( + models=[ModelConfig(name="m1")], + benchmarks=[BenchmarkConfig(name="supergpqa")], + ) + configs = expand_suite(suite) + assert len(configs) == 1 + assert configs[0].model == "m1" + assert configs[0].benchmark == "supergpqa" + + def test_cross_product(self): + suite = EvalSuiteConfig( + models=[ModelConfig(name="m1"), ModelConfig(name="m2")], + benchmarks=[ + BenchmarkConfig(name="supergpqa"), + BenchmarkConfig(name="gaia"), + BenchmarkConfig(name="frames"), + ], + ) + configs = expand_suite(suite) + assert len(configs) == 6 # 2 x 3 + + def test_temperature_merge_defaults(self): + suite = EvalSuiteConfig( + defaults=DefaultsConfig(temperature=0.5), + models=[ModelConfig(name="m1")], + benchmarks=[BenchmarkConfig(name="supergpqa")], + ) + configs = expand_suite(suite) + assert configs[0].temperature == 0.5 + + def test_temperature_model_overrides_defaults(self): + suite = EvalSuiteConfig( + defaults=DefaultsConfig(temperature=0.5), + models=[ModelConfig(name="m1", temperature=0.3)], + benchmarks=[BenchmarkConfig(name="supergpqa")], + ) + configs = expand_suite(suite) + assert configs[0].temperature == 0.3 + + def test_temperature_benchmark_overrides_model(self): + suite = EvalSuiteConfig( + defaults=DefaultsConfig(temperature=0.5), + models=[ModelConfig(name="m1", temperature=0.3)], + benchmarks=[BenchmarkConfig(name="supergpqa", temperature=0.9)], + ) + configs = expand_suite(suite) + assert configs[0].temperature == 0.9 + + def test_max_tokens_merge_precedence(self): + suite = EvalSuiteConfig( + defaults=DefaultsConfig(max_tokens=1000), + models=[ModelConfig(name="m1", max_tokens=2000)], + benchmarks=[BenchmarkConfig(name="supergpqa", max_tokens=3000)], + ) + configs = expand_suite(suite) + assert configs[0].max_tokens == 3000 + + def test_max_tokens_model_only(self): + suite = EvalSuiteConfig( + defaults=DefaultsConfig(max_tokens=1000), + models=[ModelConfig(name="m1", max_tokens=2000)], + benchmarks=[BenchmarkConfig(name="supergpqa")], + ) + configs = expand_suite(suite) + assert configs[0].max_tokens == 2000 + + def test_judge_model_from_judge_config(self): + suite = EvalSuiteConfig( + judge=JudgeConfig(model="custom-judge"), + models=[ModelConfig(name="m1")], + benchmarks=[BenchmarkConfig(name="supergpqa")], + ) + configs = expand_suite(suite) + assert configs[0].judge_model == "custom-judge" + + def test_judge_model_benchmark_override(self): + suite = EvalSuiteConfig( + judge=JudgeConfig(model="default-judge"), + models=[ModelConfig(name="m1")], + benchmarks=[BenchmarkConfig(name="supergpqa", judge_model="bench-judge")], + ) + configs = expand_suite(suite) + assert configs[0].judge_model == "bench-judge" + + def test_output_path_auto_generated(self): + suite = EvalSuiteConfig( + run=ExecutionConfig(output_dir="out/"), + models=[ModelConfig(name="qwen3:8b")], + benchmarks=[BenchmarkConfig(name="supergpqa")], + ) + configs = expand_suite(suite) + assert configs[0].output_path == "out/supergpqa_qwen3-8b.jsonl" + + def test_output_path_model_slug(self): + suite = EvalSuiteConfig( + run=ExecutionConfig(output_dir="results"), + models=[ModelConfig(name="org/model:v2")], + benchmarks=[BenchmarkConfig(name="gaia")], + ) + configs = expand_suite(suite) + assert configs[0].output_path == "results/gaia_org-model-v2.jsonl" + + def test_engine_key_from_model(self): + suite = EvalSuiteConfig( + models=[ModelConfig(name="m1", engine="ollama")], + benchmarks=[BenchmarkConfig(name="supergpqa")], + ) + configs = expand_suite(suite) + assert configs[0].engine_key == "ollama" + + def test_agent_from_benchmark(self): + suite = EvalSuiteConfig( + models=[ModelConfig(name="m1")], + benchmarks=[BenchmarkConfig(name="gaia", agent="orchestrator")], + ) + configs = expand_suite(suite) + assert configs[0].agent_name == "orchestrator" + + def test_tools_from_benchmark(self): + suite = EvalSuiteConfig( + models=[ModelConfig(name="m1")], + benchmarks=[BenchmarkConfig(name="gaia", tools=["calc", "think"])], + ) + configs = expand_suite(suite) + assert configs[0].tools == ["calc", "think"] + + def test_tools_list_not_shared(self): + """Each RunConfig should get its own tools list (no shared mutation).""" + suite = EvalSuiteConfig( + models=[ModelConfig(name="m1"), ModelConfig(name="m2")], + benchmarks=[BenchmarkConfig(name="gaia", tools=["calc"])], + ) + configs = expand_suite(suite) + configs[0].tools.append("extra") + assert configs[1].tools == ["calc"] + + def test_max_workers_and_seed_from_run(self): + suite = EvalSuiteConfig( + run=ExecutionConfig(max_workers=16, seed=99), + models=[ModelConfig(name="m1")], + benchmarks=[BenchmarkConfig(name="supergpqa")], + ) + configs = expand_suite(suite) + assert configs[0].max_workers == 16 + assert configs[0].seed == 99 + + def test_max_samples_and_split_from_benchmark(self): + suite = EvalSuiteConfig( + models=[ModelConfig(name="m1")], + benchmarks=[ + BenchmarkConfig(name="supergpqa", max_samples=50, split="test") + ], + ) + configs = expand_suite(suite) + assert configs[0].max_samples == 50 + assert configs[0].dataset_split == "test" + + def test_backend_from_benchmark(self): + suite = EvalSuiteConfig( + models=[ModelConfig(name="m1")], + benchmarks=[BenchmarkConfig(name="gaia", backend="jarvis-agent")], + ) + configs = expand_suite(suite) + assert configs[0].backend == "jarvis-agent" + + def test_expand_returns_run_config_type(self): + suite = EvalSuiteConfig( + models=[ModelConfig(name="m1")], + benchmarks=[BenchmarkConfig(name="supergpqa")], + ) + configs = expand_suite(suite) + assert all(isinstance(c, RunConfig) for c in configs) + + +# --------------------------------------------------------------------------- +# CLI integration +# --------------------------------------------------------------------------- + + +class TestCLIConfig: + def test_run_missing_benchmark_and_config(self): + from click.testing import CliRunner + from evals.cli import main + + runner = CliRunner() + result = runner.invoke(main, ["run", "-m", "qwen3:8b"]) + assert result.exit_code != 0 + assert "--benchmark" in result.output or "benchmark" in result.output.lower() + + def test_run_missing_model_and_config(self): + from click.testing import CliRunner + from evals.cli import main + + runner = CliRunner() + result = runner.invoke(main, ["run", "-b", "supergpqa"]) + assert result.exit_code != 0 + assert "--model" in result.output or "model" in result.output.lower() + + def test_run_config_file_not_found(self): + from click.testing import CliRunner + from evals.cli import main + + runner = CliRunner() + result = runner.invoke(main, ["run", "--config", "/nonexistent.toml"]) + assert result.exit_code != 0 + + def test_run_config_loads_and_prints_suite_info(self, tmp_path): + """Verify --config loads config and prints suite header. + + We don't actually run the eval (requires backends), but we verify + the config is loaded and the matrix expansion starts. + """ + from unittest.mock import patch + + from click.testing import CliRunner + from evals.cli import main + + p = _write_toml(tmp_path, """\ + [meta] + name = "test-suite" + + [[models]] + name = "qwen3:8b" + + [[benchmarks]] + name = "supergpqa" + """) + + runner = CliRunner() + with patch("evals.cli._run_single", side_effect=Exception("mock")): + result = runner.invoke(main, ["run", "--config", str(p)]) + + # Should print suite info before failing + assert "test-suite" in result.output + assert "1 model(s) x 1 benchmark(s)" in result.output diff --git a/evals/tests/test_types.py b/evals/tests/test_types.py index 64cfcfd6..fda32629 100644 --- a/evals/tests/test_types.py +++ b/evals/tests/test_types.py @@ -2,7 +2,19 @@ from __future__ import annotations -from evals.core.types import EvalRecord, EvalResult, RunConfig, RunSummary +from evals.core.types import ( + BenchmarkConfig, + DefaultsConfig, + EvalRecord, + EvalResult, + EvalSuiteConfig, + ExecutionConfig, + JudgeConfig, + MetaConfig, + ModelConfig, + RunConfig, + RunSummary, +) class TestEvalRecord: @@ -86,3 +98,138 @@ class TestRunSummary: assert s.accuracy == 0.495 assert s.per_subject["math"]["accuracy"] == 0.5 assert s.started_at == 0.0 + + +# --------------------------------------------------------------------------- +# Eval suite config dataclasses +# --------------------------------------------------------------------------- + + +class TestMetaConfig: + def test_defaults(self): + m = MetaConfig() + assert m.name == "" + assert m.description == "" + + def test_with_values(self): + m = MetaConfig(name="suite-1", description="First suite") + assert m.name == "suite-1" + assert m.description == "First suite" + + +class TestDefaultsConfig: + def test_defaults(self): + d = DefaultsConfig() + assert d.temperature == 0.0 + assert d.max_tokens == 2048 + + def test_with_values(self): + d = DefaultsConfig(temperature=0.7, max_tokens=4096) + assert d.temperature == 0.7 + assert d.max_tokens == 4096 + + +class TestJudgeConfig: + def test_defaults(self): + j = JudgeConfig() + assert j.model == "gpt-4o" + assert j.provider is None + assert j.temperature == 0.0 + assert j.max_tokens == 1024 + + def test_with_values(self): + j = JudgeConfig(model="claude", provider="anthropic", temperature=0.1) + assert j.model == "claude" + assert j.provider == "anthropic" + assert j.temperature == 0.1 + + +class TestExecutionConfig: + def test_defaults(self): + e = ExecutionConfig() + assert e.max_workers == 4 + assert e.output_dir == "results/" + assert e.seed == 42 + + def test_with_values(self): + e = ExecutionConfig(max_workers=16, output_dir="out/", seed=99) + assert e.max_workers == 16 + assert e.output_dir == "out/" + assert e.seed == 99 + + +class TestModelConfig: + def test_required_name(self): + m = ModelConfig(name="qwen3:8b") + assert m.name == "qwen3:8b" + assert m.engine is None + assert m.provider is None + assert m.temperature is None + assert m.max_tokens is None + + def test_with_overrides(self): + m = ModelConfig( + name="gpt-4o", engine="cloud", provider="openai", + temperature=0.5, max_tokens=4096, + ) + assert m.engine == "cloud" + assert m.provider == "openai" + assert m.temperature == 0.5 + assert m.max_tokens == 4096 + + +class TestBenchmarkConfig: + def test_defaults(self): + b = BenchmarkConfig(name="supergpqa") + assert b.name == "supergpqa" + assert b.backend == "jarvis-direct" + assert b.max_samples is None + assert b.split is None + assert b.agent is None + assert b.tools == [] + assert b.judge_model is None + assert b.temperature is None + assert b.max_tokens is None + + def test_with_overrides(self): + b = BenchmarkConfig( + name="gaia", backend="jarvis-agent", max_samples=50, + split="test", agent="orchestrator", + tools=["calc", "think"], judge_model="custom-judge", + temperature=0.3, max_tokens=1024, + ) + assert b.backend == "jarvis-agent" + assert b.max_samples == 50 + assert b.split == "test" + assert b.agent == "orchestrator" + assert b.tools == ["calc", "think"] + assert b.judge_model == "custom-judge" + assert b.temperature == 0.3 + + def test_tools_list_independent(self): + """Each BenchmarkConfig instance should have its own tools list.""" + b1 = BenchmarkConfig(name="a") + b2 = BenchmarkConfig(name="b") + b1.tools.append("calc") + assert b2.tools == [] + + +class TestEvalSuiteConfig: + def test_defaults(self): + s = EvalSuiteConfig() + assert isinstance(s.meta, MetaConfig) + assert isinstance(s.defaults, DefaultsConfig) + assert isinstance(s.judge, JudgeConfig) + assert isinstance(s.run, ExecutionConfig) + assert s.models == [] + assert s.benchmarks == [] + + def test_with_entries(self): + s = EvalSuiteConfig( + meta=MetaConfig(name="test"), + models=[ModelConfig(name="m1"), ModelConfig(name="m2")], + benchmarks=[BenchmarkConfig(name="supergpqa")], + ) + assert s.meta.name == "test" + assert len(s.models) == 2 + assert len(s.benchmarks) == 1 diff --git a/mkdocs.yml b/mkdocs.yml index c3977047..be391411 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -130,6 +130,7 @@ nav: - Tools: user-guide/tools.md - Telemetry & Traces: user-guide/telemetry.md - Benchmarks: user-guide/benchmarks.md + - Evaluations: user-guide/evaluations.md - Security: user-guide/security.md - Channels: user-guide/channels.md - Architecture: @@ -156,6 +157,7 @@ nav: - Traces: api/traces.md - Telemetry: api/telemetry.md - Benchmarks: api/bench.md + - Evals: api/evals.md - Server: api/server.md - Security: api/security.md - Channels: api/channels.md diff --git a/src/openjarvis/cli/ask.py b/src/openjarvis/cli/ask.py index 9bf58126..bce2217d 100644 --- a/src/openjarvis/cli/ask.py +++ b/src/openjarvis/cli/ask.py @@ -124,7 +124,7 @@ def _run_agent( ctx = AgentContext() # Inject memory context into conversation if available - if config.memory.context_injection: + if config.agent.context_from_memory: try: from openjarvis.memory.context import ContextConfig, inject_context @@ -151,9 +151,13 @@ def _run_agent( @click.option("-m", "--model", "model_name", default=None, help="Model to use.") @click.option("-e", "--engine", "engine_key", default=None, help="Engine backend.") @click.option( - "-t", "--temperature", default=0.7, type=float, help="Sampling temperature." + "-t", "--temperature", default=None, type=float, + help="Sampling temperature (default: from config).", +) +@click.option( + "--max-tokens", default=None, type=int, + help="Max tokens to generate (default: from config).", ) -@click.option("--max-tokens", default=1024, type=int, help="Max tokens to generate.") @click.option("--json", "output_json", is_flag=True, help="Output raw JSON result.") @click.option("--no-stream", is_flag=True, help="Disable streaming (sync mode).") @click.option( @@ -187,6 +191,12 @@ def ask( # Load config config = load_config() + # Fall back to config values for generation params + if temperature is None: + temperature = config.intelligence.temperature + if max_tokens is None: + max_tokens = config.intelligence.max_tokens + # Set up telemetry bus = EventBus(record_history=True) telem_store: TelemetryStore | None = None @@ -200,7 +210,8 @@ def ask( # Discover engines register_builtin_models() - resolved = get_engine(config, engine_key) + effective_engine_key = engine_key or config.intelligence.preferred_engine or None + resolved = get_engine(config, effective_engine_key) if resolved is None: console.print( "[red bold]No inference engine available.[/red bold]\n\n" @@ -273,7 +284,7 @@ def ask( messages = [Message(role=Role.USER, content=query_text)] # Memory-augmented context injection - if not no_context and config.memory.context_injection: + if not no_context and config.agent.context_from_memory: try: from openjarvis.memory.context import ( ContextConfig, diff --git a/src/openjarvis/core/config.py b/src/openjarvis/core/config.py index fb28e434..e05bba24 100644 --- a/src/openjarvis/core/config.py +++ b/src/openjarvis/core/config.py @@ -190,35 +190,220 @@ def recommend_engine(hw: HardwareInfo) -> str: @dataclass(slots=True) +class OllamaEngineConfig: + """Per-engine config for Ollama.""" + + host: str = "http://localhost:11434" + + +@dataclass(slots=True) +class VLLMEngineConfig: + """Per-engine config for vLLM.""" + + host: str = "http://localhost:8000" + + +@dataclass(slots=True) +class SGLangEngineConfig: + """Per-engine config for SGLang.""" + + host: str = "http://localhost:30000" + + +@dataclass(slots=True) +class LlamaCppEngineConfig: + """Per-engine config for llama.cpp.""" + + host: str = "http://localhost:8080" + binary_path: str = "" + + +@dataclass class EngineConfig: - """Inference engine settings.""" + """Inference engine settings with nested per-engine configs.""" default: str = "ollama" - ollama_host: str = "http://localhost:11434" - vllm_host: str = "http://localhost:8000" - llamacpp_host: str = "http://localhost:8080" - llamacpp_path: str = "" - sglang_host: str = "http://localhost:30000" + ollama: OllamaEngineConfig = field(default_factory=OllamaEngineConfig) + vllm: VLLMEngineConfig = field(default_factory=VLLMEngineConfig) + sglang: SGLangEngineConfig = field(default_factory=SGLangEngineConfig) + llamacpp: LlamaCppEngineConfig = field(default_factory=LlamaCppEngineConfig) + + # Backward-compat properties for old flat attribute names + @property + def ollama_host(self) -> str: + """Deprecated: use ``engine.ollama.host``.""" + return self.ollama.host + + @ollama_host.setter + def ollama_host(self, value: str) -> None: + self.ollama.host = value + + @property + def vllm_host(self) -> str: + """Deprecated: use ``engine.vllm.host``.""" + return self.vllm.host + + @vllm_host.setter + def vllm_host(self, value: str) -> None: + self.vllm.host = value + + @property + def llamacpp_host(self) -> str: + """Deprecated: use ``engine.llamacpp.host``.""" + return self.llamacpp.host + + @llamacpp_host.setter + def llamacpp_host(self, value: str) -> None: + self.llamacpp.host = value + + @property + def llamacpp_path(self) -> str: + """Deprecated: use ``engine.llamacpp.binary_path``.""" + return self.llamacpp.binary_path + + @llamacpp_path.setter + def llamacpp_path(self, value: str) -> None: + self.llamacpp.binary_path = value + + @property + def sglang_host(self) -> str: + """Deprecated: use ``engine.sglang.host``.""" + return self.sglang.host + + @sglang_host.setter + def sglang_host(self, value: str) -> None: + self.sglang.host = value @dataclass(slots=True) class IntelligenceConfig: - """Model routing defaults.""" + """The model — identity, paths, quantization, and generation defaults.""" default_model: str = "" fallback_model: str = "" + model_path: str = "" # Local weights (HF repo, GGUF file, etc.) + checkpoint_path: str = "" # Checkpoint/adapter path + quantization: str = "none" # none, fp8, int8, int4, gguf_q4, gguf_q8 + preferred_engine: str = "" # Override engine for this model (e.g., "vllm") + provider: str = "" # local, openai, anthropic, google + # Generation defaults (overridable per-call) + temperature: float = 0.7 + max_tokens: int = 1024 + top_p: float = 0.9 + top_k: int = 40 + repetition_penalty: float = 1.0 + stop_sequences: str = "" # Comma-separated stop strings @dataclass(slots=True) -class LearningConfig: - """Learning / router policy settings.""" +class RoutingLearningConfig: + """Routing sub-policy config within Learning.""" - default_policy: str = "heuristic" - intelligence_policy: str = "none" # "none" | "sft" — updates model routing - agent_policy: str = "none" # "none" | "agent_advisor" — updates agent logic - tools_policy: str = "none" # "none" | "icl_updater" — updates tool usage - reward_weights: str = "" # comma-separated key=value, e.g. "latency=0.4,cost=0.3" - update_interval: int = 10 # traces between learning updates + policy: str = "heuristic" # heuristic | learned | grpo + min_samples: int = 5 # Min traces before trusting learned routing + + +@dataclass(slots=True) +class IntelligenceLearningConfig: + """Intelligence sub-policy config within Learning.""" + + policy: str = "none" # none | sft + + +@dataclass(slots=True) +class AgentLearningConfig: + """Agent sub-policy config within Learning.""" + + policy: str = "none" # none | agent_advisor | icl_updater + max_icl_examples: int = 20 + advisor_confidence_threshold: float = 0.7 + + +@dataclass(slots=True) +class MetricsConfig: + """Reward / optimization metric weights.""" + + accuracy_weight: float = 0.6 + latency_weight: float = 0.2 + cost_weight: float = 0.1 + efficiency_weight: float = 0.1 + + +@dataclass +class LearningConfig: + """Learning system settings with per-pillar sub-policies.""" + + enabled: bool = False + update_interval: int = 100 # traces between automatic policy updates + auto_update: bool = False # auto-trigger updates on interval + routing: RoutingLearningConfig = field(default_factory=RoutingLearningConfig) + intelligence: IntelligenceLearningConfig = field( + default_factory=IntelligenceLearningConfig, + ) + agent: AgentLearningConfig = field(default_factory=AgentLearningConfig) + metrics: MetricsConfig = field(default_factory=MetricsConfig) + + # Backward-compat properties for old flat field names + @property + def default_policy(self) -> str: + """Deprecated: use ``learning.routing.policy``.""" + return self.routing.policy + + @default_policy.setter + def default_policy(self, value: str) -> None: + self.routing.policy = value + + @property + def intelligence_policy(self) -> str: + """Deprecated: use ``learning.intelligence.policy``.""" + return self.intelligence.policy + + @intelligence_policy.setter + def intelligence_policy(self, value: str) -> None: + self.intelligence.policy = value + + @property + def agent_policy(self) -> str: + """Deprecated: use ``learning.agent.policy``.""" + return self.agent.policy + + @agent_policy.setter + def agent_policy(self, value: str) -> None: + self.agent.policy = value + + @property + def reward_weights(self) -> str: + """Deprecated: use ``learning.metrics.*``.""" + parts = [] + m = self.metrics + if m.latency_weight: + parts.append(f"latency={m.latency_weight}") + if m.cost_weight: + parts.append(f"cost={m.cost_weight}") + if m.efficiency_weight: + parts.append(f"efficiency={m.efficiency_weight}") + if m.accuracy_weight: + parts.append(f"accuracy={m.accuracy_weight}") + return ",".join(parts) + + @reward_weights.setter + def reward_weights(self, value: str) -> None: + if not value: + return + for part in value.split(","): + if "=" not in part: + continue + key, val = part.strip().split("=", 1) + key = key.strip() + fval = float(val.strip()) + if key == "accuracy": + self.metrics.accuracy_weight = fval + elif key == "latency": + self.metrics.latency_weight = fval + elif key == "cost": + self.metrics.cost_weight = fval + elif key == "efficiency": + self.metrics.efficiency_weight = fval @dataclass(slots=True) @@ -227,7 +412,6 @@ class StorageConfig: default_backend: str = "sqlite" db_path: str = str(DEFAULT_CONFIG_DIR / "memory.db") - context_injection: bool = True context_top_k: int = 5 context_min_score: float = 0.1 context_max_tokens: int = 2048 @@ -245,8 +429,6 @@ class MCPConfig: enabled: bool = True servers: str = "" # JSON list of MCP server configs - expose_storage: bool = True - expose_llm: bool = True @dataclass(slots=True) @@ -258,15 +440,27 @@ class ToolsConfig: enabled: str = "" # comma-separated default tools -@dataclass(slots=True) +@dataclass class AgentConfig: - """Agent defaults.""" + """Agent harness settings — orchestration, tools, system prompt.""" default_agent: str = "simple" - max_turns: int = 3 - default_tools: str = "" # comma-separated tool names - temperature: float = 0.7 - max_tokens: int = 1024 + max_turns: int = 10 + tools: str = "" # comma-separated tool names + objective: str = "" # concise purpose for routing/learning/docs + system_prompt: str = "" # inline system prompt (takes precedence if set) + system_prompt_path: str = "" # path to system prompt file (.txt, .md) + context_from_memory: bool = True # inject relevant memory context into prompts + + # Backward-compat property for old field name + @property + def default_tools(self) -> str: + """Deprecated: use ``agent.tools``.""" + return self.tools + + @default_tools.setter + def default_tools(self, value: str) -> None: + self.tools = value @dataclass(slots=True) @@ -495,10 +689,53 @@ class JarvisConfig: def _apply_toml_section(target: Any, section: Dict[str, Any]) -> None: - """Overlay TOML key/value pairs onto a dataclass instance.""" + """Overlay TOML key/value pairs onto a dataclass instance. + + Recursively handles nested dicts when the target attribute is itself + a dataclass. + """ for key, value in section.items(): if hasattr(target, key): - setattr(target, key, value) + if isinstance(value, dict): + nested = getattr(target, key) + if hasattr(nested, "__dataclass_fields__"): + _apply_toml_section(nested, value) + else: + setattr(target, key, value) + else: + setattr(target, key, value) + + +def _migrate_toml_data(data: Dict[str, Any], cfg: "JarvisConfig") -> None: + """Migrate old-format TOML keys to new structure in-place. + + Handles cross-section moves that can't be solved by backward-compat + properties alone (e.g. ``agent.temperature`` → ``intelligence.temperature``). + """ + # agent.temperature / agent.max_tokens → intelligence.* + if "agent" in data: + agent_data = data["agent"] + intel_data = data.setdefault("intelligence", {}) + for moved_key in ("temperature", "max_tokens"): + if moved_key in agent_data: + intel_data.setdefault(moved_key, agent_data.pop(moved_key)) + + # context_injection from memory / tools.storage → agent.context_from_memory + for src_section in ("memory",): + src = data.get(src_section, {}) + if isinstance(src, dict) and "context_injection" in src: + data.setdefault("agent", {}).setdefault( + "context_from_memory", src.pop("context_injection"), + ) + + if "tools" in data: + tools_data = data["tools"] + if isinstance(tools_data, dict): + storage_sub = tools_data.get("storage", {}) + if isinstance(storage_sub, dict) and "context_injection" in storage_sub: + data.setdefault("agent", {}).setdefault( + "context_from_memory", storage_sub.pop("context_injection"), + ) def load_config(path: Optional[Path] = None) -> JarvisConfig: @@ -518,49 +755,26 @@ def load_config(path: Optional[Path] = None) -> JarvisConfig: with open(config_path, "rb") as fh: data = tomllib.load(fh) - # Simple top-level sections - simple_sections = ( - "engine", "intelligence", "learning", - "agent", "server", "telemetry", "traces", "security", + # Run backward-compat migrations before applying + _migrate_toml_data(data, cfg) + + # All top-level sections — recursive _apply_toml_section handles + # nested sub-configs (engine.ollama, learning.routing, channel.*, etc.) + top_sections = ( + "engine", "intelligence", "learning", "agent", + "server", "telemetry", "traces", "security", + "channel", "tools", ) - for section_name in simple_sections: + for section_name in top_sections: if section_name in data: - _apply_toml_section(getattr(cfg, section_name), data[section_name]) + _apply_toml_section( + getattr(cfg, section_name), data[section_name], + ) # Memory: accept [memory] (old) → maps to tools.storage if "memory" in data: _apply_toml_section(cfg.tools.storage, data["memory"]) - # [channel] with nested per-channel sub-configs - if "channel" in data: - ch_data = data["channel"] - # Top-level channel keys (enabled, default_channel, etc.) - for key, value in ch_data.items(): - if not isinstance(value, dict) and hasattr(cfg.channel, key): - setattr(cfg.channel, key, value) - # Nested per-channel configs - for sub in ( - "telegram", "discord", "slack", "webhook", "email", - "whatsapp", "signal", "google_chat", "irc", "webchat", - "teams", "matrix", "mattermost", "feishu", "bluebubbles", - ): - if sub in ch_data and isinstance(ch_data[sub], dict): - _apply_toml_section(getattr(cfg.channel, sub), ch_data[sub]) - - # Tools: accept [tools] and nested [tools.storage], [tools.mcp] - if "tools" in data: - tools_data = data["tools"] - # Top-level tools keys (e.g. enabled) - for key, value in tools_data.items(): - if not isinstance(value, dict) and hasattr(cfg.tools, key): - setattr(cfg.tools, key, value) - # [tools.storage] - if "storage" in tools_data: - _apply_toml_section(cfg.tools.storage, tools_data["storage"]) - # [tools.mcp] - if "mcp" in tools_data: - _apply_toml_section(cfg.tools.mcp, tools_data["mcp"]) - return cfg @@ -585,20 +799,49 @@ def generate_default_toml(hw: HardwareInfo) -> str: [engine] default = "{engine}" -ollama_host = "http://localhost:11434" -vllm_host = "http://localhost:8000" -sglang_host = "http://localhost:30000" + +[engine.ollama] +host = "http://localhost:11434" + +[engine.vllm] +host = "http://localhost:8000" + +[engine.sglang] +host = "http://localhost:30000" + +# [engine.llamacpp] +# host = "http://localhost:8080" +# binary_path = "" [intelligence] default_model = "" fallback_model = "" - -[memory] -default_backend = "sqlite" +# model_path = "" # Local weights (HF repo, GGUF file, etc.) +# checkpoint_path = "" # Checkpoint/adapter path +# quantization = "none" # none, fp8, int8, int4, gguf_q4, gguf_q8 +# preferred_engine = "" # Override engine for this model (e.g., "vllm") +# provider = "" # local, openai, anthropic, google +temperature = 0.7 +max_tokens = 1024 +# top_p = 0.9 +# top_k = 40 +# repetition_penalty = 1.0 +# stop_sequences = "" [agent] default_agent = "simple" max_turns = 10 +# tools = "" # Comma-separated tool names +# objective = "" # Concise purpose string +# system_prompt = "" # Inline system prompt +# system_prompt_path = "" # Path to system prompt file +context_from_memory = true + +[tools.storage] +default_backend = "sqlite" + +[tools.mcp] +enabled = true [server] host = "0.0.0.0" @@ -606,11 +849,25 @@ port = 8000 agent = "orchestrator" [learning] -default_policy = "heuristic" -# intelligence_policy = "none" # "sft" to learn from traces -# agent_policy = "none" # "agent_advisor" for LM-guided restructuring -# tools_policy = "none" # "icl_updater" for ICL example + skill discovery -# update_interval = 10 +enabled = false +update_interval = 100 +# auto_update = false + +[learning.routing] +policy = "heuristic" +# min_samples = 5 + +# [learning.intelligence] +# policy = "none" # "sft" to learn from traces + +# [learning.agent] +# policy = "none" # "agent_advisor" | "icl_updater" + +# [learning.metrics] +# accuracy_weight = 0.6 +# latency_weight = 0.2 +# cost_weight = 0.1 +# efficiency_weight = 0.1 [telemetry] enabled = true @@ -684,6 +941,7 @@ enforce_tool_confirmation = true __all__ = [ "AgentConfig", + "AgentLearningConfig", "BlueBubblesChannelConfig", "ChannelConfig", "DEFAULT_CONFIG_DIR", @@ -697,12 +955,18 @@ __all__ = [ "HardwareInfo", "IRCChannelConfig", "IntelligenceConfig", + "IntelligenceLearningConfig", "JarvisConfig", "LearningConfig", + "LlamaCppEngineConfig", "MCPConfig", "MatrixChannelConfig", "MattermostChannelConfig", "MemoryConfig", + "MetricsConfig", + "OllamaEngineConfig", + "RoutingLearningConfig", + "SGLangEngineConfig", "SecurityConfig", "ServerConfig", "SignalChannelConfig", @@ -713,6 +977,7 @@ __all__ = [ "TelemetryConfig", "ToolsConfig", "TracesConfig", + "VLLMEngineConfig", "WebChatChannelConfig", "WebhookChannelConfig", "WhatsAppChannelConfig", diff --git a/src/openjarvis/intelligence/__init__.py b/src/openjarvis/intelligence/__init__.py index fbba9e86..7b25f8c5 100644 --- a/src/openjarvis/intelligence/__init__.py +++ b/src/openjarvis/intelligence/__init__.py @@ -1,4 +1,4 @@ -"""Intelligence pillar — model management and query routing.""" +"""Intelligence pillar — the model definition and catalog.""" from __future__ import annotations @@ -7,12 +7,5 @@ from openjarvis.intelligence.model_catalog import ( merge_discovered_models, register_builtin_models, ) -from openjarvis.intelligence.router import HeuristicRouter, build_routing_context -__all__ = [ - "BUILTIN_MODELS", - "HeuristicRouter", - "build_routing_context", - "merge_discovered_models", - "register_builtin_models", -] +__all__ = ["BUILTIN_MODELS", "merge_discovered_models", "register_builtin_models"] diff --git a/src/openjarvis/intelligence/_stubs.py b/src/openjarvis/intelligence/_stubs.py index 821185d5..9b729fd7 100644 --- a/src/openjarvis/intelligence/_stubs.py +++ b/src/openjarvis/intelligence/_stubs.py @@ -1,28 +1,5 @@ -"""Intelligence pillar ABCs — model routing and query analysis.""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from openjarvis.core.types import RoutingContext - - -class RouterPolicy(ABC): - """Abstract interface for model selection policies.""" - - @abstractmethod - def select_model(self, context: "RoutingContext") -> str: - """Select the best model key for the given routing context.""" - - -class QueryAnalyzer(ABC): - """Abstract interface for analyzing queries into routing contexts.""" - - @abstractmethod - def analyze(self, query: str, **kwargs: object) -> "RoutingContext": - """Analyze a query and return a RoutingContext.""" +"""Backward-compat shim — canonical location is learning._stubs.""" +from openjarvis.learning._stubs import QueryAnalyzer, RouterPolicy # noqa: F401 __all__ = ["QueryAnalyzer", "RouterPolicy"] diff --git a/src/openjarvis/intelligence/router.py b/src/openjarvis/intelligence/router.py index 6bedb81f..d57decfb 100644 --- a/src/openjarvis/intelligence/router.py +++ b/src/openjarvis/intelligence/router.py @@ -1,164 +1,9 @@ -"""Heuristic model router — selects the best model based on query characteristics.""" +"""Backward-compat shim — canonical location is learning.router.""" -from __future__ import annotations - -import re -from typing import List, Optional - -from openjarvis.core.registry import ModelRegistry -from openjarvis.core.types import RoutingContext -from openjarvis.intelligence._stubs import QueryAnalyzer, RouterPolicy - -# Detection patterns -_CODE_PATTERNS = re.compile( - r"```|`[^`]+`|\bdef\s|\bclass\s|\bimport\s|\bfunction\s|\bconst\s|\bvar\s|\blet\s|" - r"\bif\s*\(|->|=>|\{\s*\}|\bfor\s+\w+\s+in\s|#include|System\.out", - re.IGNORECASE, +from openjarvis.learning.router import ( # noqa: F401 + DefaultQueryAnalyzer, + HeuristicRouter, + build_routing_context, ) -_MATH_PATTERNS = re.compile( - r"\bsolve\b|\bintegral\b|\bequation\b|\bproof\b|\bderivative\b|\bmatrix\b|" - r"\btheorem\b|\bcalculate\b|\bcompute\b|\bsigma\b|\bsum\b|\blimit\b|\bprobability\b", - re.IGNORECASE, -) -_REASONING_KEYWORDS = re.compile( - r"\bexplain\b|\banalyze\b|\bcompare\b|\bwhy\b" - r"|\bstep[- ]by[- ]step\b|\breason\b|\bthink\b", - re.IGNORECASE, -) - - -def build_routing_context(query: str, *, urgency: float = 0.5) -> RoutingContext: - """Populate a ``RoutingContext`` from a raw query string.""" - return RoutingContext( - query=query, - query_length=len(query), - has_code=bool(_CODE_PATTERNS.search(query)), - has_math=bool(_MATH_PATTERNS.search(query)), - urgency=urgency, - ) - - -def _model_size(key: str) -> float: - """Return parameter count for a registered model, or 0 if not found.""" - try: - spec = ModelRegistry.get(key) - return spec.parameter_count_b - except (KeyError, AttributeError): - return 0.0 - - -def _find_model_by_tag(available: List[str], tag: str) -> Optional[str]: - """Find the first available model whose key contains *tag* (case-insensitive).""" - tag_lower = tag.lower() - for key in available: - if tag_lower in key.lower(): - return key - return None - - -def _largest_model(available: List[str]) -> Optional[str]: - """Return the model with the largest parameter count from the available list.""" - if not available: - return None - best = available[0] - best_size = _model_size(best) - for key in available[1:]: - size = _model_size(key) - if size > best_size: - best = key - best_size = size - return best - - -def _smallest_model(available: List[str]) -> Optional[str]: - """Return the smallest-parameter model from *available*.""" - if not available: - return None - best = available[0] - best_size = _model_size(best) or float("inf") - for key in available[1:]: - size = _model_size(key) - if 0 < size < best_size: - best = key - best_size = size - return best - - -class HeuristicRouter(RouterPolicy): - """Rule-based model router. - - Rules (applied in order): - 1. Code detected → prefer model with "code"/"coder" in name - 2. Math detected → prefer larger model - 3. Short query (<50 chars, no code/math) → prefer smaller/faster model - 4. Long/complex query (>500 chars OR reasoning keywords) → prefer larger model - 5. High urgency (>0.8) → override to smaller model - 6. Default fallback → default_model → fallback_model → first available - """ - - def __init__( - self, - available_models: List[str] | None = None, - *, - default_model: str = "", - fallback_model: str = "", - ) -> None: - self._available = available_models or [] - self._default = default_model - self._fallback = fallback_model - - @property - def available_models(self) -> List[str]: - return list(self._available) - - def select_model(self, context: RoutingContext) -> str: - available = self._available or list(ModelRegistry.keys()) - if not available: - return self._default or self._fallback or "" - - # Rule 5: High urgency overrides everything → smallest model - if context.urgency > 0.8: - return _smallest_model(available) or available[0] - - # Rule 1: Code detected → prefer model with code/coder in name - if context.has_code: - code_model = ( - _find_model_by_tag(available, "code") - or _find_model_by_tag(available, "coder") - ) - if code_model: - return code_model - # Fall through to larger model for code - return _largest_model(available) or available[0] - - # Rule 2: Math detected → prefer larger model - if context.has_math: - return _largest_model(available) or available[0] - - # Rule 3: Short simple query → prefer smaller model - if context.query_length < 50: - return _smallest_model(available) or available[0] - - # Rule 4: Long/complex query → prefer larger model - if context.query_length > 500 or _REASONING_KEYWORDS.search(context.query): - return _largest_model(available) or available[0] - - # Rule 6: Default fallback - if self._default and self._default in available: - return self._default - if self._fallback and self._fallback in available: - return self._fallback - return available[0] - - -class DefaultQueryAnalyzer(QueryAnalyzer): - """Default query analyzer wrapping the heuristic build_routing_context function.""" - - def analyze(self, query: str, **kwargs: object) -> RoutingContext: - urgency = kwargs.get("urgency", 0.5) - if not isinstance(urgency, (int, float)): - urgency = 0.5 - return build_routing_context(query, urgency=urgency) - __all__ = ["DefaultQueryAnalyzer", "HeuristicRouter", "build_routing_context"] diff --git a/src/openjarvis/learning/__init__.py b/src/openjarvis/learning/__init__.py index 3eb5b469..c1d5668d 100644 --- a/src/openjarvis/learning/__init__.py +++ b/src/openjarvis/learning/__init__.py @@ -1,9 +1,18 @@ -"""Learning pillar — router policy and reward functions.""" +"""Learning pillar — router policies, reward functions, and trace-driven learning.""" from __future__ import annotations -from openjarvis.learning._stubs import RewardFunction, RouterPolicy, RoutingContext +from openjarvis.learning._stubs import ( + QueryAnalyzer, + RewardFunction, + RouterPolicy, + RoutingContext, +) from openjarvis.learning.heuristic_reward import HeuristicRewardFunction +from openjarvis.learning.router import ( + HeuristicRouter, + build_routing_context, +) def ensure_registered() -> None: @@ -56,8 +65,11 @@ def ensure_registered() -> None: __all__ = [ "HeuristicRewardFunction", + "HeuristicRouter", + "QueryAnalyzer", "RewardFunction", "RouterPolicy", "RoutingContext", + "build_routing_context", "ensure_registered", ] diff --git a/src/openjarvis/learning/_stubs.py b/src/openjarvis/learning/_stubs.py index ecb1efbe..4a7ef29d 100644 --- a/src/openjarvis/learning/_stubs.py +++ b/src/openjarvis/learning/_stubs.py @@ -1,4 +1,4 @@ -"""Learning pillar ABCs and re-exports for backward compatibility.""" +"""Learning pillar ABCs — router policies, reward functions, and learning policies.""" from __future__ import annotations @@ -7,14 +7,29 @@ from typing import TYPE_CHECKING, Any, ClassVar, Dict from openjarvis.core.registry import LearningRegistry # noqa: F401 -# Re-export from canonical locations for backward compatibility +# Re-export from canonical location for backward compatibility from openjarvis.core.types import RoutingContext # noqa: F401 -from openjarvis.intelligence._stubs import RouterPolicy # noqa: F401 if TYPE_CHECKING: pass +class RouterPolicy(ABC): + """Model selection policy (used by the learning system).""" + + @abstractmethod + def select_model(self, context: "RoutingContext") -> str: + """Select the best model key for the given routing context.""" + + +class QueryAnalyzer(ABC): + """Query analysis for routing contexts.""" + + @abstractmethod + def analyze(self, query: str, **kwargs: object) -> "RoutingContext": + """Analyze a query and return a RoutingContext.""" + + class RewardFunction(ABC): """Compute a scalar reward for a routing decision.""" @@ -56,6 +71,7 @@ __all__ = [ "IntelligenceLearningPolicy", "LearningPolicy", "LearningRegistry", + "QueryAnalyzer", "RewardFunction", "RouterPolicy", "RoutingContext", diff --git a/src/openjarvis/learning/grpo_policy.py b/src/openjarvis/learning/grpo_policy.py index d3847511..bc2535a6 100644 --- a/src/openjarvis/learning/grpo_policy.py +++ b/src/openjarvis/learning/grpo_policy.py @@ -6,7 +6,7 @@ from typing import Any from openjarvis.core.registry import RouterPolicyRegistry from openjarvis.core.types import RoutingContext -from openjarvis.intelligence._stubs import RouterPolicy +from openjarvis.learning._stubs import RouterPolicy class GRPORouterPolicy(RouterPolicy): diff --git a/src/openjarvis/learning/heuristic_policy.py b/src/openjarvis/learning/heuristic_policy.py index 60a89956..8614b411 100644 --- a/src/openjarvis/learning/heuristic_policy.py +++ b/src/openjarvis/learning/heuristic_policy.py @@ -3,7 +3,7 @@ from __future__ import annotations from openjarvis.core.registry import RouterPolicyRegistry -from openjarvis.intelligence.router import HeuristicRouter +from openjarvis.learning.router import HeuristicRouter def ensure_registered() -> None: diff --git a/src/openjarvis/learning/router.py b/src/openjarvis/learning/router.py new file mode 100644 index 00000000..839982a1 --- /dev/null +++ b/src/openjarvis/learning/router.py @@ -0,0 +1,164 @@ +"""Heuristic model router — selects the best model based on query characteristics.""" + +from __future__ import annotations + +import re +from typing import List, Optional + +from openjarvis.core.registry import ModelRegistry +from openjarvis.core.types import RoutingContext +from openjarvis.learning._stubs import QueryAnalyzer, RouterPolicy + +# Detection patterns +_CODE_PATTERNS = re.compile( + r"```|`[^`]+`|\bdef\s|\bclass\s|\bimport\s|\bfunction\s|\bconst\s|\bvar\s|\blet\s|" + r"\bif\s*\(|->|=>|\{\s*\}|\bfor\s+\w+\s+in\s|#include|System\.out", + re.IGNORECASE, +) +_MATH_PATTERNS = re.compile( + r"\bsolve\b|\bintegral\b|\bequation\b|\bproof\b|\bderivative\b|\bmatrix\b|" + r"\btheorem\b|\bcalculate\b|\bcompute\b|\bsigma\b|\bsum\b|\blimit\b|\bprobability\b", + re.IGNORECASE, +) +_REASONING_KEYWORDS = re.compile( + r"\bexplain\b|\banalyze\b|\bcompare\b|\bwhy\b" + r"|\bstep[- ]by[- ]step\b|\breason\b|\bthink\b", + re.IGNORECASE, +) + + +def build_routing_context(query: str, *, urgency: float = 0.5) -> RoutingContext: + """Populate a ``RoutingContext`` from a raw query string.""" + return RoutingContext( + query=query, + query_length=len(query), + has_code=bool(_CODE_PATTERNS.search(query)), + has_math=bool(_MATH_PATTERNS.search(query)), + urgency=urgency, + ) + + +def _model_size(key: str) -> float: + """Return parameter count for a registered model, or 0 if not found.""" + try: + spec = ModelRegistry.get(key) + return spec.parameter_count_b + except (KeyError, AttributeError): + return 0.0 + + +def _find_model_by_tag(available: List[str], tag: str) -> Optional[str]: + """Find the first available model whose key contains *tag* (case-insensitive).""" + tag_lower = tag.lower() + for key in available: + if tag_lower in key.lower(): + return key + return None + + +def _largest_model(available: List[str]) -> Optional[str]: + """Return the model with the largest parameter count from the available list.""" + if not available: + return None + best = available[0] + best_size = _model_size(best) + for key in available[1:]: + size = _model_size(key) + if size > best_size: + best = key + best_size = size + return best + + +def _smallest_model(available: List[str]) -> Optional[str]: + """Return the smallest-parameter model from *available*.""" + if not available: + return None + best = available[0] + best_size = _model_size(best) or float("inf") + for key in available[1:]: + size = _model_size(key) + if 0 < size < best_size: + best = key + best_size = size + return best + + +class HeuristicRouter(RouterPolicy): + """Rule-based model router. + + Rules (applied in order): + 1. Code detected → prefer model with "code"/"coder" in name + 2. Math detected → prefer larger model + 3. Short query (<50 chars, no code/math) → prefer smaller/faster model + 4. Long/complex query (>500 chars OR reasoning keywords) → prefer larger model + 5. High urgency (>0.8) → override to smaller model + 6. Default fallback → default_model → fallback_model → first available + """ + + def __init__( + self, + available_models: List[str] | None = None, + *, + default_model: str = "", + fallback_model: str = "", + ) -> None: + self._available = available_models or [] + self._default = default_model + self._fallback = fallback_model + + @property + def available_models(self) -> List[str]: + return list(self._available) + + def select_model(self, context: RoutingContext) -> str: + available = self._available or list(ModelRegistry.keys()) + if not available: + return self._default or self._fallback or "" + + # Rule 5: High urgency overrides everything → smallest model + if context.urgency > 0.8: + return _smallest_model(available) or available[0] + + # Rule 1: Code detected → prefer model with code/coder in name + if context.has_code: + code_model = ( + _find_model_by_tag(available, "code") + or _find_model_by_tag(available, "coder") + ) + if code_model: + return code_model + # Fall through to larger model for code + return _largest_model(available) or available[0] + + # Rule 2: Math detected → prefer larger model + if context.has_math: + return _largest_model(available) or available[0] + + # Rule 3: Short simple query → prefer smaller model + if context.query_length < 50: + return _smallest_model(available) or available[0] + + # Rule 4: Long/complex query → prefer larger model + if context.query_length > 500 or _REASONING_KEYWORDS.search(context.query): + return _largest_model(available) or available[0] + + # Rule 6: Default fallback + if self._default and self._default in available: + return self._default + if self._fallback and self._fallback in available: + return self._fallback + return available[0] + + +class DefaultQueryAnalyzer(QueryAnalyzer): + """Default query analyzer wrapping the heuristic build_routing_context function.""" + + def analyze(self, query: str, **kwargs: object) -> RoutingContext: + urgency = kwargs.get("urgency", 0.5) + if not isinstance(urgency, (int, float)): + urgency = 0.5 + return build_routing_context(query, urgency=urgency) + + +__all__ = ["DefaultQueryAnalyzer", "HeuristicRouter", "build_routing_context"] diff --git a/src/openjarvis/learning/trace_policy.py b/src/openjarvis/learning/trace_policy.py index 2bec3b18..f7b60beb 100644 --- a/src/openjarvis/learning/trace_policy.py +++ b/src/openjarvis/learning/trace_policy.py @@ -14,7 +14,7 @@ from typing import Any, Dict, List, Optional from openjarvis.core.registry import RouterPolicyRegistry from openjarvis.core.types import RoutingContext -from openjarvis.intelligence._stubs import RouterPolicy +from openjarvis.learning._stubs import RouterPolicy from openjarvis.traces.analyzer import TraceAnalyzer # Query classification for grouping traces diff --git a/src/openjarvis/sdk.py b/src/openjarvis/sdk.py index 3acac3ee..56942df1 100644 --- a/src/openjarvis/sdk.py +++ b/src/openjarvis/sdk.py @@ -187,7 +187,9 @@ class Jarvis: except ImportError: pass - resolved = get_engine(self._config, self._engine_key) + pref = self._config.intelligence.preferred_engine + engine_key = self._engine_key or pref or None + resolved = get_engine(self._config, engine_key) if resolved is None: raise RuntimeError( "No inference engine available. " @@ -229,8 +231,8 @@ class Jarvis: model: Optional[str] = None, agent: Optional[str] = None, tools: Optional[List[str]] = None, - temperature: float = 0.7, - max_tokens: int = 1024, + temperature: Optional[float] = None, + max_tokens: Optional[int] = None, context: bool = True, ) -> str: """Send a query and return the response text.""" @@ -252,8 +254,8 @@ class Jarvis: model: Optional[str] = None, agent: Optional[str] = None, tools: Optional[List[str]] = None, - temperature: float = 0.7, - max_tokens: int = 1024, + temperature: Optional[float] = None, + max_tokens: Optional[int] = None, context: bool = True, ) -> Dict[str, Any]: """Send a query and return the full result dict. @@ -261,6 +263,11 @@ class Jarvis: Returns a dict with keys: content, usage, tool_results (if agent mode). """ self._ensure_engine() + if temperature is None: + temperature = self._config.intelligence.temperature + if max_tokens is None: + max_tokens = self._config.intelligence.max_tokens + model_name = model or self._model_override # Resolve model via router if not specified @@ -285,7 +292,7 @@ class Jarvis: messages = [Message(role=Role.USER, content=query)] # Memory context injection - if context and self._config.memory.context_injection: + if context and self._config.agent.context_from_memory: messages = self._inject_context(query, messages) result = instrumented_generate( @@ -364,7 +371,7 @@ class Jarvis: ctx = AgentContext() # Context injection - if context and self._config.memory.context_injection: + if context and self._config.agent.context_from_memory: try: from openjarvis.cli.ask import _get_memory_backend from openjarvis.memory.context import ContextConfig, inject_context diff --git a/src/openjarvis/system.py b/src/openjarvis/system.py index 74f620f9..9b468bdc 100644 --- a/src/openjarvis/system.py +++ b/src/openjarvis/system.py @@ -38,16 +38,21 @@ class JarvisSystem: query: str, *, context: bool = True, - temperature: float = 0.7, - max_tokens: int = 1024, + temperature: Optional[float] = None, + max_tokens: Optional[int] = None, agent: Optional[str] = None, tools: Optional[List[str]] = None, ) -> Dict[str, Any]: """Execute a query through the system and return a result dict.""" + if temperature is None: + temperature = self.config.intelligence.temperature + if max_tokens is None: + max_tokens = self.config.intelligence.max_tokens + messages = [Message(role=Role.USER, content=query)] # Context injection from memory - if context and self.memory_backend and self.config.memory.context_injection: + if context and self.memory_backend and self.config.agent.context_from_memory: try: from openjarvis.tools.storage.context import ( ContextConfig, @@ -299,7 +304,8 @@ class SystemBuilder: """Resolve the inference engine.""" from openjarvis.engine._discovery import get_engine - key = self._engine_key or config.engine.default + pref = config.intelligence.preferred_engine + key = self._engine_key or pref or config.engine.default resolved = get_engine(config, key) if resolved is None: raise RuntimeError( @@ -511,7 +517,7 @@ class SystemBuilder: # 3. Determine which tool names to include tool_names = self._tool_names if tool_names is None: - raw = config.tools.enabled or config.agent.default_tools + raw = config.tools.enabled or config.agent.tools if raw: tool_names = [n.strip() for n in raw.split(",") if n.strip()] else: diff --git a/tests/cli/test_ask_router.py b/tests/cli/test_ask_router.py index e9581a04..22f5b2e4 100644 --- a/tests/cli/test_ask_router.py +++ b/tests/cli/test_ask_router.py @@ -83,7 +83,7 @@ class TestAskModelResolution: cfg.telemetry.enabled = False cfg.intelligence.default_model = "" cfg.intelligence.fallback_model = "" - cfg.memory.context_injection = False + cfg.agent.context_from_memory = False result = CliRunner().invoke(cli, ["ask", "Hello"]) assert result.exit_code == 0 @@ -107,6 +107,6 @@ class TestAskModelResolution: cfg.telemetry.enabled = False cfg.intelligence.default_model = "" cfg.intelligence.fallback_model = "fallback-model" - cfg.memory.context_injection = False + cfg.agent.context_from_memory = False result = CliRunner().invoke(cli, ["ask", "Hello"]) assert result.exit_code == 0 diff --git a/tests/core/test_config.py b/tests/core/test_config.py index e96805e9..f6181691 100644 --- a/tests/core/test_config.py +++ b/tests/core/test_config.py @@ -5,11 +5,14 @@ from __future__ import annotations from pathlib import Path from openjarvis.core.config import ( + AgentConfig, ChannelConfig, EngineConfig, GpuInfo, HardwareInfo, + IntelligenceConfig, JarvisConfig, + LearningConfig, SecurityConfig, generate_default_toml, load_config, @@ -26,6 +29,13 @@ class TestDefaults: def test_engine_config_defaults(self) -> None: ec = EngineConfig() + # Nested configs + assert ec.ollama.host == "http://localhost:11434" + assert ec.vllm.host == "http://localhost:8000" + assert ec.sglang.host == "http://localhost:30000" + assert ec.llamacpp.host == "http://localhost:8080" + assert ec.llamacpp.binary_path == "" + # Backward-compat properties still work assert ec.ollama_host == "http://localhost:11434" assert ec.vllm_host == "http://localhost:8000" @@ -145,3 +155,208 @@ class TestChannelConfig: def test_channel_config_in_default_toml(self) -> None: output = generate_default_toml(HardwareInfo()) assert "[channel]" in output + + +# --------------------------------------------------------------------------- +# New config structure tests +# --------------------------------------------------------------------------- + + +class TestIntelligenceGenerationDefaults: + def test_generation_fields_exist(self) -> None: + ic = IntelligenceConfig() + assert ic.temperature == 0.7 + assert ic.max_tokens == 1024 + assert ic.top_p == 0.9 + assert ic.top_k == 40 + assert ic.repetition_penalty == 1.0 + assert ic.stop_sequences == "" + + def test_custom_generation_values(self) -> None: + ic = IntelligenceConfig(temperature=0.3, max_tokens=512, top_p=0.5) + assert ic.temperature == 0.3 + assert ic.max_tokens == 512 + assert ic.top_p == 0.5 + + +class TestAgentConfigNew: + def test_new_fields(self) -> None: + ac = AgentConfig() + assert ac.objective == "" + assert ac.system_prompt == "" + assert ac.system_prompt_path == "" + assert ac.context_from_memory is True + assert ac.tools == "" + assert ac.max_turns == 10 + + def test_default_tools_backward_compat(self) -> None: + ac = AgentConfig() + ac.default_tools = "calculator,think" + assert ac.tools == "calculator,think" + assert ac.default_tools == "calculator,think" + + def test_no_temperature_or_max_tokens(self) -> None: + ac = AgentConfig() + assert not hasattr(ac.__class__, "temperature") or isinstance( + getattr(ac.__class__, "temperature", None), property + ) is False + + +class TestNestedEngineConfig: + def test_nested_access(self) -> None: + ec = EngineConfig() + assert ec.ollama.host == "http://localhost:11434" + assert ec.vllm.host == "http://localhost:8000" + assert ec.sglang.host == "http://localhost:30000" + assert ec.llamacpp.host == "http://localhost:8080" + assert ec.llamacpp.binary_path == "" + + def test_backward_compat_setter(self) -> None: + ec = EngineConfig() + ec.ollama_host = "http://custom:1234" + assert ec.ollama.host == "http://custom:1234" + assert ec.ollama_host == "http://custom:1234" + + def test_llamacpp_path_compat(self) -> None: + ec = EngineConfig() + ec.llamacpp_path = "/usr/bin/llama-server" + assert ec.llamacpp.binary_path == "/usr/bin/llama-server" + + def test_loads_nested_toml(self, tmp_path: Path) -> None: + toml_file = tmp_path / "config.toml" + toml_file.write_text( + '[engine]\ndefault = "vllm"\n\n' + '[engine.ollama]\nhost = "http://custom:11434"\n\n' + '[engine.llamacpp]\nbinary_path = "/opt/llama"\n' + ) + cfg = load_config(toml_file) + assert cfg.engine.default == "vllm" + assert cfg.engine.ollama.host == "http://custom:11434" + assert cfg.engine.llamacpp.binary_path == "/opt/llama" + + def test_loads_old_flat_toml(self, tmp_path: Path) -> None: + """Old flat engine keys still work via backward-compat properties.""" + toml_file = tmp_path / "config.toml" + toml_file.write_text( + '[engine]\ndefault = "ollama"\n' + 'ollama_host = "http://old:11434"\n' + 'vllm_host = "http://old:8000"\n' + ) + cfg = load_config(toml_file) + assert cfg.engine.ollama.host == "http://old:11434" + assert cfg.engine.vllm.host == "http://old:8000" + + +class TestNestedLearningConfig: + def test_defaults(self) -> None: + lc = LearningConfig() + assert lc.enabled is False + assert lc.update_interval == 100 + assert lc.auto_update is False + assert lc.routing.policy == "heuristic" + assert lc.routing.min_samples == 5 + assert lc.intelligence.policy == "none" + assert lc.agent.policy == "none" + assert lc.agent.max_icl_examples == 20 + assert lc.metrics.accuracy_weight == 0.6 + assert lc.metrics.latency_weight == 0.2 + + def test_backward_compat_default_policy(self) -> None: + lc = LearningConfig() + assert lc.default_policy == "heuristic" + lc.default_policy = "grpo" + assert lc.routing.policy == "grpo" + + def test_backward_compat_intelligence_policy(self) -> None: + lc = LearningConfig() + lc.intelligence_policy = "sft" + assert lc.intelligence.policy == "sft" + + def test_backward_compat_agent_policy(self) -> None: + lc = LearningConfig() + lc.agent_policy = "agent_advisor" + assert lc.agent.policy == "agent_advisor" + + def test_backward_compat_reward_weights(self) -> None: + lc = LearningConfig() + lc.reward_weights = "latency=0.4,cost=0.3" + assert lc.metrics.latency_weight == 0.4 + assert lc.metrics.cost_weight == 0.3 + + def test_loads_nested_toml(self, tmp_path: Path) -> None: + toml_file = tmp_path / "config.toml" + toml_file.write_text( + '[learning]\nenabled = true\nupdate_interval = 50\n\n' + '[learning.routing]\npolicy = "learned"\n\n' + '[learning.metrics]\nlatency_weight = 0.5\n' + ) + cfg = load_config(toml_file) + assert cfg.learning.enabled is True + assert cfg.learning.update_interval == 50 + assert cfg.learning.routing.policy == "learned" + assert cfg.learning.metrics.latency_weight == 0.5 + + def test_loads_old_flat_learning_toml(self, tmp_path: Path) -> None: + """Old flat learning keys still work via backward-compat properties.""" + toml_file = tmp_path / "config.toml" + toml_file.write_text( + '[learning]\ndefault_policy = "grpo"\n' + 'intelligence_policy = "sft"\n' + 'reward_weights = "latency=0.5"\n' + ) + cfg = load_config(toml_file) + assert cfg.learning.routing.policy == "grpo" + assert cfg.learning.intelligence.policy == "sft" + assert cfg.learning.metrics.latency_weight == 0.5 + + +class TestBackwardCompatMigration: + def test_agent_temperature_migrates_to_intelligence(self, tmp_path: Path) -> None: + toml_file = tmp_path / "config.toml" + toml_file.write_text( + '[agent]\ntemperature = 0.3\nmax_tokens = 512\n' + ) + cfg = load_config(toml_file) + assert cfg.intelligence.temperature == 0.3 + assert cfg.intelligence.max_tokens == 512 + + def test_memory_context_injection_migrates_to_agent(self, tmp_path: Path) -> None: + toml_file = tmp_path / "config.toml" + toml_file.write_text('[memory]\ncontext_injection = false\n') + cfg = load_config(toml_file) + assert cfg.agent.context_from_memory is False + + def test_tools_storage_context_injection_migrates(self, tmp_path: Path) -> None: + toml_file = tmp_path / "config.toml" + toml_file.write_text( + '[tools.storage]\ncontext_injection = false\ndefault_backend = "faiss"\n' + ) + cfg = load_config(toml_file) + assert cfg.agent.context_from_memory is False + assert cfg.tools.storage.default_backend == "faiss" + + +class TestGenerateDefaultTomlNew: + def test_nested_engine_sections(self) -> None: + hw = HardwareInfo() + toml_str = generate_default_toml(hw) + assert "[engine.ollama]" in toml_str + assert "[engine.vllm]" in toml_str + assert "[engine.sglang]" in toml_str + + def test_intelligence_generation_params(self) -> None: + hw = HardwareInfo() + toml_str = generate_default_toml(hw) + assert "temperature = 0.7" in toml_str + assert "max_tokens = 1024" in toml_str + + def test_agent_new_fields(self) -> None: + hw = HardwareInfo() + toml_str = generate_default_toml(hw) + assert "context_from_memory = true" in toml_str + + def test_learning_nested_sections(self) -> None: + hw = HardwareInfo() + toml_str = generate_default_toml(hw) + assert "[learning.routing]" in toml_str + assert 'policy = "heuristic"' in toml_str diff --git a/tests/core/test_config_phase3.py b/tests/core/test_config_phase3.py index 603fee47..b2b0cb17 100644 --- a/tests/core/test_config_phase3.py +++ b/tests/core/test_config_phase3.py @@ -15,21 +15,23 @@ class TestAgentConfig: def test_defaults(self): cfg = AgentConfig() assert cfg.default_agent == "simple" - assert cfg.max_turns == 3 - assert cfg.default_tools == "" - assert cfg.temperature == 0.7 - assert cfg.max_tokens == 1024 + assert cfg.max_turns == 10 + assert cfg.tools == "" + assert cfg.default_tools == "" # backward-compat property + assert cfg.objective == "" + assert cfg.system_prompt == "" + assert cfg.system_prompt_path == "" + assert cfg.context_from_memory is True def test_custom_values(self): cfg = AgentConfig( default_agent="orchestrator", max_turns=5, - default_tools="calculator,think", - temperature=0.1, - max_tokens=512, + tools="calculator,think", ) assert cfg.default_agent == "orchestrator" - assert cfg.default_tools == "calculator,think" + assert cfg.tools == "calculator,think" + assert cfg.default_tools == "calculator,think" # backward-compat class TestServerConfig: @@ -55,9 +57,11 @@ class TestJarvisConfig: def test_agent_config_expanded(self): cfg = JarvisConfig() - assert hasattr(cfg.agent, "default_tools") - assert hasattr(cfg.agent, "temperature") - assert hasattr(cfg.agent, "max_tokens") + assert hasattr(cfg.agent, "default_tools") # backward-compat property + assert hasattr(cfg.agent, "tools") + assert hasattr(cfg.agent, "objective") + assert hasattr(cfg.agent, "system_prompt") + assert hasattr(cfg.agent, "context_from_memory") class TestGenerateDefaultToml: diff --git a/tests/core/test_config_phase4.py b/tests/core/test_config_phase4.py index 930e4154..c69c266f 100644 --- a/tests/core/test_config_phase4.py +++ b/tests/core/test_config_phase4.py @@ -16,22 +16,31 @@ from openjarvis.core.config import ( class TestLearningConfig: def test_defaults(self) -> None: cfg = LearningConfig() + assert cfg.enabled is False + assert cfg.update_interval == 100 + assert cfg.auto_update is False + assert cfg.routing.policy == "heuristic" + assert cfg.intelligence.policy == "none" + assert cfg.agent.policy == "none" + assert cfg.metrics.accuracy_weight == 0.6 + # Backward-compat properties assert cfg.default_policy == "heuristic" - assert cfg.reward_weights == "" - def test_custom_values(self) -> None: - cfg = LearningConfig( - default_policy="grpo", - reward_weights="latency=0.4,cost=0.3,efficiency=0.3", - ) - assert cfg.default_policy == "grpo" - assert cfg.reward_weights == "latency=0.4,cost=0.3,efficiency=0.3" + def test_backward_compat_custom_values(self) -> None: + cfg = LearningConfig() + cfg.default_policy = "grpo" + cfg.reward_weights = "latency=0.4,cost=0.3,efficiency=0.3" + assert cfg.routing.policy == "grpo" + assert cfg.metrics.latency_weight == 0.4 + assert cfg.metrics.cost_weight == 0.3 + assert cfg.metrics.efficiency_weight == 0.3 def test_jarvis_config_has_learning(self) -> None: cfg = JarvisConfig() assert hasattr(cfg, "learning") assert isinstance(cfg.learning, LearningConfig) - assert cfg.learning.default_policy == "heuristic" + assert cfg.learning.routing.policy == "heuristic" + assert cfg.learning.default_policy == "heuristic" # backward-compat def test_toml_loading_with_learning(self, tmp_path: Path) -> None: toml_file = tmp_path / "config.toml" @@ -40,17 +49,30 @@ class TestLearningConfig: 'reward_weights = "latency=0.5"\n' ) cfg = load_config(toml_file) - assert cfg.learning.default_policy == "grpo" - assert cfg.learning.reward_weights == "latency=0.5" + assert cfg.learning.routing.policy == "grpo" + assert cfg.learning.metrics.latency_weight == 0.5 + + def test_toml_loading_nested(self, tmp_path: Path) -> None: + toml_file = tmp_path / "config.toml" + toml_file.write_text( + '[learning]\nenabled = true\n\n' + '[learning.routing]\npolicy = "learned"\n\n' + '[learning.metrics]\nlatency_weight = 0.5\n' + ) + cfg = load_config(toml_file) + assert cfg.learning.enabled is True + assert cfg.learning.routing.policy == "learned" + assert cfg.learning.metrics.latency_weight == 0.5 def test_toml_loading_without_learning(self, tmp_path: Path) -> None: toml_file = tmp_path / "config.toml" toml_file.write_text("[engine]\n") cfg = load_config(toml_file) - assert cfg.learning.default_policy == "heuristic" + assert cfg.learning.routing.policy == "heuristic" def test_generate_default_toml_includes_learning(self) -> None: hw = HardwareInfo() toml_str = generate_default_toml(hw) assert "[learning]" in toml_str - assert 'default_policy = "heuristic"' in toml_str + assert "[learning.routing]" in toml_str + assert 'policy = "heuristic"' in toml_str diff --git a/tests/intelligence/test_intelligence_stubs.py b/tests/intelligence/test_intelligence_stubs.py index ada03fe4..18807e66 100644 --- a/tests/intelligence/test_intelligence_stubs.py +++ b/tests/intelligence/test_intelligence_stubs.py @@ -1,70 +1,58 @@ -"""Tests for Intelligence pillar ABCs.""" +"""Tests for Intelligence pillar backward-compat shims.""" from __future__ import annotations import pytest -from openjarvis.core.types import RoutingContext -from openjarvis.intelligence._stubs import QueryAnalyzer, RouterPolicy -from openjarvis.intelligence.router import DefaultQueryAnalyzer +class TestBackwardCompatShims: + """Verify ABCs are still importable from intelligence._stubs.""" -class _DummyRouter(RouterPolicy): - def select_model(self, context: RoutingContext) -> str: - return "test-model" + def test_router_policy_from_intelligence(self) -> None: + from openjarvis.intelligence._stubs import RouterPolicy - -class _DummyAnalyzer(QueryAnalyzer): - def analyze(self, query: str, **kwargs: object) -> RoutingContext: - return RoutingContext(query=query, query_length=len(query)) - - -class TestRouterPolicy: - def test_abc_cannot_instantiate(self) -> None: with pytest.raises(TypeError): RouterPolicy() # type: ignore[abstract] - def test_concrete_implementation(self) -> None: - router = _DummyRouter() - ctx = RoutingContext(query="hello") - assert router.select_model(ctx) == "test-model" + def test_query_analyzer_from_intelligence(self) -> None: + from openjarvis.intelligence._stubs import QueryAnalyzer - -class TestQueryAnalyzer: - def test_abc_cannot_instantiate(self) -> None: with pytest.raises(TypeError): QueryAnalyzer() # type: ignore[abstract] - def test_concrete_implementation(self) -> None: - analyzer = _DummyAnalyzer() - ctx = analyzer.analyze("hello world") - assert ctx.query == "hello world" - assert ctx.query_length == 11 + def test_same_class_as_learning(self) -> None: + from openjarvis.intelligence._stubs import QueryAnalyzer as IQA + from openjarvis.intelligence._stubs import RouterPolicy as IRP + from openjarvis.learning._stubs import QueryAnalyzer as LQA + from openjarvis.learning._stubs import RouterPolicy as LRP + assert IRP is LRP + assert IQA is LQA + + def test_router_from_intelligence_module(self) -> None: + """HeuristicRouter still importable from intelligence.router.""" + from openjarvis.intelligence.router import ( + HeuristicRouter, + build_routing_context, + ) + + ctx = build_routing_context("hello") + assert ctx.query == "hello" + router = HeuristicRouter( + available_models=[], default_model="m", + ) + assert router.select_model(ctx) == "m" + + def test_default_query_analyzer_from_intelligence( + self, + ) -> None: + from openjarvis.intelligence.router import ( + DefaultQueryAnalyzer, + ) -class TestDefaultQueryAnalyzer: - def test_analyze_basic(self) -> None: analyzer = DefaultQueryAnalyzer() ctx = analyzer.analyze("Hello world") assert ctx.query == "Hello world" - assert ctx.query_length == 11 - assert ctx.has_code is False - assert ctx.has_math is False - - def test_analyze_code_query(self) -> None: - analyzer = DefaultQueryAnalyzer() - ctx = analyzer.analyze("def hello(): pass") - assert ctx.has_code is True - - def test_analyze_math_query(self) -> None: - analyzer = DefaultQueryAnalyzer() - ctx = analyzer.analyze("solve the integral of x^2") - assert ctx.has_math is True - - def test_analyze_with_urgency(self) -> None: - analyzer = DefaultQueryAnalyzer() - ctx = analyzer.analyze("quick question", urgency=0.9) - assert ctx.urgency == 0.9 class TestRoutingContextInCoreTypes: @@ -72,17 +60,19 @@ class TestRoutingContextInCoreTypes: def test_import_from_core_types(self) -> None: from openjarvis.core.types import RoutingContext as RC + ctx = RC(query="test", has_code=True) assert ctx.query == "test" assert ctx.has_code is True def test_backward_compat_import(self) -> None: - """RoutingContext should still be importable from learning._stubs.""" from openjarvis.learning._stubs import RoutingContext as RC + ctx = RC(query="compat") assert ctx.query == "compat" def test_router_policy_backward_compat(self) -> None: - """RouterPolicy should still be importable from learning._stubs.""" + from openjarvis.intelligence._stubs import RouterPolicy from openjarvis.learning._stubs import RouterPolicy as RP + assert RP is RouterPolicy diff --git a/tests/intelligence/test_router.py b/tests/intelligence/test_router.py index 8591b092..b6ea9319 100644 --- a/tests/intelligence/test_router.py +++ b/tests/intelligence/test_router.py @@ -1,15 +1,21 @@ -"""Tests for the heuristic model router.""" +"""Backward-compat: verify router is still importable from intelligence. + +The canonical tests live in tests/learning/test_router.py. This file +verifies the backward-compat shim in intelligence/router.py works. +""" from __future__ import annotations from openjarvis.core.registry import ModelRegistry from openjarvis.core.types import ModelSpec -from openjarvis.intelligence.router import HeuristicRouter, build_routing_context +from openjarvis.intelligence.router import ( + HeuristicRouter, + build_routing_context, +) from openjarvis.learning._stubs import RoutingContext def _register_models() -> None: - """Register a small set of models for testing.""" ModelRegistry.register_value( "small", ModelSpec( @@ -24,96 +30,17 @@ def _register_models() -> None: parameter_count_b=70.0, context_length=131072, ), ) - ModelRegistry.register_value( - "coder", - ModelSpec( - model_id="coder", name="DeepSeek Coder", - parameter_count_b=16.0, context_length=131072, - ), - ) -class TestBuildRoutingContext: - def test_code_detection(self) -> None: +class TestShimImports: + def test_build_routing_context(self) -> None: ctx = build_routing_context("def hello():\n pass") assert ctx.has_code is True - assert ctx.has_math is False - def test_math_detection(self) -> None: - ctx = build_routing_context("solve the integral of x^2") - assert ctx.has_math is True - assert ctx.has_code is False - - def test_length(self) -> None: - ctx = build_routing_context("Hi") - assert ctx.query_length == 2 - - def test_urgency_default(self) -> None: - ctx = build_routing_context("test") - assert ctx.urgency == 0.5 - - -class TestHeuristicRouter: - def test_short_query_prefers_small(self) -> None: - _register_models() - router = HeuristicRouter(available_models=["small", "large", "coder"]) - ctx = RoutingContext(query="Hi", query_length=2) - assert router.select_model(ctx) == "small" - - def test_code_prefers_coder(self) -> None: - _register_models() - router = HeuristicRouter(available_models=["small", "large", "coder"]) - ctx = RoutingContext(query="def foo():", query_length=10, has_code=True) - assert router.select_model(ctx) == "coder" - - def test_math_prefers_large(self) -> None: - _register_models() - router = HeuristicRouter(available_models=["small", "large", "coder"]) - ctx = RoutingContext(query="solve x", query_length=7, has_math=True) - assert router.select_model(ctx) == "large" - - def test_long_query_prefers_large(self) -> None: - _register_models() - router = HeuristicRouter(available_models=["small", "large", "coder"]) - ctx = RoutingContext(query="x" * 501, query_length=501) - assert router.select_model(ctx) == "large" - - def test_high_urgency_overrides_to_small(self) -> None: - _register_models() - router = HeuristicRouter(available_models=["small", "large", "coder"]) - ctx = RoutingContext(query="x" * 501, query_length=501, urgency=0.9) - assert router.select_model(ctx) == "small" - - def test_fallback_chain(self) -> None: + def test_heuristic_router(self) -> None: _register_models() router = HeuristicRouter( available_models=["small", "large"], - default_model="large", - fallback_model="small", ) - # Medium-length, no code/math, no reasoning → falls to default - ctx = RoutingContext(query="Tell me about cats", query_length=60) - assert router.select_model(ctx) == "large" - - def test_no_available_models(self) -> None: - router = HeuristicRouter( - available_models=[], default_model="fallback-model" - ) - ctx = RoutingContext(query="test", query_length=4) - assert router.select_model(ctx) == "fallback-model" - - def test_reasoning_keywords_prefer_large(self) -> None: - _register_models() - router = HeuristicRouter(available_models=["small", "large"]) - query = ( - "Please explain step by step how the process" - " of photosynthesis works in plants" - ) - ctx = build_routing_context(query) - assert router.select_model(ctx) == "large" - - def test_code_without_coder_falls_to_large(self) -> None: - _register_models() - router = HeuristicRouter(available_models=["small", "large"]) - ctx = RoutingContext(query="def foo():", query_length=10, has_code=True) - assert router.select_model(ctx) == "large" + ctx = RoutingContext(query="Hi", query_length=2) + assert router.select_model(ctx) == "small" diff --git a/tests/intelligence/test_routing_models.py b/tests/intelligence/test_routing_models.py index 919ca233..9ff19ccf 100644 --- a/tests/intelligence/test_routing_models.py +++ b/tests/intelligence/test_routing_models.py @@ -1,178 +1,35 @@ -"""Tests for router behavior with the extended model catalog.""" +"""Backward-compat: verify router with model catalog via intelligence imports. + +Canonical tests live in tests/learning/test_routing_models.py. +""" from __future__ import annotations -import pytest - from openjarvis.intelligence.model_catalog import register_builtin_models -from openjarvis.intelligence.router import HeuristicRouter, build_routing_context +from openjarvis.intelligence.router import ( + HeuristicRouter, + build_routing_context, +) from openjarvis.learning._stubs import RoutingContext -# New local model keys for testing -NEW_LOCAL_MODELS = [ - "gpt-oss:120b", # 117B total, 5.1B active, MoE - "qwen3:8b", # 8.2B, dense - "glm-4.7-flash", # 30B total, 3.0B active, MoE - "trinity-mini", # 26B total, 3.0B active, MoE -] - -# Cloud model keys -CLOUD_MODELS = [ - "gpt-5-mini", - "claude-opus-4-6", - "claude-sonnet-4-6", - "claude-haiku-4-5", - "gemini-2.5-pro", - "gemini-3-flash", -] - def _setup_models() -> None: - """Register builtin models needed for the tests.""" register_builtin_models() -class TestRouterWithNewModels: - """Router behavior when using the new local models.""" - - def test_short_query_routes_to_smallest(self) -> None: - _setup_models() - router = HeuristicRouter(available_models=NEW_LOCAL_MODELS) - ctx = RoutingContext(query="hi", query_length=2) - selected = router.select_model(ctx) - # qwen3:8b is 8.2B -- the smallest by parameter_count_b - assert selected == "qwen3:8b" - - def test_code_query_routes_to_largest(self) -> None: - _setup_models() - router = HeuristicRouter(available_models=NEW_LOCAL_MODELS) - ctx = RoutingContext( - query="def merge_sort(arr):", query_length=22, has_code=True - ) - selected = router.select_model(ctx) - # No model with "code"/"coder" in name, falls to largest → gpt-oss:120b (117B) - assert selected == "gpt-oss:120b" - - def test_code_query_with_coder_available(self) -> None: - _setup_models() - models = NEW_LOCAL_MODELS + ["deepseek-coder-v2:16b"] - router = HeuristicRouter(available_models=models) - ctx = RoutingContext( - query="import numpy as np", query_length=18, has_code=True - ) - selected = router.select_model(ctx) - assert selected == "deepseek-coder-v2:16b" - - def test_math_query_routes_to_largest(self) -> None: - _setup_models() - router = HeuristicRouter(available_models=NEW_LOCAL_MODELS) - ctx = RoutingContext( - query="solve the integral of x^2 dx", query_length=29, has_math=True - ) - selected = router.select_model(ctx) - assert selected == "gpt-oss:120b" - - def test_long_context_routes_to_largest(self) -> None: - _setup_models() - router = HeuristicRouter(available_models=NEW_LOCAL_MODELS) - ctx = RoutingContext(query="x" * 501, query_length=501) - selected = router.select_model(ctx) - assert selected == "gpt-oss:120b" - - def test_high_urgency_routes_to_smallest(self) -> None: - _setup_models() - router = HeuristicRouter(available_models=NEW_LOCAL_MODELS) - ctx = RoutingContext( - query="solve the integral of x^2", query_length=25, - has_math=True, urgency=0.9, - ) - selected = router.select_model(ctx) - # High urgency overrides everything → smallest params → qwen3:8b (8.2B) - assert selected == "qwen3:8b" - - def test_reasoning_query_routes_to_largest(self) -> None: - _setup_models() - router = HeuristicRouter(available_models=NEW_LOCAL_MODELS) - ctx = build_routing_context( - "Please explain step by step how neural networks learn" - ) - selected = router.select_model(ctx) - assert selected == "gpt-oss:120b" - - def test_medium_query_uses_default(self) -> None: +class TestShimRouterWithModels: + def test_short_query(self) -> None: _setup_models() router = HeuristicRouter( - available_models=NEW_LOCAL_MODELS, - default_model="glm-4.7-flash", + available_models=["qwen3:8b", "gpt-oss:120b"], ) - # Medium length query, no code/math/reasoning → rule 6 default - ctx = RoutingContext(query="Tell me about the weather today", query_length=60) - selected = router.select_model(ctx) - assert selected == "glm-4.7-flash" - - -class TestRouterCloudFallback: - """Router behavior when only cloud models are available.""" - - def test_no_local_falls_to_cloud(self) -> None: - _setup_models() - router = HeuristicRouter(available_models=CLOUD_MODELS) ctx = RoutingContext(query="hi", query_length=2) - selected = router.select_model(ctx) - # Should return one of the cloud models (smallest params) - assert selected in CLOUD_MODELS - - def test_cloud_model_selection_with_math(self) -> None: - _setup_models() - router = HeuristicRouter(available_models=CLOUD_MODELS) - ctx = RoutingContext(query="solve x", query_length=7, has_math=True) - selected = router.select_model(ctx) - # All cloud models have parameter_count_b=0 so falls to first - assert selected in CLOUD_MODELS - - def test_empty_models_returns_fallback(self) -> None: - router = HeuristicRouter( - available_models=[], - fallback_model="gpt-5-mini", - ) - ctx = RoutingContext(query="hello", query_length=5) - assert router.select_model(ctx) == "gpt-5-mini" - - -class TestRouterParameterized: - """Parametrized cross-product tests for model/query combinations.""" - - @pytest.mark.parametrize("query,expected_is_largest", [ - ("hi", False), - ("solve the integral of sin(x)", True), - ("def foo(): pass", True), # code → largest when no coder - ("x" * 501, True), - ]) - def test_query_type_selects_expected_size( - self, query: str, expected_is_largest: bool, - ) -> None: - _setup_models() - router = HeuristicRouter(available_models=NEW_LOCAL_MODELS) - ctx = build_routing_context(query) - selected = router.select_model(ctx) - if expected_is_largest: - assert selected == "gpt-oss:120b" - else: - assert selected == "qwen3:8b" - - @pytest.mark.parametrize("model_id", NEW_LOCAL_MODELS) - def test_single_model_always_returns_it(self, model_id: str) -> None: - _setup_models() - router = HeuristicRouter(available_models=[model_id]) - ctx = RoutingContext(query="hello world", query_length=11) - assert router.select_model(ctx) == model_id - - @pytest.mark.parametrize("urgency", [0.85, 0.9, 1.0]) - def test_high_urgency_always_smallest(self, urgency: float) -> None: - _setup_models() - router = HeuristicRouter(available_models=NEW_LOCAL_MODELS) - ctx = RoutingContext( - query="complex reasoning task", query_length=23, - has_math=True, urgency=urgency, - ) assert router.select_model(ctx) == "qwen3:8b" + + def test_code_query(self) -> None: + _setup_models() + router = HeuristicRouter( + available_models=["qwen3:8b", "gpt-oss:120b"], + ) + ctx = build_routing_context("def merge_sort(arr):") + assert router.select_model(ctx) == "gpt-oss:120b" diff --git a/tests/learning/test_heuristic_policy.py b/tests/learning/test_heuristic_policy.py index 416b12e2..6392db03 100644 --- a/tests/learning/test_heuristic_policy.py +++ b/tests/learning/test_heuristic_policy.py @@ -3,8 +3,8 @@ from __future__ import annotations from openjarvis.core.registry import RouterPolicyRegistry -from openjarvis.intelligence.router import HeuristicRouter from openjarvis.learning.heuristic_policy import ensure_registered +from openjarvis.learning.router import HeuristicRouter class TestHeuristicPolicy: diff --git a/tests/learning/test_router.py b/tests/learning/test_router.py new file mode 100644 index 00000000..5919c75a --- /dev/null +++ b/tests/learning/test_router.py @@ -0,0 +1,144 @@ +"""Tests for the heuristic model router (canonical location).""" + +from __future__ import annotations + +from openjarvis.core.registry import ModelRegistry +from openjarvis.core.types import ModelSpec +from openjarvis.learning._stubs import RoutingContext +from openjarvis.learning.router import ( + HeuristicRouter, + build_routing_context, +) + + +def _register_models() -> None: + """Register a small set of models for testing.""" + ModelRegistry.register_value( + "small", + ModelSpec( + model_id="small", name="Small", + parameter_count_b=3.0, context_length=4096, + ), + ) + ModelRegistry.register_value( + "large", + ModelSpec( + model_id="large", name="Large", + parameter_count_b=70.0, context_length=131072, + ), + ) + ModelRegistry.register_value( + "coder", + ModelSpec( + model_id="coder", name="DeepSeek Coder", + parameter_count_b=16.0, context_length=131072, + ), + ) + + +class TestBuildRoutingContext: + def test_code_detection(self) -> None: + ctx = build_routing_context("def hello():\n pass") + assert ctx.has_code is True + assert ctx.has_math is False + + def test_math_detection(self) -> None: + ctx = build_routing_context("solve the integral of x^2") + assert ctx.has_math is True + assert ctx.has_code is False + + def test_length(self) -> None: + ctx = build_routing_context("Hi") + assert ctx.query_length == 2 + + def test_urgency_default(self) -> None: + ctx = build_routing_context("test") + assert ctx.urgency == 0.5 + + +class TestHeuristicRouter: + def test_short_query_prefers_small(self) -> None: + _register_models() + router = HeuristicRouter( + available_models=["small", "large", "coder"], + ) + ctx = RoutingContext(query="Hi", query_length=2) + assert router.select_model(ctx) == "small" + + def test_code_prefers_coder(self) -> None: + _register_models() + router = HeuristicRouter( + available_models=["small", "large", "coder"], + ) + ctx = RoutingContext( + query="def foo():", query_length=10, has_code=True, + ) + assert router.select_model(ctx) == "coder" + + def test_math_prefers_large(self) -> None: + _register_models() + router = HeuristicRouter( + available_models=["small", "large", "coder"], + ) + ctx = RoutingContext( + query="solve x", query_length=7, has_math=True, + ) + assert router.select_model(ctx) == "large" + + def test_long_query_prefers_large(self) -> None: + _register_models() + router = HeuristicRouter( + available_models=["small", "large", "coder"], + ) + ctx = RoutingContext(query="x" * 501, query_length=501) + assert router.select_model(ctx) == "large" + + def test_high_urgency_overrides_to_small(self) -> None: + _register_models() + router = HeuristicRouter( + available_models=["small", "large", "coder"], + ) + ctx = RoutingContext( + query="x" * 501, query_length=501, urgency=0.9, + ) + assert router.select_model(ctx) == "small" + + def test_fallback_chain(self) -> None: + _register_models() + router = HeuristicRouter( + available_models=["small", "large"], + default_model="large", + fallback_model="small", + ) + # Medium-length, no code/math, no reasoning → falls to default + ctx = RoutingContext( + query="Tell me about cats", query_length=60, + ) + assert router.select_model(ctx) == "large" + + def test_no_available_models(self) -> None: + router = HeuristicRouter( + available_models=[], default_model="fallback-model", + ) + ctx = RoutingContext(query="test", query_length=4) + assert router.select_model(ctx) == "fallback-model" + + def test_reasoning_keywords_prefer_large(self) -> None: + _register_models() + router = HeuristicRouter(available_models=["small", "large"]) + query = ( + "Please explain step by step how the process" + " of photosynthesis works in plants" + ) + ctx = build_routing_context(query) + assert router.select_model(ctx) == "large" + + def test_code_without_coder_falls_to_large(self) -> None: + _register_models() + router = HeuristicRouter( + available_models=["small", "large"], + ) + ctx = RoutingContext( + query="def foo():", query_length=10, has_code=True, + ) + assert router.select_model(ctx) == "large" diff --git a/tests/learning/test_router_stubs.py b/tests/learning/test_router_stubs.py new file mode 100644 index 00000000..1820b9c0 --- /dev/null +++ b/tests/learning/test_router_stubs.py @@ -0,0 +1,71 @@ +"""Tests for RouterPolicy and QueryAnalyzer ABCs (canonical location).""" + +from __future__ import annotations + +import pytest + +from openjarvis.core.types import RoutingContext +from openjarvis.learning._stubs import QueryAnalyzer, RouterPolicy +from openjarvis.learning.router import DefaultQueryAnalyzer + + +class _DummyRouter(RouterPolicy): + def select_model(self, context: RoutingContext) -> str: + return "test-model" + + +class _DummyAnalyzer(QueryAnalyzer): + def analyze( + self, query: str, **kwargs: object, + ) -> RoutingContext: + return RoutingContext( + query=query, query_length=len(query), + ) + + +class TestRouterPolicy: + def test_abc_cannot_instantiate(self) -> None: + with pytest.raises(TypeError): + RouterPolicy() # type: ignore[abstract] + + def test_concrete_implementation(self) -> None: + router = _DummyRouter() + ctx = RoutingContext(query="hello") + assert router.select_model(ctx) == "test-model" + + +class TestQueryAnalyzer: + def test_abc_cannot_instantiate(self) -> None: + with pytest.raises(TypeError): + QueryAnalyzer() # type: ignore[abstract] + + def test_concrete_implementation(self) -> None: + analyzer = _DummyAnalyzer() + ctx = analyzer.analyze("hello world") + assert ctx.query == "hello world" + assert ctx.query_length == 11 + + +class TestDefaultQueryAnalyzer: + def test_analyze_basic(self) -> None: + analyzer = DefaultQueryAnalyzer() + ctx = analyzer.analyze("Hello world") + assert ctx.query == "Hello world" + assert ctx.query_length == 11 + assert ctx.has_code is False + assert ctx.has_math is False + + def test_analyze_code_query(self) -> None: + analyzer = DefaultQueryAnalyzer() + ctx = analyzer.analyze("def hello(): pass") + assert ctx.has_code is True + + def test_analyze_math_query(self) -> None: + analyzer = DefaultQueryAnalyzer() + ctx = analyzer.analyze("solve the integral of x^2") + assert ctx.has_math is True + + def test_analyze_with_urgency(self) -> None: + analyzer = DefaultQueryAnalyzer() + ctx = analyzer.analyze("quick question", urgency=0.9) + assert ctx.urgency == 0.9 diff --git a/tests/learning/test_routing_models.py b/tests/learning/test_routing_models.py new file mode 100644 index 00000000..cc707f69 --- /dev/null +++ b/tests/learning/test_routing_models.py @@ -0,0 +1,218 @@ +"""Tests for router behavior with the extended model catalog.""" + +from __future__ import annotations + +import pytest + +from openjarvis.intelligence.model_catalog import register_builtin_models +from openjarvis.learning._stubs import RoutingContext +from openjarvis.learning.router import ( + HeuristicRouter, + build_routing_context, +) + +# New local model keys for testing +NEW_LOCAL_MODELS = [ + "gpt-oss:120b", # 117B total, 5.1B active, MoE + "qwen3:8b", # 8.2B, dense + "glm-4.7-flash", # 30B total, 3.0B active, MoE + "trinity-mini", # 26B total, 3.0B active, MoE +] + +# Cloud model keys +CLOUD_MODELS = [ + "gpt-5-mini", + "claude-opus-4-6", + "claude-sonnet-4-6", + "claude-haiku-4-5", + "gemini-2.5-pro", + "gemini-3-flash", +] + + +def _setup_models() -> None: + """Register builtin models needed for the tests.""" + register_builtin_models() + + +class TestRouterWithNewModels: + """Router behavior when using the new local models.""" + + def test_short_query_routes_to_smallest(self) -> None: + _setup_models() + router = HeuristicRouter( + available_models=NEW_LOCAL_MODELS, + ) + ctx = RoutingContext(query="hi", query_length=2) + selected = router.select_model(ctx) + assert selected == "qwen3:8b" + + def test_code_query_routes_to_largest(self) -> None: + _setup_models() + router = HeuristicRouter( + available_models=NEW_LOCAL_MODELS, + ) + ctx = RoutingContext( + query="def merge_sort(arr):", + query_length=22, + has_code=True, + ) + selected = router.select_model(ctx) + assert selected == "gpt-oss:120b" + + def test_code_query_with_coder_available(self) -> None: + _setup_models() + models = NEW_LOCAL_MODELS + ["deepseek-coder-v2:16b"] + router = HeuristicRouter(available_models=models) + ctx = RoutingContext( + query="import numpy as np", + query_length=18, + has_code=True, + ) + selected = router.select_model(ctx) + assert selected == "deepseek-coder-v2:16b" + + def test_math_query_routes_to_largest(self) -> None: + _setup_models() + router = HeuristicRouter( + available_models=NEW_LOCAL_MODELS, + ) + ctx = RoutingContext( + query="solve the integral of x^2 dx", + query_length=29, + has_math=True, + ) + selected = router.select_model(ctx) + assert selected == "gpt-oss:120b" + + def test_long_context_routes_to_largest(self) -> None: + _setup_models() + router = HeuristicRouter( + available_models=NEW_LOCAL_MODELS, + ) + ctx = RoutingContext(query="x" * 501, query_length=501) + selected = router.select_model(ctx) + assert selected == "gpt-oss:120b" + + def test_high_urgency_routes_to_smallest(self) -> None: + _setup_models() + router = HeuristicRouter( + available_models=NEW_LOCAL_MODELS, + ) + ctx = RoutingContext( + query="solve the integral of x^2", + query_length=25, + has_math=True, + urgency=0.9, + ) + selected = router.select_model(ctx) + assert selected == "qwen3:8b" + + def test_reasoning_query_routes_to_largest(self) -> None: + _setup_models() + router = HeuristicRouter( + available_models=NEW_LOCAL_MODELS, + ) + ctx = build_routing_context( + "Please explain step by step how neural networks" + " learn" + ) + selected = router.select_model(ctx) + assert selected == "gpt-oss:120b" + + def test_medium_query_uses_default(self) -> None: + _setup_models() + router = HeuristicRouter( + available_models=NEW_LOCAL_MODELS, + default_model="glm-4.7-flash", + ) + ctx = RoutingContext( + query="Tell me about the weather today", + query_length=60, + ) + selected = router.select_model(ctx) + assert selected == "glm-4.7-flash" + + +class TestRouterCloudFallback: + """Router behavior when only cloud models are available.""" + + def test_no_local_falls_to_cloud(self) -> None: + _setup_models() + router = HeuristicRouter(available_models=CLOUD_MODELS) + ctx = RoutingContext(query="hi", query_length=2) + selected = router.select_model(ctx) + assert selected in CLOUD_MODELS + + def test_cloud_model_selection_with_math(self) -> None: + _setup_models() + router = HeuristicRouter(available_models=CLOUD_MODELS) + ctx = RoutingContext( + query="solve x", query_length=7, has_math=True, + ) + selected = router.select_model(ctx) + assert selected in CLOUD_MODELS + + def test_empty_models_returns_fallback(self) -> None: + router = HeuristicRouter( + available_models=[], + fallback_model="gpt-5-mini", + ) + ctx = RoutingContext(query="hello", query_length=5) + assert router.select_model(ctx) == "gpt-5-mini" + + +class TestRouterParameterized: + """Parametrized tests for model/query combinations.""" + + @pytest.mark.parametrize( + "query,expected_is_largest", + [ + ("hi", False), + ("solve the integral of sin(x)", True), + ("def foo(): pass", True), + ("x" * 501, True), + ], + ) + def test_query_type_selects_expected_size( + self, + query: str, + expected_is_largest: bool, + ) -> None: + _setup_models() + router = HeuristicRouter( + available_models=NEW_LOCAL_MODELS, + ) + ctx = build_routing_context(query) + selected = router.select_model(ctx) + if expected_is_largest: + assert selected == "gpt-oss:120b" + else: + assert selected == "qwen3:8b" + + @pytest.mark.parametrize("model_id", NEW_LOCAL_MODELS) + def test_single_model_always_returns_it( + self, model_id: str, + ) -> None: + _setup_models() + router = HeuristicRouter(available_models=[model_id]) + ctx = RoutingContext( + query="hello world", query_length=11, + ) + assert router.select_model(ctx) == model_id + + @pytest.mark.parametrize("urgency", [0.85, 0.9, 1.0]) + def test_high_urgency_always_smallest( + self, urgency: float, + ) -> None: + _setup_models() + router = HeuristicRouter( + available_models=NEW_LOCAL_MODELS, + ) + ctx = RoutingContext( + query="complex reasoning task", + query_length=23, + has_math=True, + urgency=urgency, + ) + assert router.select_model(ctx) == "qwen3:8b" diff --git a/tests/test_integration.py b/tests/test_integration.py index 64010644..4d609356 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -405,9 +405,9 @@ class TestAskFlowWithRouterPolicy: """Full ask flow with router policy (mocked).""" def test_mocked_ask_with_registry_router(self): - from openjarvis.intelligence.router import HeuristicRouter from openjarvis.learning._stubs import RoutingContext from openjarvis.learning.heuristic_policy import ensure_registered + from openjarvis.learning.router import HeuristicRouter ensure_registered() router_cls = RouterPolicyRegistry.get("heuristic")