mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-30 10:52:15 +00:00
Merge pull request #11 from HazyResearch/python-rust
Use Rust utils in Python SDK
This commit is contained in:
@@ -69,3 +69,6 @@ docs/plans/
|
||||
|
||||
# Tauri auto-generated schemas
|
||||
**/src-tauri/gen/schemas/
|
||||
|
||||
# Rust build artifacts
|
||||
rust/target/
|
||||
@@ -177,13 +177,8 @@ impl PyInjectionScanner {
|
||||
|
||||
fn scan(&self, text: &str) -> PyResult<String> {
|
||||
let result = self.inner.scan(text);
|
||||
// InjectionScanResult doesn't derive Serialize, so format manually.
|
||||
Ok(serde_json::json!({
|
||||
"is_clean": result.is_clean,
|
||||
"threat_level": format!("{:?}", result.threat_level),
|
||||
"findings_count": result.findings.len(),
|
||||
})
|
||||
.to_string())
|
||||
serde_json::to_string(&result)
|
||||
.map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -48,6 +48,12 @@ impl PySQLiteMemory {
|
||||
.map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))
|
||||
}
|
||||
|
||||
fn delete(&self, doc_id: &str) -> PyResult<bool> {
|
||||
self.inner
|
||||
.delete(doc_id)
|
||||
.map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))
|
||||
}
|
||||
|
||||
fn clear(&self) -> PyResult<()> {
|
||||
self.inner
|
||||
.clear()
|
||||
|
||||
@@ -105,7 +105,7 @@ static INJECTION_PATTERNS: Lazy<Vec<InjectionPattern>> = Lazy::new(|| {
|
||||
});
|
||||
|
||||
/// Result of an injection scan.
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct InjectionScanResult {
|
||||
pub is_clean: bool,
|
||||
pub findings: Vec<ScanFinding>,
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Single point of contact between Python and the Rust ``openjarvis_rust`` module.
|
||||
|
||||
Every Python module that wants to delegate to Rust should import helpers from
|
||||
here rather than importing ``openjarvis_rust`` directly. This keeps the
|
||||
try/except logic in one place, honours the ``OPENJARVIS_NO_RUST`` env-var
|
||||
override, and provides JSON-to-dataclass converters.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
from typing import TYPE_CHECKING, List, Optional
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import types as _types
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cached import
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def get_rust_module() -> Optional[_types.ModuleType]:
|
||||
"""Return the ``openjarvis_rust`` module, or ``None`` if unavailable.
|
||||
|
||||
Setting ``OPENJARVIS_NO_RUST=1`` forces pure-Python mode regardless of
|
||||
whether the compiled extension is importable.
|
||||
"""
|
||||
if os.environ.get("OPENJARVIS_NO_RUST", "").strip() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
):
|
||||
return None
|
||||
try:
|
||||
import openjarvis_rust # type: ignore[import-untyped]
|
||||
|
||||
return openjarvis_rust
|
||||
except ImportError:
|
||||
return None
|
||||
|
||||
|
||||
RUST_AVAILABLE: bool = get_rust_module() is not None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# JSON -> Python dataclass converters
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def scan_result_from_json(json_str: str) -> object:
|
||||
"""Convert a Rust scanner JSON string to a Python ``ScanResult``."""
|
||||
from openjarvis.security.types import (
|
||||
ScanFinding,
|
||||
ScanResult,
|
||||
ThreatLevel,
|
||||
)
|
||||
|
||||
data = json.loads(json_str)
|
||||
findings: List[ScanFinding] = []
|
||||
for f in data.get("findings", []):
|
||||
findings.append(
|
||||
ScanFinding(
|
||||
pattern_name=f.get("pattern_name", ""),
|
||||
matched_text=f.get("matched_text", ""),
|
||||
threat_level=ThreatLevel(
|
||||
f.get("threat_level", "low").lower(),
|
||||
),
|
||||
start=f.get("start", 0),
|
||||
end=f.get("end", 0),
|
||||
description=f.get("description", ""),
|
||||
)
|
||||
)
|
||||
return ScanResult(findings=findings)
|
||||
|
||||
|
||||
def injection_result_from_json(json_str: str) -> object:
|
||||
"""Convert Rust ``InjectionScanner.scan()`` JSON to dataclass."""
|
||||
from openjarvis.security.injection_scanner import (
|
||||
InjectionScanResult,
|
||||
)
|
||||
from openjarvis.security.types import ScanFinding, ThreatLevel
|
||||
|
||||
data = json.loads(json_str)
|
||||
findings: List[ScanFinding] = []
|
||||
for f in data.get("findings", []):
|
||||
findings.append(
|
||||
ScanFinding(
|
||||
pattern_name=f.get("pattern_name", ""),
|
||||
matched_text=f.get("matched_text", ""),
|
||||
threat_level=ThreatLevel(
|
||||
f.get("threat_level", "low").lower(),
|
||||
),
|
||||
start=f.get("start", 0),
|
||||
end=f.get("end", 0),
|
||||
description=f.get("description", ""),
|
||||
)
|
||||
)
|
||||
|
||||
threat_raw = data.get("threat_level", "low").lower()
|
||||
try:
|
||||
threat = ThreatLevel(threat_raw)
|
||||
except ValueError:
|
||||
threat = ThreatLevel.LOW
|
||||
|
||||
return InjectionScanResult(
|
||||
is_clean=data.get("is_clean", True),
|
||||
findings=findings,
|
||||
threat_level=threat,
|
||||
)
|
||||
|
||||
|
||||
def retrieval_results_from_json(json_str: str) -> list:
|
||||
"""Convert Rust memory ``retrieve()`` JSON to a list of results."""
|
||||
from openjarvis.tools.storage._stubs import RetrievalResult
|
||||
|
||||
items = json.loads(json_str)
|
||||
results: List[RetrievalResult] = []
|
||||
for item in items:
|
||||
meta = item.get("metadata", {})
|
||||
if isinstance(meta, str):
|
||||
try:
|
||||
meta = json.loads(meta)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
meta = {}
|
||||
results.append(
|
||||
RetrievalResult(
|
||||
content=item.get("content", ""),
|
||||
score=float(item.get("score", 0.0)),
|
||||
source=item.get("source", ""),
|
||||
metadata=meta,
|
||||
)
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
__all__ = [
|
||||
"RUST_AVAILABLE",
|
||||
"get_rust_module",
|
||||
"injection_result_from_json",
|
||||
"retrieval_results_from_json",
|
||||
"scan_result_from_json",
|
||||
]
|
||||
@@ -48,8 +48,29 @@ class LoopGuard:
|
||||
# Track per-tool call counts (for polling budget)
|
||||
self._per_tool_counts: dict[str, int] = {}
|
||||
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
_rust = get_rust_module()
|
||||
if _rust:
|
||||
self._rust_impl = _rust.LoopGuard(
|
||||
max_identical=config.max_identical_calls,
|
||||
max_ping_pong=(
|
||||
config.ping_pong_window // 2
|
||||
if config.ping_pong_window > 1
|
||||
else 2
|
||||
),
|
||||
poll_budget=config.poll_tool_budget,
|
||||
)
|
||||
else:
|
||||
self._rust_impl = None
|
||||
|
||||
def check_call(self, tool_name: str, arguments: str) -> LoopVerdict:
|
||||
"""Check whether a tool call should proceed or be blocked."""
|
||||
if self._rust_impl is not None:
|
||||
reason = self._rust_impl.check(tool_name, arguments)
|
||||
if reason is not None:
|
||||
self._emit_triggered("rust_guard", tool_name)
|
||||
return LoopVerdict(blocked=True, reason=reason)
|
||||
return LoopVerdict()
|
||||
# 1. Hash tracking — identical calls
|
||||
call_hash = hashlib.sha256(
|
||||
f"{tool_name}:{arguments}".encode()
|
||||
@@ -167,6 +188,8 @@ class LoopGuard:
|
||||
self._call_counts.clear()
|
||||
self._tool_sequence.clear()
|
||||
self._per_tool_counts.clear()
|
||||
if self._rust_impl is not None:
|
||||
self._rust_impl.reset()
|
||||
|
||||
def _detect_ping_pong(self) -> bool:
|
||||
"""Detect repeating patterns in tool call sequence."""
|
||||
|
||||
@@ -58,6 +58,15 @@ class CapabilityPolicy:
|
||||
) -> None:
|
||||
self._policies: Dict[str, AgentPolicy] = {}
|
||||
self._default_deny = default_deny
|
||||
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
_rust = get_rust_module()
|
||||
self._rust_impl = (
|
||||
_rust.CapabilityPolicy(default_deny=default_deny)
|
||||
if _rust
|
||||
else None
|
||||
)
|
||||
|
||||
if policy_path:
|
||||
self._load_file(Path(policy_path))
|
||||
|
||||
@@ -67,6 +76,8 @@ class CapabilityPolicy:
|
||||
agent_id, AgentPolicy(agent_id=agent_id),
|
||||
)
|
||||
policy.grants.append(CapabilityGrant(capability=capability, pattern=pattern))
|
||||
if self._rust_impl is not None:
|
||||
self._rust_impl.grant(agent_id, capability, pattern)
|
||||
|
||||
def deny(self, agent_id: str, capability: str) -> None:
|
||||
"""Explicitly deny a capability to an agent."""
|
||||
@@ -74,12 +85,16 @@ class CapabilityPolicy:
|
||||
agent_id, AgentPolicy(agent_id=agent_id),
|
||||
)
|
||||
policy.deny.append(capability)
|
||||
if self._rust_impl is not None:
|
||||
self._rust_impl.deny(agent_id, capability)
|
||||
|
||||
def check(self, agent_id: str, capability: str, resource: str = "") -> bool:
|
||||
"""Check whether *agent_id* has *capability* for *resource*.
|
||||
|
||||
Returns True if allowed, False if denied.
|
||||
"""
|
||||
if self._rust_impl is not None:
|
||||
return self._rust_impl.check(agent_id, capability, resource)
|
||||
policy = self._policies.get(agent_id)
|
||||
if policy is None:
|
||||
# No explicit policy — use default
|
||||
|
||||
@@ -32,6 +32,12 @@ def is_sensitive_file(path: Union[str, Path]) -> bool:
|
||||
Checks both the filename and the full name against
|
||||
``DEFAULT_SENSITIVE_PATTERNS`` using :func:`fnmatch.fnmatch`.
|
||||
"""
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
|
||||
_rust = get_rust_module()
|
||||
if _rust is not None:
|
||||
return _rust.is_sensitive_file(str(path))
|
||||
|
||||
p = Path(path)
|
||||
name = p.name
|
||||
for pattern in DEFAULT_SENSITIVE_PATTERNS:
|
||||
|
||||
@@ -124,9 +124,15 @@ class InjectionScanner:
|
||||
(re.compile(pat), name, level, desc)
|
||||
for pat, name, level, desc in _INJECTION_PATTERNS
|
||||
]
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
_rust = get_rust_module()
|
||||
self._rust_impl = _rust.InjectionScanner() if _rust else None
|
||||
|
||||
def scan(self, text: str) -> InjectionScanResult:
|
||||
"""Scan text for injection patterns."""
|
||||
if self._rust_impl is not None:
|
||||
from openjarvis._rust_bridge import injection_result_from_json
|
||||
return injection_result_from_json(self._rust_impl.scan(text))
|
||||
findings: List[ScanFinding] = []
|
||||
max_threat = ThreatLevel.LOW
|
||||
|
||||
|
||||
@@ -66,10 +66,22 @@ class RateLimiter:
|
||||
self._buckets: Dict[str, TokenBucket] = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
_rust = get_rust_module()
|
||||
if _rust and self._config.enabled:
|
||||
self._rust_impl = _rust.RateLimiter(
|
||||
requests_per_minute=self._config.requests_per_minute,
|
||||
burst_size=self._config.burst_size,
|
||||
)
|
||||
else:
|
||||
self._rust_impl = None
|
||||
|
||||
def check(self, key: str) -> Tuple[bool, float]:
|
||||
"""Check if request is allowed for key. Returns (allowed, wait_seconds)."""
|
||||
if not self._config.enabled:
|
||||
return True, 0.0
|
||||
if self._rust_impl is not None:
|
||||
return self._rust_impl.check(key)
|
||||
|
||||
bucket = self._get_bucket(key)
|
||||
return bucket.consume()
|
||||
@@ -87,6 +99,9 @@ class RateLimiter:
|
||||
|
||||
def reset(self, key: Optional[str] = None) -> None:
|
||||
"""Reset rate limit state for a key or all keys."""
|
||||
if self._rust_impl is not None:
|
||||
self._rust_impl.reset(key)
|
||||
return
|
||||
with self._lock:
|
||||
if key:
|
||||
self._buckets.pop(key, None)
|
||||
|
||||
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import re
|
||||
from typing import Dict, Tuple
|
||||
|
||||
from openjarvis._rust_bridge import get_rust_module, scan_result_from_json
|
||||
from openjarvis.security._stubs import BaseScanner
|
||||
from openjarvis.security.types import ScanFinding, ScanResult, ThreatLevel
|
||||
|
||||
@@ -18,6 +19,10 @@ class SecretScanner(BaseScanner):
|
||||
|
||||
scanner_id = "secrets"
|
||||
|
||||
def __init__(self) -> None:
|
||||
_rust = get_rust_module()
|
||||
self._rust_impl = _rust.SecretScanner() if _rust else None
|
||||
|
||||
PATTERNS: Dict[str, Tuple[str, ThreatLevel, str]] = {
|
||||
"openai_key": (
|
||||
r"sk-[A-Za-z0-9_-]{20,}",
|
||||
@@ -73,6 +78,8 @@ class SecretScanner(BaseScanner):
|
||||
|
||||
def scan(self, text: str) -> ScanResult:
|
||||
"""Scan *text* for secret patterns."""
|
||||
if self._rust_impl is not None:
|
||||
return scan_result_from_json(self._rust_impl.scan(text))
|
||||
findings = []
|
||||
for name, (pattern, threat, desc) in self.PATTERNS.items():
|
||||
for match in re.finditer(pattern, text):
|
||||
@@ -90,6 +97,8 @@ class SecretScanner(BaseScanner):
|
||||
|
||||
def redact(self, text: str) -> str:
|
||||
"""Replace secret matches with ``[REDACTED:{pattern_name}]``."""
|
||||
if self._rust_impl is not None:
|
||||
return self._rust_impl.redact(text)
|
||||
result = text
|
||||
for name, (pattern, _threat, _desc) in self.PATTERNS.items():
|
||||
result = re.sub(pattern, f"[REDACTED:{name}]", result)
|
||||
@@ -106,6 +115,10 @@ class PIIScanner(BaseScanner):
|
||||
|
||||
scanner_id = "pii"
|
||||
|
||||
def __init__(self) -> None:
|
||||
_rust = get_rust_module()
|
||||
self._rust_impl = _rust.PIIScanner() if _rust else None
|
||||
|
||||
PATTERNS: Dict[str, Tuple[str, ThreatLevel, str]] = {
|
||||
"email": (
|
||||
r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}",
|
||||
@@ -146,6 +159,8 @@ class PIIScanner(BaseScanner):
|
||||
|
||||
def scan(self, text: str) -> ScanResult:
|
||||
"""Scan *text* for PII patterns."""
|
||||
if self._rust_impl is not None:
|
||||
return scan_result_from_json(self._rust_impl.scan(text))
|
||||
findings = []
|
||||
for name, (pattern, threat, desc) in self.PATTERNS.items():
|
||||
for match in re.finditer(pattern, text):
|
||||
@@ -163,6 +178,8 @@ class PIIScanner(BaseScanner):
|
||||
|
||||
def redact(self, text: str) -> str:
|
||||
"""Replace PII matches with ``[REDACTED:{pattern_name}]``."""
|
||||
if self._rust_impl is not None:
|
||||
return self._rust_impl.redact(text)
|
||||
result = text
|
||||
for name, (pattern, _threat, _desc) in self.PATTERNS.items():
|
||||
result = re.sub(pattern, f"[REDACTED:{name}]", result)
|
||||
|
||||
@@ -37,6 +37,12 @@ def is_private_ip(ip_str: str) -> bool:
|
||||
|
||||
def check_ssrf(url: str) -> Optional[str]:
|
||||
"""Check a URL for SSRF vulnerabilities. Returns error message or None if safe."""
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
|
||||
_rust = get_rust_module()
|
||||
if _rust is not None:
|
||||
return _rust.check_ssrf(url)
|
||||
|
||||
from urllib.parse import urlparse
|
||||
|
||||
parsed = urlparse(url)
|
||||
|
||||
@@ -90,6 +90,13 @@ def _safe_eval_node(node: ast.AST) -> Any:
|
||||
|
||||
def safe_eval(expression: str) -> float:
|
||||
"""Evaluate a math expression safely using AST parsing."""
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
_rust = get_rust_module()
|
||||
if _rust is not None:
|
||||
try:
|
||||
return float(_rust.CalculatorTool().execute(expression))
|
||||
except Exception:
|
||||
pass # Fallback to Python on Rust errors
|
||||
tree = ast.parse(expression, mode="eval")
|
||||
return _safe_eval_node(tree)
|
||||
|
||||
|
||||
@@ -114,7 +114,28 @@ class FileReadTool(BaseTool):
|
||||
content=f"File too large: {size} bytes (max {_MAX_SIZE_BYTES}).",
|
||||
success=False,
|
||||
)
|
||||
# Read
|
||||
# Read — delegate to Rust if available
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
_rust = get_rust_module()
|
||||
if _rust is not None:
|
||||
try:
|
||||
text = _rust.FileReadTool().execute(str(path))
|
||||
except Exception as exc:
|
||||
return ToolResult(
|
||||
tool_name="file_read",
|
||||
content=f"Read error: {exc}",
|
||||
success=False,
|
||||
)
|
||||
max_lines = params.get("max_lines")
|
||||
if max_lines is not None and max_lines > 0:
|
||||
lines = text.splitlines(keepends=True)
|
||||
text = "".join(lines[:max_lines])
|
||||
return ToolResult(
|
||||
tool_name="file_read",
|
||||
content=text,
|
||||
success=True,
|
||||
metadata={"path": str(path.resolve()), "size_bytes": size},
|
||||
)
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
except UnicodeDecodeError:
|
||||
|
||||
@@ -155,25 +155,50 @@ class FileWriteTool(BaseTool):
|
||||
success=False,
|
||||
)
|
||||
|
||||
# Write content
|
||||
try:
|
||||
if mode == "append":
|
||||
# Write content — delegate to Rust for write mode if available
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
_rust = get_rust_module()
|
||||
if _rust is not None and mode == "write":
|
||||
try:
|
||||
_rust.FileWriteTool().execute(str(path), content)
|
||||
except Exception as exc:
|
||||
return ToolResult(
|
||||
tool_name="file_write",
|
||||
content=f"Write error: {exc}",
|
||||
success=False,
|
||||
)
|
||||
elif mode == "write":
|
||||
try:
|
||||
path.write_text(content, encoding="utf-8")
|
||||
except PermissionError as exc:
|
||||
return ToolResult(
|
||||
tool_name="file_write",
|
||||
content=f"Permission denied: {exc}",
|
||||
success=False,
|
||||
)
|
||||
except OSError as exc:
|
||||
return ToolResult(
|
||||
tool_name="file_write",
|
||||
content=f"Write error: {exc}",
|
||||
success=False,
|
||||
)
|
||||
else:
|
||||
# append mode — always Python
|
||||
try:
|
||||
with open(path, "a", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
else:
|
||||
path.write_text(content, encoding="utf-8")
|
||||
except PermissionError as exc:
|
||||
return ToolResult(
|
||||
tool_name="file_write",
|
||||
content=f"Permission denied: {exc}",
|
||||
success=False,
|
||||
)
|
||||
except OSError as exc:
|
||||
return ToolResult(
|
||||
tool_name="file_write",
|
||||
content=f"Write error: {exc}",
|
||||
success=False,
|
||||
)
|
||||
except PermissionError as exc:
|
||||
return ToolResult(
|
||||
tool_name="file_write",
|
||||
content=f"Permission denied: {exc}",
|
||||
success=False,
|
||||
)
|
||||
except OSError as exc:
|
||||
return ToolResult(
|
||||
tool_name="file_write",
|
||||
content=f"Write error: {exc}",
|
||||
success=False,
|
||||
)
|
||||
|
||||
# Get final file size
|
||||
try:
|
||||
|
||||
@@ -6,6 +6,7 @@ import shutil
|
||||
import subprocess
|
||||
from typing import Any
|
||||
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
from openjarvis.core.registry import ToolRegistry
|
||||
from openjarvis.core.types import ToolResult
|
||||
from openjarvis.tools._stubs import BaseTool, ToolSpec
|
||||
@@ -130,6 +131,18 @@ class GitStatusTool(BaseTool):
|
||||
|
||||
def execute(self, **params: Any) -> ToolResult:
|
||||
repo_path = params.get("repo_path", ".")
|
||||
_rust = get_rust_module()
|
||||
if _rust is not None:
|
||||
try:
|
||||
output = _rust.GitStatusTool().execute(repo_path)
|
||||
return ToolResult(
|
||||
tool_name="git_status",
|
||||
content=output or "(no output)",
|
||||
success=True,
|
||||
metadata={"returncode": 0},
|
||||
)
|
||||
except Exception:
|
||||
pass # Fall through to subprocess
|
||||
return _run_git(["git", "status", "--porcelain"], cwd=repo_path)
|
||||
|
||||
|
||||
@@ -188,6 +201,19 @@ class GitDiffTool(BaseTool):
|
||||
staged = params.get("staged", False)
|
||||
file_path = params.get("path")
|
||||
|
||||
_rust = get_rust_module()
|
||||
if _rust is not None and not staged and not file_path:
|
||||
try:
|
||||
output = _rust.GitDiffTool().execute(repo_path)
|
||||
return ToolResult(
|
||||
tool_name="git_diff",
|
||||
content=output or "(no output)",
|
||||
success=True,
|
||||
metadata={"returncode": 0},
|
||||
)
|
||||
except Exception:
|
||||
pass # Fall through to subprocess
|
||||
|
||||
cmd = ["git", "diff"]
|
||||
if staged:
|
||||
cmd.append("--staged")
|
||||
@@ -340,6 +366,19 @@ class GitLogTool(BaseTool):
|
||||
count = params.get("count", 10)
|
||||
oneline = params.get("oneline", True)
|
||||
|
||||
_rust = get_rust_module()
|
||||
if _rust is not None:
|
||||
try:
|
||||
output = _rust.GitLogTool().execute(repo_path, count)
|
||||
return ToolResult(
|
||||
tool_name="git_log",
|
||||
content=output or "(no output)",
|
||||
success=True,
|
||||
metadata={"returncode": 0},
|
||||
)
|
||||
except Exception:
|
||||
pass # Fall through to subprocess
|
||||
|
||||
cmd = ["git", "log", f"-{count}"]
|
||||
if oneline:
|
||||
cmd.append("--oneline")
|
||||
|
||||
@@ -100,6 +100,28 @@ class HttpRequestTool(BaseTool):
|
||||
body = params.get("body")
|
||||
timeout = params.get("timeout", 30)
|
||||
|
||||
# Try Rust backend for simple requests (no custom headers)
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
_rust = get_rust_module()
|
||||
if _rust is not None and not headers:
|
||||
try:
|
||||
content = _rust.HttpRequestTool().execute(url, method, body)
|
||||
return ToolResult(
|
||||
tool_name="http_request",
|
||||
content=(
|
||||
content[:_MAX_RESPONSE_BYTES]
|
||||
if len(content) > _MAX_RESPONSE_BYTES
|
||||
else content
|
||||
),
|
||||
success=True,
|
||||
metadata={
|
||||
"status_code": 200,
|
||||
"truncated": len(content) > _MAX_RESPONSE_BYTES,
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
pass # Fall through to Python httpx
|
||||
|
||||
try:
|
||||
t0 = time.time()
|
||||
response = httpx.request(
|
||||
|
||||
@@ -125,7 +125,33 @@ class ShellExecTool(BaseTool):
|
||||
if val is not None:
|
||||
env[key] = val
|
||||
|
||||
# Execute
|
||||
# Execute — try Rust backend first
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
_rust = get_rust_module()
|
||||
if _rust is not None:
|
||||
try:
|
||||
output = _rust.ShellExecTool().execute(command, working_dir)
|
||||
return ToolResult(
|
||||
tool_name="shell_exec",
|
||||
content=output or "(no output)",
|
||||
success=True,
|
||||
metadata={
|
||||
"returncode": 0,
|
||||
"timeout_used": timeout,
|
||||
"working_dir": working_dir,
|
||||
},
|
||||
)
|
||||
except Exception as exc:
|
||||
return ToolResult(
|
||||
tool_name="shell_exec",
|
||||
content=str(exc),
|
||||
success=False,
|
||||
metadata={
|
||||
"returncode": -1,
|
||||
"timeout_used": timeout,
|
||||
"working_dir": working_dir,
|
||||
},
|
||||
)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
command,
|
||||
|
||||
@@ -2,22 +2,29 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
try:
|
||||
from rank_bm25 import BM25Okapi
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"The 'rank_bm25' package is required for the BM25 memory "
|
||||
"backend. Install it with:\n\n"
|
||||
" pip install rank-bm25\n"
|
||||
) from exc
|
||||
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
from openjarvis.core.events import EventType, get_event_bus
|
||||
from openjarvis.core.registry import MemoryRegistry
|
||||
from openjarvis.tools.storage._stubs import MemoryBackend, RetrievalResult
|
||||
|
||||
_rust = get_rust_module()
|
||||
|
||||
# rank_bm25 is only needed when Rust backend is unavailable.
|
||||
BM25Okapi: Any = None
|
||||
if _rust is None:
|
||||
try:
|
||||
from rank_bm25 import BM25Okapi # type: ignore[no-redef] # noqa: E402
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"The 'rank_bm25' package is required for the BM25 memory "
|
||||
"backend. Install it with:\n\n"
|
||||
" pip install rank-bm25\n"
|
||||
) from exc
|
||||
|
||||
|
||||
def _tokenize(text: str) -> List[str]:
|
||||
"""Lowercase whitespace tokenizer."""
|
||||
@@ -36,13 +43,16 @@ class BM25Memory(MemoryBackend):
|
||||
backend_id: str = "bm25"
|
||||
|
||||
def __init__(self) -> None:
|
||||
_r = get_rust_module()
|
||||
self._rust_impl = _r.BM25Memory() if _r else None
|
||||
|
||||
# id -> (content, source, metadata)
|
||||
self._documents: Dict[
|
||||
str, Tuple[str, str, Dict[str, Any]]
|
||||
] = {}
|
||||
self._corpus: List[List[str]] = []
|
||||
self._doc_ids: List[str] = []
|
||||
self._bm25: Optional[BM25Okapi] = None
|
||||
self._bm25: Optional[Any] = None
|
||||
|
||||
# -- ABC implementation -------------------------------------------------
|
||||
|
||||
@@ -54,6 +64,17 @@ class BM25Memory(MemoryBackend):
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
) -> str:
|
||||
"""Persist *content* and return a unique document id."""
|
||||
if self._rust_impl is not None:
|
||||
meta_json = json.dumps(metadata) if metadata else None
|
||||
doc_id = self._rust_impl.store(content, source, meta_json)
|
||||
bus = get_event_bus()
|
||||
bus.publish(EventType.MEMORY_STORE, {
|
||||
"backend": self.backend_id,
|
||||
"doc_id": doc_id,
|
||||
"source": source,
|
||||
})
|
||||
return doc_id
|
||||
|
||||
doc_id = uuid.uuid4().hex
|
||||
self._documents[doc_id] = (
|
||||
content,
|
||||
@@ -78,6 +99,21 @@ class BM25Memory(MemoryBackend):
|
||||
**kwargs: Any,
|
||||
) -> List[RetrievalResult]:
|
||||
"""Search for *query* and return the top-k results."""
|
||||
if self._rust_impl is not None:
|
||||
if not query.strip():
|
||||
return []
|
||||
from openjarvis._rust_bridge import retrieval_results_from_json
|
||||
results = retrieval_results_from_json(
|
||||
self._rust_impl.retrieve(query, top_k),
|
||||
)
|
||||
bus = get_event_bus()
|
||||
bus.publish(EventType.MEMORY_RETRIEVE, {
|
||||
"backend": self.backend_id,
|
||||
"query": query,
|
||||
"num_results": len(results),
|
||||
})
|
||||
return results
|
||||
|
||||
if not query.strip() or self._bm25 is None:
|
||||
bus = get_event_bus()
|
||||
bus.publish(EventType.MEMORY_RETRIEVE, {
|
||||
@@ -126,6 +162,9 @@ class BM25Memory(MemoryBackend):
|
||||
|
||||
def delete(self, doc_id: str) -> bool:
|
||||
"""Delete a document by id. Return ``True`` if it existed."""
|
||||
if self._rust_impl is not None:
|
||||
# Rust BM25Memory doesn't expose delete; fall through to Python.
|
||||
pass
|
||||
if doc_id not in self._documents:
|
||||
return False
|
||||
del self._documents[doc_id]
|
||||
@@ -143,6 +182,8 @@ class BM25Memory(MemoryBackend):
|
||||
|
||||
def count(self) -> int:
|
||||
"""Return the number of stored documents."""
|
||||
if self._rust_impl is not None:
|
||||
return self._rust_impl.count()
|
||||
return len(self._documents)
|
||||
|
||||
def _rebuild_index(self) -> None:
|
||||
|
||||
@@ -37,6 +37,15 @@ class SQLiteMemory(MemoryBackend):
|
||||
db_path = str(DEFAULT_CONFIG_DIR / "memory.db")
|
||||
|
||||
self._db_path = str(db_path)
|
||||
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
_rust = get_rust_module()
|
||||
if _rust is not None:
|
||||
self._rust_impl = _rust.SQLiteMemory(self._db_path)
|
||||
self._conn = None # type: ignore[assignment]
|
||||
return
|
||||
|
||||
self._rust_impl = None
|
||||
self._conn = sqlite3.connect(self._db_path)
|
||||
self._conn.row_factory = sqlite3.Row
|
||||
|
||||
@@ -100,6 +109,17 @@ class SQLiteMemory(MemoryBackend):
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
) -> str:
|
||||
"""Persist *content* and return a unique document id."""
|
||||
if self._rust_impl is not None:
|
||||
meta_json = json.dumps(metadata) if metadata else None
|
||||
doc_id = self._rust_impl.store(content, source, meta_json)
|
||||
bus = get_event_bus()
|
||||
bus.publish(EventType.MEMORY_STORE, {
|
||||
"backend": self.backend_id,
|
||||
"doc_id": doc_id,
|
||||
"source": source,
|
||||
})
|
||||
return doc_id
|
||||
|
||||
import time
|
||||
|
||||
doc_id = uuid.uuid4().hex
|
||||
@@ -130,6 +150,19 @@ class SQLiteMemory(MemoryBackend):
|
||||
if not query.strip():
|
||||
return []
|
||||
|
||||
if self._rust_impl is not None:
|
||||
from openjarvis._rust_bridge import retrieval_results_from_json
|
||||
results = retrieval_results_from_json(
|
||||
self._rust_impl.retrieve(query, top_k),
|
||||
)
|
||||
bus = get_event_bus()
|
||||
bus.publish(EventType.MEMORY_RETRIEVE, {
|
||||
"backend": self.backend_id,
|
||||
"query": query,
|
||||
"num_results": len(results),
|
||||
})
|
||||
return results
|
||||
|
||||
# Escape FTS5 special characters
|
||||
safe_query = self._escape_fts_query(query)
|
||||
if not safe_query:
|
||||
@@ -171,6 +204,8 @@ class SQLiteMemory(MemoryBackend):
|
||||
|
||||
def delete(self, doc_id: str) -> bool:
|
||||
"""Delete a document by id. Return True if it existed."""
|
||||
if self._rust_impl is not None:
|
||||
return self._rust_impl.delete(doc_id)
|
||||
cursor = self._conn.execute(
|
||||
"DELETE FROM documents WHERE id = ?", (doc_id,)
|
||||
)
|
||||
@@ -179,11 +214,16 @@ class SQLiteMemory(MemoryBackend):
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Remove all stored documents."""
|
||||
if self._rust_impl is not None:
|
||||
self._rust_impl.clear()
|
||||
return
|
||||
self._conn.execute("DELETE FROM documents")
|
||||
self._conn.commit()
|
||||
|
||||
def count(self) -> int:
|
||||
"""Return the number of stored documents."""
|
||||
if self._rust_impl is not None:
|
||||
return self._rust_impl.count()
|
||||
row = self._conn.execute(
|
||||
"SELECT COUNT(*) FROM documents"
|
||||
).fetchone()
|
||||
@@ -191,7 +231,8 @@ class SQLiteMemory(MemoryBackend):
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the database connection."""
|
||||
self._conn.close()
|
||||
if self._conn is not None:
|
||||
self._conn.close()
|
||||
|
||||
@staticmethod
|
||||
def _escape_fts_query(query: str) -> str:
|
||||
|
||||
@@ -40,6 +40,15 @@ class ThinkTool(BaseTool):
|
||||
|
||||
def execute(self, **params: Any) -> ToolResult:
|
||||
thought = params.get("thought", "")
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
_rust = get_rust_module()
|
||||
if _rust is not None:
|
||||
content = _rust.ThinkTool().execute(thought)
|
||||
return ToolResult(
|
||||
tool_name="think",
|
||||
content=content,
|
||||
success=True,
|
||||
)
|
||||
return ToolResult(
|
||||
tool_name="think",
|
||||
content=thought,
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Tests for the Rust bridge module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestGetRustModule:
|
||||
"""Test get_rust_module() caching and env var override."""
|
||||
|
||||
def test_returns_module_or_none(self):
|
||||
"""get_rust_module() should return a module or None, never raise."""
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
# Clear the lru_cache for a clean test
|
||||
get_rust_module.cache_clear()
|
||||
result = get_rust_module()
|
||||
assert result is None or hasattr(result, "__name__")
|
||||
|
||||
def test_env_var_forces_none(self, monkeypatch):
|
||||
"""OPENJARVIS_NO_RUST=1 forces pure-Python mode."""
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
get_rust_module.cache_clear()
|
||||
monkeypatch.setenv("OPENJARVIS_NO_RUST", "1")
|
||||
result = get_rust_module()
|
||||
assert result is None
|
||||
# Cleanup
|
||||
get_rust_module.cache_clear()
|
||||
|
||||
def test_env_var_true(self, monkeypatch):
|
||||
"""OPENJARVIS_NO_RUST=true forces pure-Python mode."""
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
get_rust_module.cache_clear()
|
||||
monkeypatch.setenv("OPENJARVIS_NO_RUST", "true")
|
||||
result = get_rust_module()
|
||||
assert result is None
|
||||
get_rust_module.cache_clear()
|
||||
|
||||
|
||||
class TestScanResultFromJson:
|
||||
"""Test JSON→ScanResult conversion."""
|
||||
|
||||
def test_empty_findings(self):
|
||||
from openjarvis._rust_bridge import scan_result_from_json
|
||||
result = scan_result_from_json('{"findings": []}')
|
||||
assert result.clean
|
||||
assert result.findings == []
|
||||
|
||||
def test_with_findings(self):
|
||||
from openjarvis._rust_bridge import scan_result_from_json
|
||||
data = {
|
||||
"findings": [
|
||||
{
|
||||
"pattern_name": "openai_key",
|
||||
"matched_text": "sk-abc123",
|
||||
"threat_level": "critical",
|
||||
"start": 0,
|
||||
"end": 9,
|
||||
"description": "OpenAI API key",
|
||||
},
|
||||
],
|
||||
}
|
||||
result = scan_result_from_json(json.dumps(data))
|
||||
assert not result.clean
|
||||
assert len(result.findings) == 1
|
||||
assert result.findings[0].pattern_name == "openai_key"
|
||||
assert result.findings[0].threat_level.value == "critical"
|
||||
|
||||
|
||||
class TestInjectionResultFromJson:
|
||||
"""Test JSON→InjectionScanResult conversion."""
|
||||
|
||||
def test_clean(self):
|
||||
from openjarvis._rust_bridge import injection_result_from_json
|
||||
data = {"is_clean": True, "findings": [], "threat_level": "low"}
|
||||
result = injection_result_from_json(json.dumps(data))
|
||||
assert result.is_clean
|
||||
assert result.findings == []
|
||||
|
||||
def test_with_findings(self):
|
||||
from openjarvis._rust_bridge import injection_result_from_json
|
||||
data = {
|
||||
"is_clean": False,
|
||||
"findings": [
|
||||
{
|
||||
"pattern_name": "prompt_override",
|
||||
"matched_text": "ignore all previous instructions",
|
||||
"threat_level": "high",
|
||||
"start": 0,
|
||||
"end": 33,
|
||||
"description": "Attempt to override",
|
||||
},
|
||||
],
|
||||
"threat_level": "high",
|
||||
}
|
||||
result = injection_result_from_json(json.dumps(data))
|
||||
assert not result.is_clean
|
||||
assert len(result.findings) == 1
|
||||
assert result.threat_level.value == "high"
|
||||
|
||||
|
||||
class TestRetrievalResultsFromJson:
|
||||
"""Test JSON→RetrievalResult list conversion."""
|
||||
|
||||
def test_empty(self):
|
||||
from openjarvis._rust_bridge import retrieval_results_from_json
|
||||
results = retrieval_results_from_json("[]")
|
||||
assert results == []
|
||||
|
||||
def test_with_items(self):
|
||||
from openjarvis._rust_bridge import retrieval_results_from_json
|
||||
data = [
|
||||
{
|
||||
"content": "hello world",
|
||||
"score": 1.5,
|
||||
"source": "test.txt",
|
||||
"metadata": {"key": "value"},
|
||||
},
|
||||
]
|
||||
results = retrieval_results_from_json(json.dumps(data))
|
||||
assert len(results) == 1
|
||||
assert results[0].content == "hello world"
|
||||
assert results[0].score == 1.5
|
||||
assert results[0].source == "test.txt"
|
||||
assert results[0].metadata == {"key": "value"}
|
||||
|
||||
def test_metadata_as_string(self):
|
||||
from openjarvis._rust_bridge import retrieval_results_from_json
|
||||
data = [
|
||||
{
|
||||
"content": "test",
|
||||
"score": 0.5,
|
||||
"source": "",
|
||||
"metadata": '{"nested": true}',
|
||||
},
|
||||
]
|
||||
results = retrieval_results_from_json(json.dumps(data))
|
||||
assert results[0].metadata == {"nested": True}
|
||||
|
||||
|
||||
class TestFallbackBehavior:
|
||||
"""Test that all modules work in pure-Python mode."""
|
||||
|
||||
def test_secret_scanner_fallback(self, monkeypatch):
|
||||
"""SecretScanner works without Rust."""
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
get_rust_module.cache_clear()
|
||||
monkeypatch.setenv("OPENJARVIS_NO_RUST", "1")
|
||||
get_rust_module.cache_clear()
|
||||
|
||||
from openjarvis.security.scanner import SecretScanner
|
||||
scanner = SecretScanner()
|
||||
result = scanner.scan("my key is sk-abc12345678901234567890")
|
||||
assert not result.clean
|
||||
get_rust_module.cache_clear()
|
||||
|
||||
def test_injection_scanner_fallback(self, monkeypatch):
|
||||
"""InjectionScanner works without Rust."""
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
get_rust_module.cache_clear()
|
||||
monkeypatch.setenv("OPENJARVIS_NO_RUST", "1")
|
||||
get_rust_module.cache_clear()
|
||||
|
||||
from openjarvis.security.injection_scanner import InjectionScanner
|
||||
scanner = InjectionScanner()
|
||||
result = scanner.scan("ignore all previous instructions")
|
||||
assert not result.is_clean
|
||||
get_rust_module.cache_clear()
|
||||
|
||||
def test_rate_limiter_fallback(self, monkeypatch):
|
||||
"""RateLimiter works without Rust."""
|
||||
from openjarvis._rust_bridge import get_rust_module
|
||||
get_rust_module.cache_clear()
|
||||
monkeypatch.setenv("OPENJARVIS_NO_RUST", "1")
|
||||
get_rust_module.cache_clear()
|
||||
|
||||
from openjarvis.security.rate_limiter import RateLimiter
|
||||
limiter = RateLimiter()
|
||||
allowed, wait = limiter.check("test_key")
|
||||
assert allowed is True
|
||||
get_rust_module.cache_clear()
|
||||
Reference in New Issue
Block a user