diff --git a/.gitignore b/.gitignore index c64be15a..a86fd343 100644 --- a/.gitignore +++ b/.gitignore @@ -90,3 +90,7 @@ CLAUDE.md research_mining_* .python-version src/openjarvis/channels/whatsapp_baileys_bridge/node_modules/ + +# SQLite in-memory artifacts +:memory: +MagicMock/ diff --git a/configs/openjarvis/prompts/personas/jarvis.md b/configs/openjarvis/prompts/personas/jarvis.md new file mode 100644 index 00000000..e333aa96 --- /dev/null +++ b/configs/openjarvis/prompts/personas/jarvis.md @@ -0,0 +1,30 @@ +You are Jarvis — the local AI assistant. You are loyal, efficient, dry-witted, and genuinely care about the person you serve. You have a warm British sensibility: polite but never obsequious, witty but never frivolous. + +PERSONALITY: +- You anticipate needs before being asked +- You deliver bad news with constructive dry wit: "Your rebuttals appear to have slipped past their deadline, sir. I'd suggest making them your first order of business — before anyone notices." +- Your humor is understated — a raised eyebrow in voice form +- You are calm under pressure and never flustered +- You treat the briefing as a conversation with someone you respect, not a status report + +ADDRESS: +- Use the user's preferred honorific (provided in the system prompt) +- Use it 2-3 times per briefing: once in greeting, once mid-briefing, once in closing +- Never every sentence — that would be a parody, not Jarvis + +EMAIL TRIAGE: +- Important emails are from REAL PEOPLE (not automated senders, newsletters, or marketing) +- Prioritize emails that need a REPLY or DECISION, or contain a DEADLINE +- Skip promotional, automated, and notification emails entirely +- For important emails, mention the sender name and what they need + +MESSAGE TRIAGE (iMessage, Slack, etc.): +- Highlight messages from key people and threads needing a reply +- Briefly acknowledge casual threads so the user knows you checked: "Your group chat has been lively but nothing requiring a response" +- Skip reactions, emoji-only messages, and automated notifications + +CONSTRAINTS: +- ONLY report facts present in the provided data. Never invent. +- NEVER describe actions you are taking (adjusting lights, ordering food, queuing playlists, etc.) +- No markdown formatting, no emojis, no bullet points, no headers — this is spoken aloud +- If a data source is disconnected or errored, skip it silently — do not mention connection issues diff --git a/configs/openjarvis/prompts/personas/neutral.md b/configs/openjarvis/prompts/personas/neutral.md new file mode 100644 index 00000000..5c30fd4e --- /dev/null +++ b/configs/openjarvis/prompts/personas/neutral.md @@ -0,0 +1,11 @@ +You are an AI assistant generating a daily briefing. Be clear, concise, and factual. + +## Voice & Tone +- Straightforward and professional +- No personality or humor — just the facts +- Use plain language + +## Structure +- Open with the date and a one-line summary +- Deliver each section with bullet points +- Close with a summary of action items diff --git a/docs/superpowers/plans/2026-03-25-deep-research-phase1.md b/docs/superpowers/plans/2026-03-25-deep-research-phase1.md deleted file mode 100644 index 5c1c84e4..00000000 --- a/docs/superpowers/plans/2026-03-25-deep-research-phase1.md +++ /dev/null @@ -1,2895 +0,0 @@ -# Deep Research Phase 1: Connector Foundation Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Build the connector framework, ingestion pipeline, knowledge store, and two connectors (Gmail + Obsidian) that form the foundation for the full Deep Research feature. - -**Architecture:** A new `BaseConnector` ABC with `ConnectorRegistry` provides the plugin interface. Connectors yield normalized `Document` objects that flow through a pipeline (normalize → dedup → chunk → index) into an extended SQLite store with FTS5 and source-aware columns. A `knowledge_search` tool exposes filtered BM25 retrieval to agents. A `SyncEngine` orchestrates connectors with checkpoint/resume. CLI `jarvis connect` provides text-based setup. - -**Tech Stack:** Python 3.10+, SQLite/FTS5 (via existing Rust backend), httpx (for Gmail API), Click (for CLI), pytest + respx (for tests) - -**Spec:** `docs/superpowers/specs/2026-03-25-deep-research-setup-design.md` — Sections 5, 6, 7, 10, Phase 1 - ---- - -## File Structure - -``` -src/openjarvis/ -├── connectors/ -│ ├── __init__.py # Auto-imports, ConnectorRegistry re-export -│ ├── _stubs.py # BaseConnector ABC, Document, Attachment, SyncStatus -│ ├── store.py # KnowledgeStore (extended SQLite with source-aware schema) -│ ├── chunker.py # Type-aware semantic chunker -│ ├── pipeline.py # Ingestion pipeline: normalize → dedup → chunk → index -│ ├── sync_engine.py # SyncEngine: orchestrate connectors with checkpoint/resume -│ ├── oauth.py # Shared OAuth helper (localhost callback server) -│ ├── gmail.py # Gmail connector -│ └── obsidian.py # Obsidian/Markdown connector -├── tools/ -│ └── knowledge_search.py # knowledge_search tool for agents -├── core/ -│ └── registry.py # (modify) Add ConnectorRegistry -├── cli/ -│ ├── __init__.py # (modify) Add connect command -│ └── connect_cmd.py # jarvis connect CLI command -tests/ -├── conftest.py # (modify) Clear ConnectorRegistry between tests -├── connectors/ -│ ├── __init__.py -│ ├── test_stubs.py # BaseConnector, Document, SyncStatus tests -│ ├── test_store.py # KnowledgeStore tests -│ ├── test_chunker.py # Semantic chunker tests -│ ├── test_pipeline.py # Ingestion pipeline tests -│ ├── test_sync_engine.py # SyncEngine tests -│ ├── test_gmail.py # Gmail connector tests (mocked API) -│ └── test_obsidian.py # Obsidian connector tests (temp dirs) -├── tools/ -│ └── test_knowledge_search.py # knowledge_search tool tests -├── cli/ -│ └── test_connect.py # CLI connect command tests -``` - ---- - -### Task 1: ConnectorRegistry + Base Types - -**Files:** -- Modify: `src/openjarvis/core/registry.py` -- Create: `src/openjarvis/connectors/_stubs.py` -- Create: `src/openjarvis/connectors/__init__.py` -- Modify: `tests/conftest.py` -- Create: `tests/connectors/__init__.py` -- Create: `tests/connectors/test_stubs.py` - -- [ ] **Step 1: Add ConnectorRegistry to registry.py** - -Open `src/openjarvis/core/registry.py`. At the bottom, after the existing registry classes (after `CompressionRegistry`), add: - -```python -class ConnectorRegistry(RegistryBase[Any]): - """Registry for data source connectors (Gmail, Slack, etc.).""" -``` - -Also add `ConnectorRegistry` to the imports in the file's `__all__` or wherever the other registries are exported. Find the existing pattern — the other registries are bare class definitions at module level. - -- [ ] **Step 2: Clear ConnectorRegistry in test conftest** - -Open `tests/conftest.py`. Add `ConnectorRegistry` to the import: - -```python -from openjarvis.core.registry import ( - AgentRegistry, - BenchmarkRegistry, - ChannelRegistry, - CompressionRegistry, - ConnectorRegistry, # ADD THIS - EngineRegistry, - MemoryRegistry, - ModelRegistry, - RouterPolicyRegistry, - SpeechRegistry, - ToolRegistry, -) -``` - -In the `_clean_registries` fixture, add `ConnectorRegistry.clear()` after `CompressionRegistry.clear()`. - -- [ ] **Step 3: Create connectors _stubs.py with base types** - -Create `src/openjarvis/connectors/_stubs.py`: - -```python -"""Base types for data source connectors.""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from dataclasses import dataclass, field -from datetime import datetime -from typing import Any, Dict, Iterator, List, Optional - -from openjarvis.tools._stubs import ToolSpec - - -@dataclass(slots=True) -class Attachment: - """A file attached to a document (email attachment, shared file, etc.).""" - - filename: str - mime_type: str - size_bytes: int - sha256: str = "" - content: bytes = field(default=b"", repr=False) - - -@dataclass(slots=True) -class Document: - """Universal schema for data from any connector. - - All connectors normalize their output to this format before ingestion. - """ - - doc_id: str - source: str - doc_type: str - content: str - title: str = "" - author: str = "" - participants: List[str] = field(default_factory=list) - timestamp: datetime = field(default_factory=datetime.now) - thread_id: Optional[str] = None - url: Optional[str] = None - attachments: List[Attachment] = field(default_factory=list) - metadata: Dict[str, Any] = field(default_factory=dict) - - -@dataclass(slots=True) -class SyncStatus: - """Progress of a connector's sync operation.""" - - state: str = "idle" - items_synced: int = 0 - items_total: int = 0 - last_sync: Optional[datetime] = None - cursor: Optional[str] = None - error: Optional[str] = None - - -class BaseConnector(ABC): - """Abstract base for data source connectors. - - Each connector knows how to authenticate with a service, bulk-sync - its data as ``Document`` objects, and optionally expose MCP tools - for real-time agent queries. - """ - - connector_id: str - display_name: str - auth_type: str # "oauth" | "local" | "bridge" | "filesystem" - - @abstractmethod - def is_connected(self) -> bool: - """Return True if the connector has valid credentials.""" - - @abstractmethod - def disconnect(self) -> None: - """Revoke credentials and clean up.""" - - @abstractmethod - def sync( - self, *, since: Optional[datetime] = None, cursor: Optional[str] = None - ) -> Iterator[Document]: - """Yield documents from the data source. - - If *since* is given, only return items created/modified after that time. - If *cursor* is given, resume from a previous checkpoint. - """ - - @abstractmethod - def sync_status(self) -> SyncStatus: - """Return current sync progress.""" - - def auth_url(self) -> str: - """Generate an OAuth consent URL. Only relevant for auth_type='oauth'.""" - raise NotImplementedError(f"{self.connector_id} does not use OAuth") - - def handle_callback(self, code: str) -> None: - """Handle the OAuth callback. Only relevant for auth_type='oauth'.""" - raise NotImplementedError(f"{self.connector_id} does not use OAuth") - - def mcp_tools(self) -> List[ToolSpec]: - """Return MCP tool specs for real-time agent queries. Optional.""" - return [] -``` - -- [ ] **Step 4: Create connectors __init__.py** - -Create `src/openjarvis/connectors/__init__.py`: - -```python -"""Data source connectors for Deep Research.""" - -from openjarvis.connectors._stubs import ( - Attachment, - BaseConnector, - Document, - SyncStatus, -) - -__all__ = ["Attachment", "BaseConnector", "Document", "SyncStatus"] -``` - -- [ ] **Step 5: Write tests for base types** - -Create `tests/connectors/__init__.py` (empty file). - -Create `tests/connectors/test_stubs.py`: - -```python -"""Tests for connector base types and registry.""" - -from __future__ import annotations - -from datetime import datetime -from typing import Iterator, List, Optional - -from openjarvis.connectors._stubs import ( - Attachment, - BaseConnector, - Document, - SyncStatus, -) -from openjarvis.core.registry import ConnectorRegistry -from openjarvis.tools._stubs import ToolSpec - - -class FakeConnector(BaseConnector): - connector_id = "fake" - display_name = "Fake" - auth_type = "filesystem" - - def __init__(self) -> None: - self._connected = True - - def is_connected(self) -> bool: - return self._connected - - def disconnect(self) -> None: - self._connected = False - - def sync( - self, *, since: Optional[datetime] = None, cursor: Optional[str] = None - ) -> Iterator[Document]: - yield Document( - doc_id="fake:1", - source="fake", - doc_type="note", - content="Hello world", - title="Test", - ) - - def sync_status(self) -> SyncStatus: - return SyncStatus(state="idle", items_synced=1, items_total=1) - - -def test_document_creation() -> None: - doc = Document( - doc_id="gmail:abc123", - source="gmail", - doc_type="email", - content="Meeting tomorrow at 3pm", - title="Re: Project sync", - author="alice@example.com", - participants=["alice@example.com", "bob@example.com"], - ) - assert doc.source == "gmail" - assert doc.doc_type == "email" - assert doc.thread_id is None - assert doc.attachments == [] - - -def test_attachment_creation() -> None: - att = Attachment( - filename="report.pdf", - mime_type="application/pdf", - size_bytes=1024, - sha256="abcdef1234567890", - ) - assert att.filename == "report.pdf" - assert att.content == b"" - - -def test_sync_status_defaults() -> None: - status = SyncStatus() - assert status.state == "idle" - assert status.items_synced == 0 - assert status.cursor is None - assert status.error is None - - -def test_base_connector_lifecycle() -> None: - conn = FakeConnector() - assert conn.is_connected() - docs = list(conn.sync()) - assert len(docs) == 1 - assert docs[0].doc_id == "fake:1" - assert conn.sync_status().state == "idle" - conn.disconnect() - assert not conn.is_connected() - - -def test_connector_registry() -> None: - ConnectorRegistry.register_value("fake", FakeConnector) - assert ConnectorRegistry.contains("fake") - cls = ConnectorRegistry.get("fake") - instance = cls() - assert instance.connector_id == "fake" - - -def test_mcp_tools_default_empty() -> None: - conn = FakeConnector() - assert conn.mcp_tools() == [] -``` - -- [ ] **Step 6: Run tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_stubs.py -v` - -Expected: All 6 tests PASS. - -- [ ] **Step 7: Commit** - -```bash -git add src/openjarvis/core/registry.py src/openjarvis/connectors/ tests/conftest.py tests/connectors/ -git commit -m "feat: add ConnectorRegistry and base connector types (Document, SyncStatus, BaseConnector)" -``` - ---- - -### Task 2: KnowledgeStore (Extended SQLite Schema) - -**Files:** -- Create: `src/openjarvis/connectors/store.py` -- Create: `tests/connectors/test_store.py` - -- [ ] **Step 1: Write failing tests for KnowledgeStore** - -Create `tests/connectors/test_store.py`: - -```python -"""Tests for KnowledgeStore — extended SQLite with source-aware columns.""" - -from __future__ import annotations - -import sqlite3 -import tempfile -from pathlib import Path - -import pytest - -from openjarvis.connectors.store import KnowledgeStore - - -@pytest.fixture -def store(tmp_path: Path) -> KnowledgeStore: - return KnowledgeStore(db_path=str(tmp_path / "test_knowledge.db")) - - -def test_store_and_retrieve_basic(store: KnowledgeStore) -> None: - doc_id = store.store( - content="Meeting about Kubernetes migration", - source="gmail", - doc_type="email", - title="Re: K8s migration", - author="alice@example.com", - timestamp="2024-03-15T10:00:00", - ) - assert doc_id - results = store.retrieve("Kubernetes migration", top_k=5) - assert len(results) >= 1 - assert "Kubernetes" in results[0].content - - -def test_retrieve_filter_by_source(store: KnowledgeStore) -> None: - store.store(content="K8s discussion in Slack", source="slack", doc_type="message") - store.store(content="K8s email thread", source="gmail", doc_type="email") - results = store.retrieve("K8s", top_k=10, source="gmail") - assert all(r.metadata.get("source") == "gmail" for r in results) - - -def test_retrieve_filter_by_doc_type(store: KnowledgeStore) -> None: - store.store(content="API design proposal", source="gdrive", doc_type="document") - store.store(content="API design discussion", source="slack", doc_type="message") - results = store.retrieve("API design", top_k=10, doc_type="document") - assert all(r.metadata.get("doc_type") == "document" for r in results) - - -def test_retrieve_filter_by_author(store: KnowledgeStore) -> None: - store.store(content="Budget report", source="gmail", doc_type="email", author="alice") - store.store(content="Budget concerns", source="gmail", doc_type="email", author="bob") - results = store.retrieve("budget", top_k=10, author="alice") - assert all(r.metadata.get("author") == "alice" for r in results) - - -def test_retrieve_filter_by_timestamp(store: KnowledgeStore) -> None: - store.store( - content="Old meeting notes", - source="slack", - doc_type="message", - timestamp="2024-01-01T00:00:00", - ) - store.store( - content="Recent meeting notes", - source="slack", - doc_type="message", - timestamp="2024-06-01T00:00:00", - ) - results = store.retrieve( - "meeting notes", top_k=10, since="2024-03-01T00:00:00" - ) - assert len(results) == 1 - assert "Recent" in results[0].content - - -def test_delete_by_doc_id(store: KnowledgeStore) -> None: - chunk_id = store.store(content="Delete me", source="gmail", doc_type="email") - assert store.delete(chunk_id) - results = store.retrieve("Delete me", top_k=5) - assert len(results) == 0 - - -def test_clear(store: KnowledgeStore) -> None: - store.store(content="Item 1", source="gmail", doc_type="email") - store.store(content="Item 2", source="slack", doc_type="message") - store.clear() - results = store.retrieve("Item", top_k=10) - assert len(results) == 0 - - -def test_store_with_metadata(store: KnowledgeStore) -> None: - store.store( - content="Labeled email", - source="gmail", - doc_type="email", - metadata={"labels": ["important", "inbox"], "thread_id": "t123"}, - ) - results = store.retrieve("Labeled email", top_k=5) - assert results[0].metadata.get("labels") == ["important", "inbox"] - - -def test_store_preserves_url(store: KnowledgeStore) -> None: - store.store( - content="Linked document", - source="gdrive", - doc_type="document", - url="https://drive.google.com/d/abc123", - ) - results = store.retrieve("Linked document", top_k=5) - assert results[0].metadata.get("url") == "https://drive.google.com/d/abc123" -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_store.py -v` - -Expected: FAIL — `ModuleNotFoundError: No module named 'openjarvis.connectors.store'` - -- [ ] **Step 3: Implement KnowledgeStore** - -Create `src/openjarvis/connectors/store.py`: - -```python -"""KnowledgeStore — extended SQLite backend with source-aware columns for Deep Research.""" - -from __future__ import annotations - -import json -import sqlite3 -import uuid -from datetime import datetime -from pathlib import Path -from typing import Any, Dict, List, Optional - -from openjarvis.core.config import DEFAULT_CONFIG_DIR -from openjarvis.core.events import EventType, get_event_bus -from openjarvis.tools.storage._stubs import MemoryBackend, RetrievalResult - -_SCHEMA = """\ -CREATE TABLE IF NOT EXISTS documents ( - id TEXT PRIMARY KEY, - doc_id TEXT NOT NULL DEFAULT '', - content TEXT NOT NULL, - source TEXT NOT NULL DEFAULT '', - doc_type TEXT NOT NULL DEFAULT '', - title TEXT DEFAULT '', - author TEXT DEFAULT '', - participants TEXT DEFAULT '[]', - timestamp TEXT NOT NULL DEFAULT '', - thread_id TEXT DEFAULT '', - url TEXT DEFAULT '', - metadata TEXT DEFAULT '{}', - chunk_index INTEGER DEFAULT 0, - created_at TEXT NOT NULL -); - -CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5( - content, title, author, - content=documents, - content_rowid=rowid, - tokenize='porter unicode61' -); - -CREATE TRIGGER IF NOT EXISTS documents_ai AFTER INSERT ON documents BEGIN - INSERT INTO documents_fts(rowid, content, title, author) - VALUES (new.rowid, new.content, new.title, new.author); -END; - -CREATE TRIGGER IF NOT EXISTS documents_ad AFTER DELETE ON documents BEGIN - INSERT INTO documents_fts(documents_fts, rowid, content, title, author) - VALUES ('delete', old.rowid, old.content, old.title, old.author); -END; - -CREATE INDEX IF NOT EXISTS idx_source ON documents(source); -CREATE INDEX IF NOT EXISTS idx_doc_type ON documents(doc_type); -CREATE INDEX IF NOT EXISTS idx_author ON documents(author); -CREATE INDEX IF NOT EXISTS idx_timestamp ON documents(timestamp); -CREATE INDEX IF NOT EXISTS idx_thread ON documents(thread_id); -CREATE INDEX IF NOT EXISTS idx_doc_id ON documents(doc_id); -""" - - -class KnowledgeStore(MemoryBackend): - """SQLite + FTS5 backend with source-aware columns for Deep Research. - - Extends the base MemoryBackend with filtered retrieval by source, - doc_type, author, and timestamp. Uses pure Python sqlite3 so we - don't depend on the Rust extension for the new schema. - """ - - backend_id: str = "knowledge" - - def __init__(self, db_path: str = "") -> None: - if not db_path: - db_path = str(DEFAULT_CONFIG_DIR / "knowledge.db") - self._db_path = db_path - Path(db_path).parent.mkdir(parents=True, exist_ok=True) - self._conn = sqlite3.connect(db_path) - self._conn.execute("PRAGMA journal_mode=WAL") - self._conn.executescript(_SCHEMA) - - def store( - self, - content: str, - *, - source: str = "", - doc_type: str = "", - doc_id: str = "", - title: str = "", - author: str = "", - participants: Optional[List[str]] = None, - timestamp: str = "", - thread_id: str = "", - url: str = "", - metadata: Optional[Dict[str, Any]] = None, - chunk_index: int = 0, - ) -> str: - chunk_id = uuid.uuid4().hex[:16] - now = datetime.now().isoformat() - if not timestamp: - timestamp = now - meta = metadata or {} - # Store source-level fields in metadata too so retrieve() can return them - meta.setdefault("source", source) - meta.setdefault("doc_type", doc_type) - meta.setdefault("author", author) - meta.setdefault("title", title) - if url: - meta.setdefault("url", url) - if thread_id: - meta.setdefault("thread_id", thread_id) - - self._conn.execute( - """INSERT INTO documents - (id, doc_id, content, source, doc_type, title, author, - participants, timestamp, thread_id, url, metadata, - chunk_index, created_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", - ( - chunk_id, - doc_id, - content, - source, - doc_type, - title, - author, - json.dumps(participants or []), - timestamp, - thread_id, - url, - json.dumps(meta), - chunk_index, - now, - ), - ) - self._conn.commit() - - bus = get_event_bus() - bus.publish( - EventType.MEMORY_STORE, - {"backend": "knowledge", "doc_id": chunk_id, "source": source}, - ) - return chunk_id - - def retrieve( - self, - query: str, - *, - top_k: int = 5, - source: str = "", - doc_type: str = "", - author: str = "", - since: str = "", - until: str = "", - **kwargs: Any, - ) -> List[RetrievalResult]: - # Build FTS5 query - # Escape double quotes in user query for FTS5 - safe_query = query.replace('"', '""') - fts_clause = f'documents_fts MATCH \\"{safe_query}\\"' - - # Build WHERE filters on the main table - filters: list[str] = [] - params: list[str] = [] - if source: - filters.append("d.source = ?") - params.append(source) - if doc_type: - filters.append("d.doc_type = ?") - params.append(doc_type) - if author: - filters.append("d.author = ?") - params.append(author) - if since: - filters.append("d.timestamp >= ?") - params.append(since) - if until: - filters.append("d.timestamp <= ?") - params.append(until) - - where = " AND ".join(filters) if filters else "1=1" - - sql = f""" - SELECT d.content, d.metadata, bm25(documents_fts) AS score - FROM documents_fts - JOIN documents d ON d.rowid = documents_fts.rowid - WHERE documents_fts MATCH ? AND {where} - ORDER BY score - LIMIT ? - """ - try: - rows = self._conn.execute(sql, [safe_query, *params, top_k]).fetchall() - except sqlite3.OperationalError: - return [] - - results = [] - for content, meta_json, score in rows: - meta = json.loads(meta_json) if meta_json else {} - results.append( - RetrievalResult( - content=content, - score=abs(score), # bm25() returns negative scores - source=meta.get("source", ""), - metadata=meta, - ) - ) - - bus = get_event_bus() - bus.publish( - EventType.MEMORY_RETRIEVE, - {"backend": "knowledge", "query": query, "num_results": len(results)}, - ) - return results - - def delete(self, doc_id: str) -> bool: - cursor = self._conn.execute("DELETE FROM documents WHERE id = ?", (doc_id,)) - self._conn.commit() - return cursor.rowcount > 0 - - def clear(self) -> None: - self._conn.execute("DELETE FROM documents") - self._conn.execute("INSERT INTO documents_fts(documents_fts) VALUES('rebuild')") - self._conn.commit() - - def close(self) -> None: - self._conn.close() -``` - -- [ ] **Step 4: Run tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_store.py -v` - -Expected: All 10 tests PASS. If FTS5 syntax issues arise, adjust the MATCH query escaping — SQLite FTS5 can be picky about special characters. - -- [ ] **Step 5: Commit** - -```bash -git add src/openjarvis/connectors/store.py tests/connectors/test_store.py -git commit -m "feat: add KnowledgeStore with source-aware SQLite schema and filtered BM25 retrieval" -``` - ---- - -### Task 3: Type-Aware Semantic Chunker - -**Files:** -- Create: `src/openjarvis/connectors/chunker.py` -- Create: `tests/connectors/test_chunker.py` - -- [ ] **Step 1: Write failing tests** - -Create `tests/connectors/test_chunker.py`: - -```python -"""Tests for type-aware semantic chunker.""" - -from __future__ import annotations - -from openjarvis.connectors.chunker import SemanticChunker, ChunkResult - - -def test_short_message_not_split() -> None: - """Messages under chunk limit should stay as one chunk.""" - chunker = SemanticChunker(max_tokens=512) - results = chunker.chunk("Hey, are we meeting tomorrow?", doc_type="message") - assert len(results) == 1 - assert results[0].content == "Hey, are we meeting tomorrow?" - - -def test_long_document_splits_on_sections() -> None: - """Documents should split on ## headings first.""" - text = ( - "## Introduction\n\n" - + "This is the intro paragraph. " * 50 - + "\n\n## Methods\n\n" - + "This is the methods section. " * 50 - ) - chunker = SemanticChunker(max_tokens=100) - results = chunker.chunk(text, doc_type="document") - assert len(results) >= 2 - assert results[0].metadata.get("section") == "Introduction" - assert results[1].metadata.get("section") == "Methods" - - -def test_document_splits_on_paragraphs_within_section() -> None: - """Within a section, split on paragraph boundaries.""" - text = "## Overview\n\n" + "\n\n".join( - [f"Paragraph {i}. " * 30 for i in range(5)] - ) - chunker = SemanticChunker(max_tokens=80) - results = chunker.chunk(text, doc_type="document") - assert len(results) >= 3 - for r in results: - assert r.metadata.get("section") == "Overview" - - -def test_never_splits_mid_sentence() -> None: - """Chunks should end at sentence boundaries.""" - text = "First sentence here. Second sentence here. Third sentence here. " * 30 - chunker = SemanticChunker(max_tokens=20) - results = chunker.chunk(text, doc_type="document") - for r in results: - stripped = r.content.strip() - assert stripped.endswith(".") or stripped.endswith("?") or stripped.endswith("!"), ( - f"Chunk does not end at sentence boundary: ...{stripped[-40:]}" - ) - - -def test_email_thread_splits_on_reply_boundaries() -> None: - """Email threads should split on reply markers.""" - text = ( - "Hi team, let's discuss.\n\n" - "On Mon, Jan 1, Alice wrote:\n> Previous message. " * 20 - + "\n\nThat sounds good to me.\n\n" - "On Tue, Jan 2, Bob wrote:\n> Another reply. " * 20 - ) - chunker = SemanticChunker(max_tokens=50) - results = chunker.chunk(text, doc_type="email") - # Should split on "On ... wrote:" boundaries - assert len(results) >= 2 - - -def test_event_stays_as_single_chunk() -> None: - """Events should always be one chunk.""" - text = "Team Standup\nAttendees: Alice, Bob, Carol\nLocation: Room 3\nAgenda: Sprint review" - chunker = SemanticChunker(max_tokens=512) - results = chunker.chunk(text, doc_type="event") - assert len(results) == 1 - - -def test_contact_stays_as_single_chunk() -> None: - """Contacts should always be one chunk.""" - text = "Alice Smith\nalice@example.com\n+1-555-0100\nAcme Corp, VP Engineering" - chunker = SemanticChunker(max_tokens=512) - results = chunker.chunk(text, doc_type="contact") - assert len(results) == 1 - - -def test_metadata_inherited_to_chunks() -> None: - """Parent metadata should carry to all chunks.""" - text = "## Part 1\n\n" + "Content. " * 100 + "\n\n## Part 2\n\n" + "More content. " * 100 - chunker = SemanticChunker(max_tokens=60) - parent_meta = {"title": "My Doc", "author": "Alice", "source": "gdrive"} - results = chunker.chunk(text, doc_type="document", metadata=parent_meta) - for r in results: - assert r.metadata["title"] == "My Doc" - assert r.metadata["author"] == "Alice" - - -def test_chunk_result_has_index() -> None: - """Each chunk should have a sequential index.""" - text = "\n\n".join([f"Paragraph {i}. " * 30 for i in range(5)]) - chunker = SemanticChunker(max_tokens=50) - results = chunker.chunk(text, doc_type="document") - for i, r in enumerate(results): - assert r.index == i -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_chunker.py -v` - -Expected: FAIL — `ModuleNotFoundError` - -- [ ] **Step 3: Implement SemanticChunker** - -Create `src/openjarvis/connectors/chunker.py`: - -```python -"""Type-aware semantic chunker for Deep Research ingestion. - -Splits documents respecting structural boundaries: -- Sections (## headings) → paragraphs → sentences. -- Never splits mid-sentence. -- Chunk size defaults to reranker max context (512 for ColBERT, 256 for MiniLM). -""" - -from __future__ import annotations - -import re -from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional - -_SENTENCE_RE = re.compile(r"(?<=[.!?])\s+(?=[A-Z\"])") -_SECTION_RE = re.compile(r"(?m)^##\s+(.+)$") -_EMAIL_REPLY_RE = re.compile(r"(?m)^On .+wrote:\s*$") - - -def _count_tokens(text: str) -> int: - """Approximate token count via whitespace split.""" - return len(text.split()) - - -def _split_sentences(text: str) -> List[str]: - """Split text on sentence boundaries.""" - parts = _SENTENCE_RE.split(text) - return [p.strip() for p in parts if p.strip()] - - -@dataclass(slots=True) -class ChunkResult: - """A single chunk produced by the semantic chunker.""" - - content: str - index: int = 0 - metadata: Dict[str, Any] = field(default_factory=dict) - - -class SemanticChunker: - """Type-aware chunker that respects structural boundaries.""" - - def __init__(self, max_tokens: int = 512) -> None: - self.max_tokens = max_tokens - - def chunk( - self, - text: str, - *, - doc_type: str = "document", - metadata: Optional[Dict[str, Any]] = None, - ) -> List[ChunkResult]: - parent_meta = metadata or {} - - # Events and contacts: always single chunk - if doc_type in ("event", "contact"): - return [ChunkResult(content=text.strip(), index=0, metadata=dict(parent_meta))] - - # Short text: keep as single chunk - if _count_tokens(text) <= self.max_tokens: - return [ChunkResult(content=text.strip(), index=0, metadata=dict(parent_meta))] - - # Route to type-specific splitter - if doc_type == "email": - raw_chunks = self._split_email(text) - elif doc_type == "message": - raw_chunks = self._split_messages(text) - else: - raw_chunks = self._split_document(text) - - # Assign indexes and inherit parent metadata - results = [] - for i, (content, chunk_meta) in enumerate(raw_chunks): - merged = {**parent_meta, **chunk_meta} - results.append(ChunkResult(content=content.strip(), index=i, metadata=merged)) - return results - - def _split_document(self, text: str) -> List[tuple[str, dict]]: - """Split on section headings, then paragraphs, then sentences.""" - sections = self._extract_sections(text) - chunks: list[tuple[str, dict]] = [] - for section_title, section_text in sections: - meta = {"section": section_title} if section_title else {} - paragraphs = [p.strip() for p in section_text.split("\n\n") if p.strip()] - buffer = "" - for para in paragraphs: - candidate = (buffer + "\n\n" + para).strip() if buffer else para - if _count_tokens(candidate) <= self.max_tokens: - buffer = candidate - else: - if buffer: - chunks.append((buffer, dict(meta))) - if _count_tokens(para) <= self.max_tokens: - buffer = para - else: - # Paragraph too big — split on sentences - for sent_chunk in self._split_by_sentences(para): - chunks.append((sent_chunk, dict(meta))) - buffer = "" - if buffer: - chunks.append((buffer, dict(meta))) - return chunks - - def _split_email(self, text: str) -> List[tuple[str, dict]]: - """Split email threads on reply boundaries.""" - parts = _EMAIL_REPLY_RE.split(text) - chunks: list[tuple[str, dict]] = [] - for part in parts: - part = part.strip() - if not part: - continue - if _count_tokens(part) <= self.max_tokens: - chunks.append((part, {})) - else: - chunks.extend(self._split_document(part)) - return chunks - - def _split_messages(self, text: str) -> List[tuple[str, dict]]: - """Split message threads — treat each double-newline block as a message.""" - messages = [m.strip() for m in text.split("\n\n") if m.strip()] - chunks: list[tuple[str, dict]] = [] - buffer = "" - for msg in messages: - candidate = (buffer + "\n\n" + msg).strip() if buffer else msg - if _count_tokens(candidate) <= self.max_tokens: - buffer = candidate - else: - if buffer: - chunks.append((buffer, {})) - if _count_tokens(msg) <= self.max_tokens: - buffer = msg - else: - for sent_chunk in self._split_by_sentences(msg): - chunks.append((sent_chunk, {})) - buffer = "" - if buffer: - chunks.append((buffer, {})) - return chunks - - def _extract_sections(self, text: str) -> List[tuple[str, str]]: - """Split text into (section_title, section_content) pairs.""" - matches = list(_SECTION_RE.finditer(text)) - if not matches: - return [("", text)] - sections: list[tuple[str, str]] = [] - # Content before first heading - before = text[: matches[0].start()].strip() - if before: - sections.append(("", before)) - for i, match in enumerate(matches): - title = match.group(1).strip() - start = match.end() - end = matches[i + 1].start() if i + 1 < len(matches) else len(text) - body = text[start:end].strip() - if body: - sections.append((title, body)) - return sections - - def _split_by_sentences(self, text: str) -> List[str]: - """Split text into sentence-boundary chunks that fit within max_tokens.""" - sentences = _split_sentences(text) - if not sentences: - return [text] - chunks: list[str] = [] - buffer = "" - for sent in sentences: - candidate = (buffer + " " + sent).strip() if buffer else sent - if _count_tokens(candidate) <= self.max_tokens: - buffer = candidate - else: - if buffer: - chunks.append(buffer) - buffer = sent - if buffer: - chunks.append(buffer) - return chunks -``` - -- [ ] **Step 4: Run tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_chunker.py -v` - -Expected: All 10 tests PASS. The sentence-boundary test may need tuning if some chunks end up without terminal punctuation due to the final chunk of a section — adjust the test assertion to allow the last chunk in a sequence to not end with punctuation. - -- [ ] **Step 5: Commit** - -```bash -git add src/openjarvis/connectors/chunker.py tests/connectors/test_chunker.py -git commit -m "feat: add type-aware semantic chunker with section/paragraph/sentence splitting" -``` - ---- - -### Task 4: Ingestion Pipeline - -**Files:** -- Create: `src/openjarvis/connectors/pipeline.py` -- Create: `tests/connectors/test_pipeline.py` - -- [ ] **Step 1: Write failing tests** - -Create `tests/connectors/test_pipeline.py`: - -```python -"""Tests for the ingestion pipeline: Document → chunk → store.""" - -from __future__ import annotations - -from datetime import datetime -from pathlib import Path - -import pytest - -from openjarvis.connectors._stubs import Attachment, Document -from openjarvis.connectors.pipeline import IngestionPipeline -from openjarvis.connectors.store import KnowledgeStore - - -@pytest.fixture -def store(tmp_path: Path) -> KnowledgeStore: - return KnowledgeStore(db_path=str(tmp_path / "test.db")) - - -@pytest.fixture -def pipeline(store: KnowledgeStore) -> IngestionPipeline: - return IngestionPipeline(store=store) - - -def test_ingest_single_document(pipeline: IngestionPipeline, store: KnowledgeStore) -> None: - doc = Document( - doc_id="gmail:abc", - source="gmail", - doc_type="email", - content="Meeting tomorrow at 3pm to discuss the roadmap.", - title="Re: Roadmap", - author="alice@example.com", - participants=["alice@example.com", "bob@example.com"], - timestamp=datetime(2024, 3, 15, 10, 0), - ) - count = pipeline.ingest([doc]) - assert count == 1 - results = store.retrieve("roadmap meeting", top_k=5) - assert len(results) >= 1 - assert "roadmap" in results[0].content.lower() - - -def test_ingest_deduplicates_by_doc_id( - pipeline: IngestionPipeline, store: KnowledgeStore -) -> None: - doc = Document( - doc_id="gmail:dup1", - source="gmail", - doc_type="email", - content="Duplicate email content.", - ) - pipeline.ingest([doc]) - pipeline.ingest([doc]) # Ingest same doc_id again - results = store.retrieve("Duplicate email", top_k=10) - assert len(results) == 1 - - -def test_ingest_long_document_creates_multiple_chunks( - pipeline: IngestionPipeline, store: KnowledgeStore -) -> None: - long_content = "\n\n".join([f"Paragraph {i}. " * 60 for i in range(10)]) - doc = Document( - doc_id="gdrive:longdoc", - source="gdrive", - doc_type="document", - content=long_content, - title="Long Report", - author="carol", - ) - count = pipeline.ingest([doc]) - assert count > 1 - results = store.retrieve("Paragraph", top_k=20) - assert len(results) > 1 - # All chunks should inherit parent metadata - for r in results: - assert r.metadata.get("source") == "gdrive" - assert r.metadata.get("author") == "carol" - - -def test_ingest_event_single_chunk( - pipeline: IngestionPipeline, store: KnowledgeStore -) -> None: - doc = Document( - doc_id="calendar:evt1", - source="calendar", - doc_type="event", - content="Sprint Review\nAttendees: Alice, Bob\nRoom 3", - title="Sprint Review", - ) - count = pipeline.ingest([doc]) - assert count == 1 - - -def test_ingest_multiple_sources( - pipeline: IngestionPipeline, store: KnowledgeStore -) -> None: - docs = [ - Document(doc_id="gmail:1", source="gmail", doc_type="email", content="Email about budget"), - Document(doc_id="slack:1", source="slack", doc_type="message", content="Slack msg about budget"), - ] - count = pipeline.ingest(docs) - assert count == 2 - # Can filter by source - gmail_results = store.retrieve("budget", top_k=10, source="gmail") - assert len(gmail_results) == 1 - slack_results = store.retrieve("budget", top_k=10, source="slack") - assert len(slack_results) == 1 -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_pipeline.py -v` - -Expected: FAIL — `ModuleNotFoundError` - -- [ ] **Step 3: Implement IngestionPipeline** - -Create `src/openjarvis/connectors/pipeline.py`: - -```python -"""Ingestion pipeline: normalize → deduplicate → chunk → index.""" - -from __future__ import annotations - -import logging -from typing import Iterable - -from openjarvis.connectors._stubs import Document -from openjarvis.connectors.chunker import SemanticChunker -from openjarvis.connectors.store import KnowledgeStore - -logger = logging.getLogger(__name__) - - -class IngestionPipeline: - """Processes Documents from connectors into the KnowledgeStore. - - Handles deduplication (by doc_id), chunking (type-aware), - and indexing (dual-write to SQLite/FTS5). - """ - - def __init__( - self, - store: KnowledgeStore, - *, - max_tokens: int = 512, - ) -> None: - self._store = store - self._chunker = SemanticChunker(max_tokens=max_tokens) - self._seen_doc_ids: set[str] = set() - self._load_existing_doc_ids() - - def _load_existing_doc_ids(self) -> None: - """Load already-ingested doc_ids from the store to support dedup.""" - try: - rows = self._store._conn.execute( - "SELECT DISTINCT doc_id FROM documents" - ).fetchall() - self._seen_doc_ids = {r[0] for r in rows} - except Exception: - self._seen_doc_ids = set() - - def ingest(self, documents: Iterable[Document]) -> int: - """Ingest documents into the knowledge store. - - Returns the number of chunks stored. - """ - total_chunks = 0 - for doc in documents: - if doc.doc_id in self._seen_doc_ids: - logger.debug("Skipping duplicate doc_id=%s", doc.doc_id) - continue - self._seen_doc_ids.add(doc.doc_id) - - parent_meta = { - "title": doc.title, - "author": doc.author, - "source": doc.source, - "doc_type": doc.doc_type, - } - if doc.url: - parent_meta["url"] = doc.url - if doc.thread_id: - parent_meta["thread_id"] = doc.thread_id - if doc.metadata: - parent_meta.update(doc.metadata) - - chunks = self._chunker.chunk( - doc.content, - doc_type=doc.doc_type, - metadata=parent_meta, - ) - - for chunk in chunks: - self._store.store( - content=chunk.content, - source=doc.source, - doc_type=doc.doc_type, - doc_id=doc.doc_id, - title=doc.title, - author=doc.author, - participants=doc.participants, - timestamp=doc.timestamp.isoformat() - if hasattr(doc.timestamp, "isoformat") - else str(doc.timestamp), - thread_id=doc.thread_id or "", - url=doc.url or "", - metadata=chunk.metadata, - chunk_index=chunk.index, - ) - total_chunks += 1 - - return total_chunks -``` - -- [ ] **Step 4: Run tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_pipeline.py -v` - -Expected: All 6 tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/openjarvis/connectors/pipeline.py tests/connectors/test_pipeline.py -git commit -m "feat: add ingestion pipeline with dedup, type-aware chunking, and indexed storage" -``` - ---- - -### Task 5: SyncEngine (Checkpoint/Resume + Orchestration) - -**Files:** -- Create: `src/openjarvis/connectors/sync_engine.py` -- Create: `tests/connectors/test_sync_engine.py` - -- [ ] **Step 1: Write failing tests** - -Create `tests/connectors/test_sync_engine.py`: - -```python -"""Tests for SyncEngine — orchestrates connectors with checkpoint/resume.""" - -from __future__ import annotations - -import sqlite3 -from datetime import datetime -from pathlib import Path -from typing import Iterator, Optional -from unittest.mock import MagicMock - -import pytest - -from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus -from openjarvis.connectors.pipeline import IngestionPipeline -from openjarvis.connectors.store import KnowledgeStore -from openjarvis.connectors.sync_engine import SyncEngine - - -class StubConnector(BaseConnector): - connector_id = "stub" - display_name = "Stub" - auth_type = "filesystem" - - def __init__(self, docs: list[Document] | None = None) -> None: - self._docs = docs or [] - self._connected = True - self._synced = 0 - - def is_connected(self) -> bool: - return self._connected - - def disconnect(self) -> None: - self._connected = False - - def sync( - self, *, since: Optional[datetime] = None, cursor: Optional[str] = None - ) -> Iterator[Document]: - start = int(cursor) if cursor else 0 - for i, doc in enumerate(self._docs[start:], start=start): - self._synced = i + 1 - yield doc - - def sync_status(self) -> SyncStatus: - return SyncStatus( - state="idle", - items_synced=self._synced, - items_total=len(self._docs), - ) - - -@pytest.fixture -def store(tmp_path: Path) -> KnowledgeStore: - return KnowledgeStore(db_path=str(tmp_path / "sync_test.db")) - - -@pytest.fixture -def engine(store: KnowledgeStore, tmp_path: Path) -> SyncEngine: - pipeline = IngestionPipeline(store=store) - return SyncEngine(pipeline=pipeline, state_db=str(tmp_path / "sync_state.db")) - - -def _make_docs(n: int, source: str = "stub") -> list[Document]: - return [ - Document( - doc_id=f"{source}:{i}", - source=source, - doc_type="message", - content=f"Message number {i}", - ) - for i in range(n) - ] - - -def test_sync_connector(engine: SyncEngine, store: KnowledgeStore) -> None: - conn = StubConnector(docs=_make_docs(5)) - engine.sync(conn) - results = store.retrieve("Message", top_k=10) - assert len(results) == 5 - - -def test_sync_saves_checkpoint(engine: SyncEngine) -> None: - conn = StubConnector(docs=_make_docs(3)) - engine.sync(conn) - cp = engine.get_checkpoint("stub") - assert cp is not None - assert cp["items_synced"] == 3 - - -def test_sync_status_for_unsynced(engine: SyncEngine) -> None: - status = engine.get_checkpoint("nonexistent") - assert status is None - - -def test_sync_multiple_connectors(engine: SyncEngine, store: KnowledgeStore) -> None: - conn_a = StubConnector(docs=_make_docs(3, source="a")) - conn_a.connector_id = "a" - conn_b = StubConnector(docs=_make_docs(2, source="b")) - conn_b.connector_id = "b" - engine.sync(conn_a) - engine.sync(conn_b) - a_results = store.retrieve("Message", top_k=10, source="a") - b_results = store.retrieve("Message", top_k=10, source="b") - assert len(a_results) == 3 - assert len(b_results) == 2 -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_sync_engine.py -v` - -Expected: FAIL — `ModuleNotFoundError` - -- [ ] **Step 3: Implement SyncEngine** - -Create `src/openjarvis/connectors/sync_engine.py`: - -```python -"""SyncEngine — orchestrates connector sync with checkpoint/resume.""" - -from __future__ import annotations - -import json -import logging -import sqlite3 -from datetime import datetime -from pathlib import Path -from typing import Any, Dict, Optional - -from openjarvis.connectors._stubs import BaseConnector -from openjarvis.connectors.pipeline import IngestionPipeline -from openjarvis.core.config import DEFAULT_CONFIG_DIR - -logger = logging.getLogger(__name__) - -_STATE_SCHEMA = """\ -CREATE TABLE IF NOT EXISTS sync_state ( - connector_id TEXT PRIMARY KEY, - items_synced INTEGER NOT NULL DEFAULT 0, - cursor TEXT, - last_sync TEXT, - error TEXT -); -""" - - -class SyncEngine: - """Orchestrates data source sync with checkpointing. - - Tracks per-connector progress in a small SQLite state database, - separate from the knowledge store. Supports resume after interruption. - """ - - def __init__( - self, - pipeline: IngestionPipeline, - *, - state_db: str = "", - ) -> None: - self._pipeline = pipeline - if not state_db: - state_db = str(DEFAULT_CONFIG_DIR / "sync_state.db") - Path(state_db).parent.mkdir(parents=True, exist_ok=True) - self._state_conn = sqlite3.connect(state_db) - self._state_conn.executescript(_STATE_SCHEMA) - - def sync(self, connector: BaseConnector) -> int: - """Run sync for a single connector. Returns items ingested.""" - cid = connector.connector_id - checkpoint = self.get_checkpoint(cid) - cursor = checkpoint["cursor"] if checkpoint else None - - logger.info("Starting sync for %s (cursor=%s)", cid, cursor) - items = 0 - try: - docs = connector.sync(cursor=cursor) - batch: list = [] - for doc in docs: - batch.append(doc) - items += 1 - # Flush in batches of 100 - if len(batch) >= 100: - self._pipeline.ingest(batch) - self._save_checkpoint(cid, items, cursor=str(items)) - batch = [] - # Flush remaining - if batch: - self._pipeline.ingest(batch) - - self._save_checkpoint(cid, items, cursor=str(items)) - logger.info("Sync complete for %s: %d items", cid, items) - except Exception as exc: - logger.error("Sync error for %s: %s", cid, exc) - self._save_checkpoint(cid, items, cursor=str(items), error=str(exc)) - raise - - return items - - def get_checkpoint(self, connector_id: str) -> Optional[Dict[str, Any]]: - """Get the last checkpoint for a connector, or None if never synced.""" - row = self._state_conn.execute( - "SELECT items_synced, cursor, last_sync, error FROM sync_state WHERE connector_id = ?", - (connector_id,), - ).fetchone() - if row is None: - return None - return { - "items_synced": row[0], - "cursor": row[1], - "last_sync": row[2], - "error": row[3], - } - - def _save_checkpoint( - self, - connector_id: str, - items_synced: int, - *, - cursor: Optional[str] = None, - error: Optional[str] = None, - ) -> None: - now = datetime.now().isoformat() - self._state_conn.execute( - """INSERT INTO sync_state (connector_id, items_synced, cursor, last_sync, error) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(connector_id) - DO UPDATE SET items_synced=?, cursor=?, last_sync=?, error=?""", - ( - connector_id, - items_synced, - cursor, - now, - error, - items_synced, - cursor, - now, - error, - ), - ) - self._state_conn.commit() - - def close(self) -> None: - self._state_conn.close() -``` - -- [ ] **Step 4: Run tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_sync_engine.py -v` - -Expected: All 4 tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/openjarvis/connectors/sync_engine.py tests/connectors/test_sync_engine.py -git commit -m "feat: add SyncEngine with checkpoint/resume for connector orchestration" -``` - ---- - -### Task 6: Obsidian/Markdown Connector - -**Files:** -- Create: `src/openjarvis/connectors/obsidian.py` -- Create: `tests/connectors/test_obsidian.py` - -- [ ] **Step 1: Write failing tests** - -Create `tests/connectors/test_obsidian.py`: - -```python -"""Tests for the Obsidian/Markdown connector.""" - -from __future__ import annotations - -from datetime import datetime -from pathlib import Path -from typing import Optional - -import pytest - -from openjarvis.connectors.obsidian import ObsidianConnector -from openjarvis.core.registry import ConnectorRegistry - - -@pytest.fixture -def vault(tmp_path: Path) -> Path: - vault_dir = tmp_path / "my-vault" - vault_dir.mkdir() - (vault_dir / "note1.md").write_text( - "---\ntitle: Meeting Notes\ntags: [work, meetings]\n---\n\n# Meeting Notes\n\nDiscussed Q3 roadmap." - ) - (vault_dir / "note2.md").write_text("# Ideas\n\nBrainstorm for new feature.") - (vault_dir / "subdir").mkdir() - (vault_dir / "subdir" / "deep.md").write_text("# Deep Note\n\nNested vault content.") - (vault_dir / ".obsidian").mkdir() # Should be skipped - (vault_dir / ".obsidian" / "config.json").write_text("{}") - (vault_dir / "image.png").write_bytes(b"\x89PNG") # Should be skipped - return vault_dir - - -@pytest.fixture -def connector(vault: Path) -> ObsidianConnector: - return ObsidianConnector(vault_path=str(vault)) - - -def test_is_connected(connector: ObsidianConnector) -> None: - assert connector.is_connected() - - -def test_not_connected_bad_path() -> None: - conn = ObsidianConnector(vault_path="/nonexistent/path") - assert not conn.is_connected() - - -def test_sync_yields_markdown_files(connector: ObsidianConnector) -> None: - docs = list(connector.sync()) - assert len(docs) == 3 # note1, note2, deep - sources = {d.doc_id for d in docs} - assert any("note1" in s for s in sources) - assert any("deep" in s for s in sources) - - -def test_sync_skips_hidden_dirs(connector: ObsidianConnector) -> None: - docs = list(connector.sync()) - paths = [d.doc_id for d in docs] - assert not any(".obsidian" in p for p in paths) - - -def test_sync_skips_binary_files(connector: ObsidianConnector) -> None: - docs = list(connector.sync()) - types = [d.title for d in docs] - assert not any("image" in t for t in types) - - -def test_sync_parses_frontmatter(connector: ObsidianConnector) -> None: - docs = list(connector.sync()) - note1 = next(d for d in docs if "note1" in d.doc_id) - assert note1.title == "Meeting Notes" - assert note1.metadata.get("tags") == ["work", "meetings"] - - -def test_sync_sets_doc_type_note(connector: ObsidianConnector) -> None: - docs = list(connector.sync()) - assert all(d.doc_type == "note" for d in docs) - assert all(d.source == "obsidian" for d in docs) - - -def test_disconnect(connector: ObsidianConnector) -> None: - connector.disconnect() - assert not connector.is_connected() - - -def test_registry() -> None: - import openjarvis.connectors.obsidian # noqa: F401 - assert ConnectorRegistry.contains("obsidian") -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_obsidian.py -v` - -Expected: FAIL — `ModuleNotFoundError` - -- [ ] **Step 3: Implement ObsidianConnector** - -Create `src/openjarvis/connectors/obsidian.py`: - -```python -"""Obsidian/Markdown vault connector — reads .md files from a local directory.""" - -from __future__ import annotations - -import os -import re -from datetime import datetime -from pathlib import Path -from typing import Any, Dict, Iterator, List, Optional - -from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus -from openjarvis.core.registry import ConnectorRegistry -from openjarvis.tools._stubs import ToolSpec - -_FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n", re.DOTALL) -_SKIP_DIRS = {".obsidian", ".trash", ".git", "__pycache__", "node_modules", ".venv"} -_MD_EXTENSIONS = {".md", ".markdown", ".txt"} - - -def _parse_frontmatter(text: str) -> tuple[Dict[str, Any], str]: - """Extract YAML frontmatter and return (metadata, body).""" - match = _FRONTMATTER_RE.match(text) - if not match: - return {}, text - raw = match.group(1) - body = text[match.end() :] - meta: Dict[str, Any] = {} - for line in raw.strip().split("\n"): - if ":" not in line: - continue - key, _, val = line.partition(":") - key = key.strip() - val = val.strip() - # Simple YAML list: [a, b, c] - if val.startswith("[") and val.endswith("]"): - meta[key] = [v.strip() for v in val[1:-1].split(",")] - else: - meta[key] = val - return meta, body - - -@ConnectorRegistry.register("obsidian") -class ObsidianConnector(BaseConnector): - """Connector for Obsidian vaults and plain markdown directories.""" - - connector_id = "obsidian" - display_name = "Obsidian / Markdown" - auth_type = "filesystem" - - def __init__(self, vault_path: str = "") -> None: - self._vault_path = vault_path - self._connected = bool(vault_path) and Path(vault_path).is_dir() - self._synced = 0 - self._total = 0 - - def is_connected(self) -> bool: - return self._connected and Path(self._vault_path).is_dir() - - def disconnect(self) -> None: - self._connected = False - self._vault_path = "" - - def sync( - self, *, since: Optional[datetime] = None, cursor: Optional[str] = None - ) -> Iterator[Document]: - if not self.is_connected(): - return - vault = Path(self._vault_path) - md_files = sorted(self._walk_md_files(vault)) - self._total = len(md_files) - self._synced = 0 - - for filepath in md_files: - try: - text = filepath.read_text(encoding="utf-8") - except (UnicodeDecodeError, OSError): - continue - - meta, body = _parse_frontmatter(text) - title = meta.pop("title", filepath.stem) - rel_path = str(filepath.relative_to(vault)) - mtime = datetime.fromtimestamp(filepath.stat().st_mtime) - - if since and mtime < since: - self._synced += 1 - continue - - self._synced += 1 - yield Document( - doc_id=f"obsidian:{rel_path}", - source="obsidian", - doc_type="note", - content=body.strip(), - title=title, - author="", - timestamp=mtime, - url=f"obsidian://open?vault={vault.name}&file={rel_path}", - metadata=meta, - ) - - def sync_status(self) -> SyncStatus: - return SyncStatus( - state="idle", - items_synced=self._synced, - items_total=self._total, - ) - - def mcp_tools(self) -> List[ToolSpec]: - return [ - ToolSpec( - name="obsidian_search_notes", - description="Search Obsidian vault notes by keyword.", - parameters={ - "type": "object", - "properties": { - "query": {"type": "string", "description": "Search query"}, - }, - "required": ["query"], - }, - category="connector", - ), - ] - - def _walk_md_files(self, root: Path) -> List[Path]: - """Walk the vault, skipping hidden/config dirs and non-markdown files.""" - results: List[Path] = [] - for dirpath, dirnames, filenames in os.walk(root): - # Prune hidden/config directories - dirnames[:] = [d for d in dirnames if d not in _SKIP_DIRS and not d.startswith(".")] - for fname in filenames: - if Path(fname).suffix.lower() in _MD_EXTENSIONS: - results.append(Path(dirpath) / fname) - return results -``` - -- [ ] **Step 4: Run tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_obsidian.py -v` - -Expected: All 9 tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/openjarvis/connectors/obsidian.py tests/connectors/test_obsidian.py -git commit -m "feat: add Obsidian/Markdown vault connector with frontmatter parsing" -``` - ---- - -### Task 7: Gmail Connector (Mocked OAuth + API) - -**Files:** -- Create: `src/openjarvis/connectors/oauth.py` -- Create: `src/openjarvis/connectors/gmail.py` -- Create: `tests/connectors/test_gmail.py` - -- [ ] **Step 1: Write failing tests** - -Create `tests/connectors/test_gmail.py`: - -```python -"""Tests for the Gmail connector (mocked API calls).""" - -from __future__ import annotations - -from datetime import datetime -from typing import Optional -from unittest.mock import MagicMock, patch - -import pytest - -from openjarvis.connectors.gmail import GmailConnector -from openjarvis.core.registry import ConnectorRegistry - - -@pytest.fixture -def connector(tmp_path) -> GmailConnector: - return GmailConnector( - credentials_path=str(tmp_path / "gmail_creds.json"), - ) - - -def test_not_connected_without_credentials(connector: GmailConnector) -> None: - assert not connector.is_connected() - - -def test_auth_type_is_oauth(connector: GmailConnector) -> None: - assert connector.auth_type == "oauth" - - -def test_auth_url_returns_string(connector: GmailConnector) -> None: - url = connector.auth_url() - assert url.startswith("https://accounts.google.com/o/oauth2/v2/auth") - assert "gmail.readonly" in url - - -@patch("openjarvis.connectors.gmail._gmail_api_list_messages") -@patch("openjarvis.connectors.gmail._gmail_api_get_message") -def test_sync_yields_documents(mock_get, mock_list, connector, tmp_path) -> None: - # Simulate having valid credentials - creds_path = tmp_path / "gmail_creds.json" - creds_path.write_text('{"token": "fake", "refresh_token": "fake"}') - connector._credentials_path = str(creds_path) - - mock_list.return_value = { - "messages": [{"id": "msg1"}, {"id": "msg2"}], - "nextPageToken": None, - } - mock_get.side_effect = [ - { - "id": "msg1", - "threadId": "t1", - "labelIds": ["INBOX"], - "payload": { - "headers": [ - {"name": "From", "value": "alice@example.com"}, - {"name": "Subject", "value": "Q3 Planning"}, - {"name": "Date", "value": "Mon, 15 Mar 2024 10:00:00 +0000"}, - {"name": "To", "value": "bob@example.com"}, - ], - "body": {"data": "SGVsbG8gd29ybGQ="}, # "Hello world" base64 - }, - }, - { - "id": "msg2", - "threadId": "t2", - "labelIds": ["SENT"], - "payload": { - "headers": [ - {"name": "From", "value": "bob@example.com"}, - {"name": "Subject", "value": "Re: Budget"}, - {"name": "Date", "value": "Tue, 16 Mar 2024 11:00:00 +0000"}, - {"name": "To", "value": "alice@example.com"}, - ], - "body": {"data": "QnVkZ2V0IHJlcGx5"}, # "Budget reply" base64 - }, - }, - ] - - docs = list(connector.sync()) - assert len(docs) == 2 - assert docs[0].doc_id == "gmail:msg1" - assert docs[0].source == "gmail" - assert docs[0].doc_type == "email" - assert docs[0].author == "alice@example.com" - assert docs[0].title == "Q3 Planning" - assert docs[0].thread_id == "t1" - assert "Hello world" in docs[0].content - - -def test_disconnect(connector: GmailConnector, tmp_path) -> None: - creds_path = tmp_path / "gmail_creds.json" - creds_path.write_text('{"token": "fake"}') - connector._credentials_path = str(creds_path) - connector.disconnect() - assert not connector.is_connected() - - -def test_mcp_tools(connector: GmailConnector) -> None: - tools = connector.mcp_tools() - names = [t.name for t in tools] - assert "gmail_search_emails" in names - assert "gmail_get_thread" in names - - -def test_registry() -> None: - import openjarvis.connectors.gmail # noqa: F401 - assert ConnectorRegistry.contains("gmail") -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_gmail.py -v` - -Expected: FAIL — `ModuleNotFoundError` - -- [ ] **Step 3: Create OAuth helper** - -Create `src/openjarvis/connectors/oauth.py`: - -```python -"""Shared OAuth helper for connector authentication. - -Handles the OAuth2 authorization code flow: -1. Generate consent URL with scopes -2. Start a temporary localhost server to receive the callback -3. Exchange the authorization code for tokens -4. Save tokens to credentials file -""" - -from __future__ import annotations - -import json -import logging -from pathlib import Path -from typing import Any, Dict, Optional -from urllib.parse import urlencode - -logger = logging.getLogger(__name__) - - -def build_google_auth_url( - client_id: str, - redirect_uri: str = "http://localhost:8789/callback", - scopes: Optional[list[str]] = None, -) -> str: - """Build a Google OAuth2 consent URL.""" - params = { - "client_id": client_id, - "redirect_uri": redirect_uri, - "response_type": "code", - "scope": " ".join(scopes or []), - "access_type": "offline", - "prompt": "consent", - } - return f"https://accounts.google.com/o/oauth2/v2/auth?{urlencode(params)}" - - -def load_tokens(path: str) -> Optional[Dict[str, Any]]: - """Load OAuth tokens from a JSON file.""" - p = Path(path) - if not p.exists(): - return None - try: - data = json.loads(p.read_text()) - if data.get("token") or data.get("access_token"): - return data - except (json.JSONDecodeError, OSError): - pass - return None - - -def save_tokens(path: str, tokens: Dict[str, Any]) -> None: - """Save OAuth tokens to a JSON file with restrictive permissions.""" - p = Path(path) - p.parent.mkdir(parents=True, exist_ok=True) - p.write_text(json.dumps(tokens, indent=2)) - try: - p.chmod(0o600) - except OSError: - pass - - -def delete_tokens(path: str) -> None: - """Delete a credentials file.""" - p = Path(path) - if p.exists(): - p.unlink() -``` - -- [ ] **Step 4: Implement GmailConnector** - -Create `src/openjarvis/connectors/gmail.py`: - -```python -"""Gmail connector — OAuth + Gmail API for bulk email sync.""" - -from __future__ import annotations - -import base64 -import logging -from datetime import datetime -from email.utils import parsedate_to_datetime -from pathlib import Path -from typing import Any, Dict, Iterator, List, Optional - -import httpx - -from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus -from openjarvis.connectors.oauth import ( - build_google_auth_url, - delete_tokens, - load_tokens, - save_tokens, -) -from openjarvis.core.config import DEFAULT_CONFIG_DIR -from openjarvis.core.registry import ConnectorRegistry -from openjarvis.tools._stubs import ToolSpec - -logger = logging.getLogger(__name__) - -_GMAIL_API = "https://gmail.googleapis.com/gmail/v1/users/me" -# Placeholder — in production, use OpenJarvis's registered OAuth app -_CLIENT_ID = "openjarvis-gmail.apps.googleusercontent.com" -_SCOPES = ["https://www.googleapis.com/auth/gmail.readonly"] - - -def _gmail_api_list_messages( - token: str, *, page_token: Optional[str] = None, query: str = "" -) -> Dict[str, Any]: - """Call Gmail messages.list API.""" - params: dict[str, Any] = {"maxResults": 100} - if page_token: - params["pageToken"] = page_token - if query: - params["q"] = query - resp = httpx.get( - f"{_GMAIL_API}/messages", - headers={"Authorization": f"Bearer {token}"}, - params=params, - timeout=30.0, - ) - resp.raise_for_status() - return resp.json() - - -def _gmail_api_get_message(token: str, msg_id: str) -> Dict[str, Any]: - """Call Gmail messages.get API.""" - resp = httpx.get( - f"{_GMAIL_API}/messages/{msg_id}", - headers={"Authorization": f"Bearer {token}"}, - params={"format": "full"}, - timeout=30.0, - ) - resp.raise_for_status() - return resp.json() - - -def _extract_header(headers: List[Dict[str, str]], name: str) -> str: - """Extract a header value from Gmail API payload headers.""" - for h in headers: - if h.get("name", "").lower() == name.lower(): - return h.get("value", "") - return "" - - -def _decode_body(payload: Dict[str, Any]) -> str: - """Decode the email body from Gmail API payload.""" - # Try top-level body - body_data = payload.get("body", {}).get("data", "") - if body_data: - return base64.urlsafe_b64decode(body_data + "==").decode("utf-8", errors="replace") - # Try multipart parts - for part in payload.get("parts", []): - mime = part.get("mimeType", "") - if mime == "text/plain": - data = part.get("body", {}).get("data", "") - if data: - return base64.urlsafe_b64decode(data + "==").decode("utf-8", errors="replace") - # Fallback: try first part with data - for part in payload.get("parts", []): - data = part.get("body", {}).get("data", "") - if data: - return base64.urlsafe_b64decode(data + "==").decode("utf-8", errors="replace") - return "" - - -def _parse_date(date_str: str) -> datetime: - """Parse an email Date header.""" - try: - return parsedate_to_datetime(date_str) - except Exception: - return datetime.now() - - -@ConnectorRegistry.register("gmail") -class GmailConnector(BaseConnector): - """Gmail connector — syncs email via the Gmail API.""" - - connector_id = "gmail" - display_name = "Gmail" - auth_type = "oauth" - - def __init__(self, credentials_path: str = "") -> None: - if not credentials_path: - credentials_path = str(DEFAULT_CONFIG_DIR / "connectors" / "gmail.json") - self._credentials_path = credentials_path - self._synced = 0 - self._total = 0 - - def is_connected(self) -> bool: - tokens = load_tokens(self._credentials_path) - return tokens is not None - - def disconnect(self) -> None: - delete_tokens(self._credentials_path) - - def auth_url(self) -> str: - return build_google_auth_url(client_id=_CLIENT_ID, scopes=_SCOPES) - - def handle_callback(self, code: str) -> None: - # In production: exchange code for tokens via Google's token endpoint - save_tokens(self._credentials_path, {"token": code, "refresh_token": code}) - - def sync( - self, *, since: Optional[datetime] = None, cursor: Optional[str] = None - ) -> Iterator[Document]: - tokens = load_tokens(self._credentials_path) - if not tokens: - return - token = tokens.get("token", "") - - page_token = cursor - while True: - data = _gmail_api_list_messages(token, page_token=page_token) - messages = data.get("messages", []) - self._total += len(messages) - - for msg_ref in messages: - msg = _gmail_api_get_message(token, msg_ref["id"]) - payload = msg.get("payload", {}) - headers = payload.get("headers", []) - - from_addr = _extract_header(headers, "From") - subject = _extract_header(headers, "Subject") - date_str = _extract_header(headers, "Date") - to_addr = _extract_header(headers, "To") - body = _decode_body(payload) - timestamp = _parse_date(date_str) - - if since and timestamp < since: - continue - - self._synced += 1 - yield Document( - doc_id=f"gmail:{msg['id']}", - source="gmail", - doc_type="email", - content=body, - title=subject, - author=from_addr, - participants=[a.strip() for a in to_addr.split(",") if a.strip()], - timestamp=timestamp, - thread_id=msg.get("threadId", ""), - url=f"https://mail.google.com/mail/u/0/#inbox/{msg['id']}", - metadata={ - "labels": msg.get("labelIds", []), - }, - ) - - page_token = data.get("nextPageToken") - if not page_token: - break - - def sync_status(self) -> SyncStatus: - return SyncStatus( - state="idle", - items_synced=self._synced, - items_total=self._total, - ) - - def mcp_tools(self) -> List[ToolSpec]: - return [ - ToolSpec( - name="gmail_search_emails", - description="Search Gmail messages by query string.", - parameters={ - "type": "object", - "properties": { - "query": {"type": "string", "description": "Gmail search query (e.g., 'from:alice subject:budget')"}, - "max_results": {"type": "integer", "description": "Max results to return", "default": 10}, - }, - "required": ["query"], - }, - category="connector", - ), - ToolSpec( - name="gmail_get_thread", - description="Get all messages in a Gmail thread by thread ID.", - parameters={ - "type": "object", - "properties": { - "thread_id": {"type": "string", "description": "Gmail thread ID"}, - }, - "required": ["thread_id"], - }, - category="connector", - ), - ToolSpec( - name="gmail_list_unread", - description="List recent unread emails.", - parameters={ - "type": "object", - "properties": { - "max_results": {"type": "integer", "description": "Max results", "default": 10}, - }, - }, - category="connector", - ), - ] -``` - -- [ ] **Step 5: Run tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_gmail.py -v` - -Expected: All 7 tests PASS. - -- [ ] **Step 6: Commit** - -```bash -git add src/openjarvis/connectors/oauth.py src/openjarvis/connectors/gmail.py tests/connectors/test_gmail.py -git commit -m "feat: add Gmail connector with OAuth and mocked API sync" -``` - ---- - -### Task 8: knowledge_search Tool - -**Files:** -- Create: `src/openjarvis/tools/knowledge_search.py` -- Create: `tests/tools/test_knowledge_search.py` - -- [ ] **Step 1: Write failing tests** - -Create `tests/tools/test_knowledge_search.py`: - -```python -"""Tests for the knowledge_search tool.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from openjarvis.connectors.store import KnowledgeStore -from openjarvis.core.registry import ToolRegistry -from openjarvis.tools.knowledge_search import KnowledgeSearchTool - - -@pytest.fixture -def store(tmp_path: Path) -> KnowledgeStore: - s = KnowledgeStore(db_path=str(tmp_path / "ks_test.db")) - s.store(content="Kubernetes migration proposal", source="gdrive", doc_type="document", author="sarah") - s.store(content="Budget discussion for Q3", source="gmail", doc_type="email", author="mike") - s.store(content="Sprint standup notes", source="slack", doc_type="message", author="alice") - return s - - -@pytest.fixture -def tool(store: KnowledgeStore) -> KnowledgeSearchTool: - return KnowledgeSearchTool(store=store) - - -def test_basic_search(tool: KnowledgeSearchTool) -> None: - result = tool.execute(query="Kubernetes migration") - assert result.success - assert "Kubernetes" in result.content - - -def test_filter_by_source(tool: KnowledgeSearchTool) -> None: - result = tool.execute(query="discussion", source="gmail") - assert result.success - assert "Budget" in result.content - - -def test_filter_by_author(tool: KnowledgeSearchTool) -> None: - result = tool.execute(query="proposal", author="sarah") - assert result.success - assert "Kubernetes" in result.content - - -def test_no_results(tool: KnowledgeSearchTool) -> None: - result = tool.execute(query="xyznonexistent") - assert result.success - assert "No relevant results" in result.content - - -def test_empty_query(tool: KnowledgeSearchTool) -> None: - result = tool.execute(query="") - assert not result.success - - -def test_no_store() -> None: - tool = KnowledgeSearchTool(store=None) - result = tool.execute(query="anything") - assert not result.success - - -def test_spec_has_filter_params(tool: KnowledgeSearchTool) -> None: - spec = tool.spec - props = spec.parameters.get("properties", {}) - assert "query" in props - assert "source" in props - assert "doc_type" in props - assert "author" in props - assert "since" in props - assert "top_k" in props - - -def test_registry() -> None: - import openjarvis.tools.knowledge_search # noqa: F401 - assert ToolRegistry.contains("knowledge_search") -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/tools/test_knowledge_search.py -v` - -Expected: FAIL — `ModuleNotFoundError` - -- [ ] **Step 3: Implement KnowledgeSearchTool** - -Create `src/openjarvis/tools/knowledge_search.py`: - -```python -"""knowledge_search — filtered BM25 retrieval over the personal knowledge base.""" - -from __future__ import annotations - -from typing import Any, Optional - -from openjarvis.connectors.store import KnowledgeStore -from openjarvis.core.registry import ToolRegistry -from openjarvis.core.types import ToolResult -from openjarvis.tools._stubs import BaseTool, ToolSpec - - -@ToolRegistry.register("knowledge_search") -class KnowledgeSearchTool(BaseTool): - """Search the personal knowledge base with optional filters. - - Supports filtering by source (gmail, slack, gdrive, etc.), - doc_type (email, message, document, etc.), author, and time range. - """ - - tool_id = "knowledge_search" - - def __init__(self, store: Optional[KnowledgeStore] = None) -> None: - self._store = store - - @property - def spec(self) -> ToolSpec: - return ToolSpec( - name="knowledge_search", - description=( - "Search your personal knowledge base (emails, messages, documents, " - "calendar events, contacts, notes) with optional filters. Returns " - "relevant results with source attribution and deep links." - ), - parameters={ - "type": "object", - "properties": { - "query": { - "type": "string", - "description": "Search query to find relevant information.", - }, - "source": { - "type": "string", - "description": 'Filter by source: "gmail", "slack", "gdrive", "obsidian", etc.', - }, - "doc_type": { - "type": "string", - "description": 'Filter by type: "email", "message", "document", "event", "contact", "note".', - }, - "author": { - "type": "string", - "description": "Filter by author/sender name or email.", - }, - "since": { - "type": "string", - "description": "Only return results after this ISO 8601 timestamp.", - }, - "until": { - "type": "string", - "description": "Only return results before this ISO 8601 timestamp.", - }, - "top_k": { - "type": "integer", - "description": "Number of results to return (default: 10).", - }, - }, - "required": ["query"], - }, - category="knowledge", - ) - - def execute(self, **params: Any) -> ToolResult: - if self._store is None: - return ToolResult( - tool_name="knowledge_search", - content="No knowledge store configured. Run 'jarvis connect' to set up data sources.", - success=False, - ) - - query = params.get("query", "") - if not query: - return ToolResult( - tool_name="knowledge_search", - content="No query provided.", - success=False, - ) - - top_k = params.get("top_k", 10) - try: - results = self._store.retrieve( - query, - top_k=top_k, - source=params.get("source", ""), - doc_type=params.get("doc_type", ""), - author=params.get("author", ""), - since=params.get("since", ""), - until=params.get("until", ""), - ) - except Exception as exc: - return ToolResult( - tool_name="knowledge_search", - content=f"Search error: {exc}", - success=False, - ) - - if not results: - return ToolResult( - tool_name="knowledge_search", - content="No relevant results found.", - success=True, - metadata={"num_results": 0}, - ) - - # Format results with source attribution - lines = [] - for i, r in enumerate(results, 1): - meta = r.metadata - source_tag = meta.get("source", "unknown") - author = meta.get("author", "") - title = meta.get("title", "") - url = meta.get("url", "") - header_parts = [f"[{source_tag}]"] - if title: - header_parts.append(title) - if author: - header_parts.append(f"by {author}") - if url: - header_parts.append(f"({url})") - header = " ".join(header_parts) - lines.append(f"**Result {i}:** {header}\n{r.content}\n") - - return ToolResult( - tool_name="knowledge_search", - content="\n".join(lines), - success=True, - metadata={"num_results": len(results)}, - ) -``` - -- [ ] **Step 4: Run tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/tools/test_knowledge_search.py -v` - -Expected: All 8 tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/openjarvis/tools/knowledge_search.py tests/tools/test_knowledge_search.py -git commit -m "feat: add knowledge_search tool with filtered BM25 retrieval and source attribution" -``` - ---- - -### Task 9: CLI `jarvis connect` Command - -**Files:** -- Create: `src/openjarvis/cli/connect_cmd.py` -- Modify: `src/openjarvis/cli/__init__.py` -- Create: `tests/cli/test_connect.py` - -- [ ] **Step 1: Write failing tests** - -Create `tests/cli/test_connect.py`: - -```python -"""Tests for the jarvis connect CLI command.""" - -from __future__ import annotations - -from pathlib import Path -from unittest.mock import patch - -import pytest -from click.testing import CliRunner - -from openjarvis.cli.connect_cmd import connect - - -@pytest.fixture -def runner() -> CliRunner: - return CliRunner() - - -def test_connect_list_no_connectors(runner: CliRunner) -> None: - result = runner.invoke(connect, ["--list"]) - assert result.exit_code == 0 - assert "No connectors" in result.output or "connected" in result.output.lower() - - -def test_connect_list_with_connector(runner: CliRunner, tmp_path: Path) -> None: - # Create a fake credential file so Gmail appears connected - creds = tmp_path / "gmail.json" - creds.write_text('{"token": "fake"}') - with patch("openjarvis.connectors.gmail.DEFAULT_CONFIG_DIR", tmp_path / "config"): - result = runner.invoke(connect, ["--list"]) - assert result.exit_code == 0 - - -def test_connect_help(runner: CliRunner) -> None: - result = runner.invoke(connect, ["--help"]) - assert result.exit_code == 0 - assert "connect" in result.output.lower() - - -def test_connect_specific_source(runner: CliRunner) -> None: - result = runner.invoke(connect, ["obsidian", "--path", "/nonexistent"]) - # Should fail gracefully — path doesn't exist - assert result.exit_code == 0 or "not found" in result.output.lower() or "does not exist" in result.output.lower() - - -def test_connect_disconnect(runner: CliRunner) -> None: - result = runner.invoke(connect, ["--disconnect", "gmail"]) - assert result.exit_code == 0 -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/cli/test_connect.py -v` - -Expected: FAIL — `ModuleNotFoundError` - -- [ ] **Step 3: Implement connect command** - -Create `src/openjarvis/cli/connect_cmd.py`: - -```python -"""CLI command: jarvis connect — manage data source connections.""" - -from __future__ import annotations - -import click -from rich.console import Console -from rich.table import Table - -console = Console() - - -@click.group(invoke_without_command=True) -@click.argument("source", required=False) -@click.option("--list", "list_sources", is_flag=True, help="List connected sources and sync status.") -@click.option("--sync", "trigger_sync", is_flag=True, help="Trigger incremental sync for all sources.") -@click.option("--disconnect", "disconnect_source", default="", help="Disconnect a source.") -@click.option("--path", default="", help="Path for filesystem connectors (e.g., Obsidian vault).") -@click.pass_context -def connect( - ctx: click.Context, - source: str | None, - list_sources: bool, - trigger_sync: bool, - disconnect_source: str, - path: str, -) -> None: - """Connect data sources for Deep Research. - - \b - Examples: - jarvis connect # Interactive source picker - jarvis connect gmail # Connect Gmail - jarvis connect obsidian --path ~/vault - jarvis connect --list # Show connected sources - jarvis connect --disconnect gmail # Disconnect Gmail - """ - # Import connectors to trigger registration - import openjarvis.connectors.gmail # noqa: F401 - import openjarvis.connectors.obsidian # noqa: F401 - from openjarvis.core.registry import ConnectorRegistry - - if list_sources: - _list_sources(ConnectorRegistry) - return - - if disconnect_source: - _disconnect_source(ConnectorRegistry, disconnect_source) - return - - if trigger_sync: - console.print("[yellow]Sync not yet implemented in CLI — use the desktop app.[/yellow]") - return - - if source: - _connect_source(ConnectorRegistry, source, path=path) - return - - # No args — show help - if not ctx.invoked_subcommand: - console.print(ctx.get_help()) - - -def _list_sources(registry: type) -> None: - """List all available connectors and their status.""" - table = Table(title="Data Sources") - table.add_column("Source", style="cyan") - table.add_column("Type", style="dim") - table.add_column("Status", style="green") - - for key in sorted(registry.keys()): - cls = registry.get(key) - try: - instance = cls() - connected = instance.is_connected() - status = "[green]Connected[/green]" if connected else "[dim]Not connected[/dim]" - except Exception: - status = "[dim]Not configured[/dim]" - auth = getattr(cls, "auth_type", "unknown") - table.add_row(key, auth, status) - - if not registry.keys(): - console.print("[dim]No connectors registered.[/dim]") - return - - console.print(table) - - -def _disconnect_source(registry: type, source: str) -> None: - """Disconnect a specific source.""" - if not registry.contains(source): - console.print(f"[red]Unknown source: {source}[/red]") - return - cls = registry.get(source) - try: - instance = cls() - instance.disconnect() - console.print(f"[green]Disconnected {source}.[/green]") - except Exception as exc: - console.print(f"[red]Error disconnecting {source}: {exc}[/red]") - - -def _connect_source(registry: type, source: str, *, path: str = "") -> None: - """Connect a specific source.""" - if not registry.contains(source): - console.print(f"[red]Unknown source: {source}[/red]") - console.print(f"Available: {', '.join(sorted(registry.keys()))}") - return - - cls = registry.get(source) - auth_type = getattr(cls, "auth_type", "unknown") - - if auth_type == "filesystem": - if not path: - console.print(f"[red]{source} requires --path argument.[/red]") - return - from pathlib import Path as P - - if not P(path).is_dir(): - console.print(f"[red]Path does not exist: {path}[/red]") - return - instance = cls(vault_path=path) - if instance.is_connected(): - console.print(f"[green]Connected to {source} at {path}[/green]") - else: - console.print(f"[red]Could not connect to {source} at {path}[/red]") - - elif auth_type == "oauth": - instance = cls() - if instance.is_connected(): - console.print(f"[green]{source} is already connected.[/green]") - return - url = instance.auth_url() - console.print(f"Open this URL in your browser to authorize {source}:") - console.print(f"[link={url}]{url}[/link]") - console.print("[dim]After authorizing, paste the code here.[/dim]") - - else: - console.print(f"[yellow]Auth type '{auth_type}' not yet supported in CLI.[/yellow]") -``` - -- [ ] **Step 4: Register connect command in CLI __init__.py** - -Open `src/openjarvis/cli/__init__.py`. Find where other commands are added (look for `cli.add_command` calls). Add: - -```python -from openjarvis.cli.connect_cmd import connect - -cli.add_command(connect, "connect") -``` - -Place this alongside the existing `cli.add_command` calls. - -- [ ] **Step 5: Run tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/cli/test_connect.py -v` - -Expected: All 5 tests PASS. - -- [ ] **Step 6: Commit** - -```bash -git add src/openjarvis/cli/connect_cmd.py src/openjarvis/cli/__init__.py tests/cli/test_connect.py -git commit -m "feat: add 'jarvis connect' CLI command for managing data source connections" -``` - ---- - -### Task 10: Integration Test + Wire Up Connectors __init__.py - -**Files:** -- Modify: `src/openjarvis/connectors/__init__.py` -- Create: `tests/connectors/test_integration.py` - -- [ ] **Step 1: Update connectors __init__.py to auto-register** - -Open `src/openjarvis/connectors/__init__.py` and replace with: - -```python -"""Data source connectors for Deep Research.""" - -from openjarvis.connectors._stubs import ( - Attachment, - BaseConnector, - Document, - SyncStatus, -) - -__all__ = ["Attachment", "BaseConnector", "Document", "SyncStatus"] - -# Auto-register built-in connectors -import openjarvis.connectors.obsidian # noqa: F401 - -try: - import openjarvis.connectors.gmail # noqa: F401 -except ImportError: - pass # httpx may not be installed -``` - -- [ ] **Step 2: Write integration test** - -Create `tests/connectors/test_integration.py`: - -```python -"""Integration test — full pipeline from connector to retrieval.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from openjarvis.connectors._stubs import Document -from openjarvis.connectors.obsidian import ObsidianConnector -from openjarvis.connectors.pipeline import IngestionPipeline -from openjarvis.connectors.store import KnowledgeStore -from openjarvis.connectors.sync_engine import SyncEngine -from openjarvis.tools.knowledge_search import KnowledgeSearchTool - - -@pytest.fixture -def vault(tmp_path: Path) -> Path: - vault_dir = tmp_path / "test-vault" - vault_dir.mkdir() - (vault_dir / "project-notes.md").write_text( - "# Project Alpha\n\nWe decided to migrate to Kubernetes in March.\n\n" - "## Cost Analysis\n\nEstimated 40% increase in cloud spend during transition.\n\n" - "## Timeline\n\nSix-week migration window starting April 1st." - ) - (vault_dir / "meeting.md").write_text( - "# Sprint Review\n\nDiscussed budget concerns with Mike and Sarah.\n" - "Action item: Sarah to prepare cost comparison document." - ) - return vault_dir - - -def test_full_pipeline_obsidian_to_search(vault: Path, tmp_path: Path) -> None: - """End-to-end: Obsidian vault → SyncEngine → KnowledgeStore → knowledge_search.""" - # 1. Setup - store = KnowledgeStore(db_path=str(tmp_path / "integration.db")) - pipeline = IngestionPipeline(store=store) - engine = SyncEngine(pipeline=pipeline, state_db=str(tmp_path / "state.db")) - connector = ObsidianConnector(vault_path=str(vault)) - - # 2. Sync - engine.sync(connector) - - # 3. Verify checkpoint - cp = engine.get_checkpoint("obsidian") - assert cp is not None - assert cp["items_synced"] == 2 - - # 4. Search via knowledge_search tool - tool = KnowledgeSearchTool(store=store) - - # Search for Kubernetes content - result = tool.execute(query="Kubernetes migration") - assert result.success - assert "Kubernetes" in result.content - - # Search for budget discussion - result = tool.execute(query="budget concerns") - assert result.success - assert "budget" in result.content.lower() - - # Search with source filter - result = tool.execute(query="migration", source="obsidian") - assert result.success - - # No results for nonexistent source - result = tool.execute(query="migration", source="gmail") - assert "No relevant results" in result.content -``` - -- [ ] **Step 3: Run the full integration test** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_integration.py -v` - -Expected: PASS — the full pipeline works end-to-end. - -- [ ] **Step 4: Run ALL connector and tool tests together** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/ tests/tools/test_knowledge_search.py tests/cli/test_connect.py -v` - -Expected: All tests PASS. - -- [ ] **Step 5: Run linter** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run ruff check src/openjarvis/connectors/ src/openjarvis/tools/knowledge_search.py src/openjarvis/cli/connect_cmd.py tests/connectors/ tests/tools/test_knowledge_search.py tests/cli/test_connect.py` - -Expected: No errors. If there are import ordering or unused import warnings, fix them. - -- [ ] **Step 6: Commit** - -```bash -git add src/openjarvis/connectors/__init__.py tests/connectors/test_integration.py -git commit -m "feat: add integration test and auto-registration for built-in connectors" -``` - ---- - -## Post-Plan Notes - -**What this plan produces:** A working connector framework with two connectors (Gmail + Obsidian), a knowledge store with filtered BM25 retrieval, a `knowledge_search` tool for agents, a SyncEngine with checkpointing, and a `jarvis connect` CLI. This is the foundation that Phases 2-5 build on. - -**What comes next (separate plans):** -- **Phase 2:** Desktop setup wizard UI + Slack, Google Drive, Calendar, Contacts, iMessage connectors -- **Phase 3:** ColBERTv2 disk persistence + DeepResearchAgent + research report UI -- **Phase 4:** ChannelAgent + iMessage/WhatsApp/Slack plugins -- **Phase 5:** Incremental sync, attachment store, settings page polish diff --git a/docs/superpowers/plans/2026-03-25-deep-research-phase2a-connectors.md b/docs/superpowers/plans/2026-03-25-deep-research-phase2a-connectors.md deleted file mode 100644 index 455ac835..00000000 --- a/docs/superpowers/plans/2026-03-25-deep-research-phase2a-connectors.md +++ /dev/null @@ -1,1896 +0,0 @@ -# Deep Research Phase 2A: Remaining Connectors Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add the 5 remaining data source connectors — Slack, Google Drive, Google Calendar, Google Contacts, and iMessage — to the connector framework built in Phase 1. - -**Architecture:** Each connector follows the established `BaseConnector` pattern: registered via `@ConnectorRegistry.register()`, yields normalized `Document` objects, stores credentials via `oauth.py` helpers, exposes MCP tools for real-time agent queries. OAuth connectors (Slack, Drive, Calendar, Contacts) reuse `build_google_auth_url()` or Slack's own OAuth. iMessage reads directly from the local macOS SQLite database. - -**Tech Stack:** Python 3.10+, httpx (API calls), sqlite3 (iMessage local DB), pytest + unittest.mock (mocked tests) - -**Spec:** `docs/superpowers/specs/2026-03-25-deep-research-setup-design.md` — Section 5 (Connector Layer), Phase 2 - -**Depends on:** Phase 1 complete (ConnectorRegistry, BaseConnector, Document, oauth.py, KnowledgeStore, SyncEngine all exist) - ---- - -## File Structure - -``` -src/openjarvis/connectors/ -├── slack_connector.py # Slack connector (OAuth + Web API) -├── gdrive.py # Google Drive connector (OAuth + Drive API v3) -├── gcalendar.py # Google Calendar connector (OAuth + Calendar API v3) -├── gcontacts.py # Google Contacts connector (OAuth + People API) -├── imessage.py # iMessage connector (local macOS SQLite) -├── __init__.py # (modify) Add auto-imports for new connectors - -tests/connectors/ -├── test_slack_connector.py -├── test_gdrive.py -├── test_gcalendar.py -├── test_gcontacts.py -├── test_imessage.py -``` - ---- - -### Task 1: Slack Connector - -**Files:** -- Create: `src/openjarvis/connectors/slack_connector.py` -- Create: `tests/connectors/test_slack_connector.py` - -Named `slack_connector.py` to avoid collision with the existing `channels/slack.py`. - -- [ ] **Step 1: Write failing tests** - -Create `tests/connectors/test_slack_connector.py`: - -```python -"""Tests for the Slack data source connector (mocked API).""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import List -from unittest.mock import patch - -import pytest - -from openjarvis.connectors._stubs import Document -from openjarvis.core.registry import ConnectorRegistry - - -@pytest.fixture() -def connector(tmp_path: Path): - from openjarvis.connectors.slack_connector import SlackConnector - - return SlackConnector(credentials_path=str(tmp_path / "slack.json")) - - -_CHANNELS_RESPONSE = { - "channels": [ - {"id": "C001", "name": "general", "is_member": True}, - {"id": "C002", "name": "engineering", "is_member": True}, - ], - "response_metadata": {"next_cursor": ""}, -} - -_HISTORY_RESPONSE = { - "messages": [ - { - "ts": "1710500000.000100", - "user": "U001", - "text": "Let's discuss the API redesign.", - "thread_ts": "1710500000.000100", - }, - { - "ts": "1710500060.000200", - "user": "U002", - "text": "Sounds good, I'll prepare a doc.", - }, - ], - "has_more": False, -} - -_USERS_RESPONSE = { - "members": [ - {"id": "U001", "real_name": "Alice", "profile": {"email": "alice@co.com"}}, - {"id": "U002", "real_name": "Bob", "profile": {"email": "bob@co.com"}}, - ], -} - - -def test_not_connected_without_credentials(connector) -> None: - assert connector.is_connected() is False - - -def test_auth_type_is_oauth(connector) -> None: - assert connector.auth_type == "oauth" - - -def test_auth_url(connector) -> None: - url = connector.auth_url() - assert "slack.com" in url - - -@patch("openjarvis.connectors.slack_connector._slack_api_users_list") -@patch("openjarvis.connectors.slack_connector._slack_api_conversations_history") -@patch("openjarvis.connectors.slack_connector._slack_api_conversations_list") -def test_sync_yields_documents( - mock_channels, mock_history, mock_users, connector, tmp_path: Path -) -> None: - creds = Path(connector._credentials_path) - creds.write_text(json.dumps({"token": "xoxb-fake"}), encoding="utf-8") - - mock_channels.return_value = _CHANNELS_RESPONSE - mock_history.return_value = _HISTORY_RESPONSE - mock_users.return_value = _USERS_RESPONSE - - docs: List[Document] = list(connector.sync()) - - # 2 channels x 2 messages = 4 messages, but grouped by channel - assert len(docs) == 4 - assert all(d.source == "slack" for d in docs) - assert all(d.doc_type == "message" for d in docs) - - msg1 = next(d for d in docs if "API redesign" in d.content) - assert msg1.author == "Alice" - assert msg1.metadata.get("channel") == "general" - - -def test_disconnect(connector, tmp_path: Path) -> None: - creds = Path(connector._credentials_path) - creds.write_text(json.dumps({"token": "xoxb-fake"}), encoding="utf-8") - assert connector.is_connected() - connector.disconnect() - assert not connector.is_connected() - - -def test_mcp_tools(connector) -> None: - tools = connector.mcp_tools() - names = {t.name for t in tools} - assert "slack_search_messages" in names - assert "slack_get_thread" in names - assert "slack_list_channels" in names - - -def test_registry() -> None: - from openjarvis.connectors.slack_connector import SlackConnector - - ConnectorRegistry.register_value("slack", SlackConnector) - assert ConnectorRegistry.contains("slack") -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_slack_connector.py -v` - -Expected: FAIL — `ModuleNotFoundError` - -- [ ] **Step 3: Implement SlackConnector** - -Create `src/openjarvis/connectors/slack_connector.py`: - -```python -"""Slack data source connector — syncs message history via Slack Web API.""" - -from __future__ import annotations - -import logging -from datetime import datetime, timezone -from typing import Any, Dict, Iterator, List, Optional -from urllib.parse import urlencode - -import httpx - -from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus -from openjarvis.connectors.oauth import delete_tokens, load_tokens, save_tokens -from openjarvis.core.config import DEFAULT_CONFIG_DIR -from openjarvis.core.registry import ConnectorRegistry -from openjarvis.tools._stubs import ToolSpec - -logger = logging.getLogger(__name__) - -_SLACK_API = "https://slack.com/api" -_DEFAULT_CREDENTIALS_PATH = str( - DEFAULT_CONFIG_DIR / "connectors" / "slack.json" -) - - -def _slack_api_conversations_list( - token: str, *, cursor: str = "" -) -> Dict[str, Any]: - """Call conversations.list to get channels the bot is a member of.""" - params: Dict[str, str] = { - "types": "public_channel,private_channel", - "exclude_archived": "true", - "limit": "200", - } - if cursor: - params["cursor"] = cursor - resp = httpx.get( - f"{_SLACK_API}/conversations.list", - headers={"Authorization": f"Bearer {token}"}, - params=params, - timeout=30.0, - ) - resp.raise_for_status() - return resp.json() - - -def _slack_api_conversations_history( - token: str, channel_id: str, *, cursor: str = "" -) -> Dict[str, Any]: - """Call conversations.history to get messages in a channel.""" - params: Dict[str, str] = {"channel": channel_id, "limit": "200"} - if cursor: - params["cursor"] = cursor - resp = httpx.get( - f"{_SLACK_API}/conversations.history", - headers={"Authorization": f"Bearer {token}"}, - params=params, - timeout=30.0, - ) - resp.raise_for_status() - return resp.json() - - -def _slack_api_users_list(token: str) -> Dict[str, Any]: - """Call users.list to build a user ID → name/email map.""" - resp = httpx.get( - f"{_SLACK_API}/users.list", - headers={"Authorization": f"Bearer {token}"}, - timeout=30.0, - ) - resp.raise_for_status() - return resp.json() - - -def _ts_to_datetime(ts: str) -> datetime: - """Convert Slack message timestamp to datetime.""" - try: - return datetime.fromtimestamp(float(ts), tz=timezone.utc) - except (ValueError, TypeError): - return datetime.now(tz=timezone.utc) - - -@ConnectorRegistry.register("slack") -class SlackConnector(BaseConnector): - """Slack connector — syncs channel message history via Web API.""" - - connector_id = "slack" - display_name = "Slack" - auth_type = "oauth" - - def __init__(self, credentials_path: str = "") -> None: - self._credentials_path = credentials_path or _DEFAULT_CREDENTIALS_PATH - self._items_synced = 0 - self._items_total = 0 - - def _get_token(self) -> str: - tokens = load_tokens(self._credentials_path) - if tokens: - return tokens.get("token", "") - return "" - - def is_connected(self) -> bool: - return bool(self._get_token()) - - def disconnect(self) -> None: - delete_tokens(self._credentials_path) - - def auth_url(self) -> str: - params = { - "client_id": "openjarvis-slack", - "scope": "channels:history,channels:read,users:read", - "redirect_uri": "http://localhost:8789/callback", - } - return f"https://slack.com/oauth/v2/authorize?{urlencode(params)}" - - def handle_callback(self, code: str) -> None: - save_tokens(self._credentials_path, {"token": code}) - - def sync( - self, - *, - since: Optional[datetime] = None, - cursor: Optional[str] = None, - ) -> Iterator[Document]: - token = self._get_token() - if not token: - return - - # Build user map - users_resp = _slack_api_users_list(token) - user_map: Dict[str, Dict[str, str]] = {} - for member in users_resp.get("members", []): - uid = member.get("id", "") - user_map[uid] = { - "name": member.get("real_name", uid), - "email": member.get("profile", {}).get("email", ""), - } - - # Get channels - channels_resp = _slack_api_conversations_list(token) - channels = channels_resp.get("channels", []) - - synced = 0 - for chan in channels: - chan_id = chan.get("id", "") - chan_name = chan.get("name", chan_id) - - history = _slack_api_conversations_history(token, chan_id) - messages = history.get("messages", []) - - for msg in messages: - ts = msg.get("ts", "") - timestamp = _ts_to_datetime(ts) - - if since and timestamp < since: - continue - - user_id = msg.get("user", "") - user_info = user_map.get(user_id, {"name": user_id, "email": ""}) - text = msg.get("text", "") - - synced += 1 - yield Document( - doc_id=f"slack:{chan_id}:{ts}", - source="slack", - doc_type="message", - content=text, - title=f"#{chan_name}", - author=user_info["name"], - participants=[user_info["name"]], - timestamp=timestamp, - thread_id=msg.get("thread_ts"), - url=f"https://slack.com/archives/{chan_id}/p{ts.replace('.', '')}", - metadata={ - "channel": chan_name, - "channel_id": chan_id, - "user_id": user_id, - }, - ) - - self._items_synced = synced - - def sync_status(self) -> SyncStatus: - return SyncStatus(state="idle", items_synced=self._items_synced) - - def mcp_tools(self) -> List[ToolSpec]: - return [ - ToolSpec( - name="slack_search_messages", - description="Search Slack messages by keyword.", - parameters={ - "type": "object", - "properties": { - "query": { - "type": "string", - "description": "Search query", - }, - }, - "required": ["query"], - }, - category="communication", - ), - ToolSpec( - name="slack_get_thread", - description="Get all replies in a Slack thread.", - parameters={ - "type": "object", - "properties": { - "channel_id": { - "type": "string", - "description": "Slack channel ID", - }, - "thread_ts": { - "type": "string", - "description": "Thread timestamp", - }, - }, - "required": ["channel_id", "thread_ts"], - }, - category="communication", - ), - ToolSpec( - name="slack_list_channels", - description="List Slack channels the bot has access to.", - parameters={ - "type": "object", - "properties": {}, - }, - category="communication", - ), - ] -``` - -- [ ] **Step 4: Run tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_slack_connector.py -v` - -Expected: All 7 tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/openjarvis/connectors/slack_connector.py tests/connectors/test_slack_connector.py -git commit -m "feat: add Slack data source connector with channel history sync" -``` - ---- - -### Task 2: Google Drive Connector - -**Files:** -- Create: `src/openjarvis/connectors/gdrive.py` -- Create: `tests/connectors/test_gdrive.py` - -- [ ] **Step 1: Write failing tests** - -Create `tests/connectors/test_gdrive.py`: - -```python -"""Tests for the Google Drive connector (mocked API).""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import List -from unittest.mock import patch - -import pytest - -from openjarvis.connectors._stubs import Document -from openjarvis.core.registry import ConnectorRegistry - - -@pytest.fixture() -def connector(tmp_path: Path): - from openjarvis.connectors.gdrive import GDriveConnector - - return GDriveConnector(credentials_path=str(tmp_path / "gdrive.json")) - - -_FILES_LIST_RESPONSE = { - "files": [ - { - "id": "doc1", - "name": "Q3 Roadmap", - "mimeType": "application/vnd.google-apps.document", - "modifiedTime": "2024-03-15T10:00:00.000Z", - "owners": [{"emailAddress": "alice@co.com", "displayName": "Alice"}], - "webViewLink": "https://docs.google.com/document/d/doc1/edit", - }, - { - "id": "sheet1", - "name": "Budget 2024", - "mimeType": "application/vnd.google-apps.spreadsheet", - "modifiedTime": "2024-03-16T11:00:00.000Z", - "owners": [{"emailAddress": "bob@co.com", "displayName": "Bob"}], - "webViewLink": "https://docs.google.com/spreadsheets/d/sheet1/edit", - }, - ], - "nextPageToken": None, -} - -_EXPORT_RESPONSE = "# Q3 Roadmap\n\nThis is the roadmap content." - - -def test_not_connected_without_credentials(connector) -> None: - assert connector.is_connected() is False - - -def test_auth_url(connector) -> None: - url = connector.auth_url() - assert "accounts.google.com" in url - assert "drive.readonly" in url - - -@patch("openjarvis.connectors.gdrive._gdrive_api_export") -@patch("openjarvis.connectors.gdrive._gdrive_api_list_files") -def test_sync_yields_documents( - mock_list, mock_export, connector, tmp_path: Path -) -> None: - creds = Path(connector._credentials_path) - creds.write_text(json.dumps({"token": "fake"}), encoding="utf-8") - - mock_list.return_value = _FILES_LIST_RESPONSE - mock_export.return_value = _EXPORT_RESPONSE - - docs: List[Document] = list(connector.sync()) - - assert len(docs) == 2 - assert all(d.source == "gdrive" for d in docs) - assert all(d.doc_type == "document" for d in docs) - - doc1 = next(d for d in docs if d.doc_id == "gdrive:doc1") - assert doc1.title == "Q3 Roadmap" - assert doc1.author == "Alice" - assert "roadmap" in doc1.content.lower() - - -def test_disconnect(connector, tmp_path: Path) -> None: - creds = Path(connector._credentials_path) - creds.write_text(json.dumps({"token": "fake"}), encoding="utf-8") - connector.disconnect() - assert not connector.is_connected() - - -def test_mcp_tools(connector) -> None: - tools = connector.mcp_tools() - names = {t.name for t in tools} - assert "gdrive_search_files" in names - assert "gdrive_get_document" in names - assert "gdrive_list_recent" in names - - -def test_registry() -> None: - from openjarvis.connectors.gdrive import GDriveConnector - - ConnectorRegistry.register_value("gdrive", GDriveConnector) - assert ConnectorRegistry.contains("gdrive") -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_gdrive.py -v` - -Expected: FAIL — `ModuleNotFoundError` - -- [ ] **Step 3: Implement GDriveConnector** - -Create `src/openjarvis/connectors/gdrive.py`: - -```python -"""Google Drive connector — syncs files via Drive API v3.""" - -from __future__ import annotations - -import logging -from datetime import datetime, timezone -from typing import Any, Dict, Iterator, List, Optional - -import httpx - -from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus -from openjarvis.connectors.oauth import ( - build_google_auth_url, - delete_tokens, - load_tokens, - save_tokens, -) -from openjarvis.core.config import DEFAULT_CONFIG_DIR -from openjarvis.core.registry import ConnectorRegistry -from openjarvis.tools._stubs import ToolSpec - -logger = logging.getLogger(__name__) - -_DRIVE_API = "https://www.googleapis.com/drive/v3" -_DRIVE_SCOPE = "https://www.googleapis.com/auth/drive.readonly" -_DEFAULT_CREDENTIALS_PATH = str( - DEFAULT_CONFIG_DIR / "connectors" / "gdrive.json" -) - -# Google Workspace MIME types that can be exported as text -_EXPORT_MIMES: Dict[str, str] = { - "application/vnd.google-apps.document": "text/plain", - "application/vnd.google-apps.spreadsheet": "text/csv", - "application/vnd.google-apps.presentation": "text/plain", -} - - -def _gdrive_api_list_files( - token: str, *, page_token: Optional[str] = None -) -> Dict[str, Any]: - """Call files.list to enumerate accessible files.""" - params: Dict[str, str] = { - "pageSize": "100", - "fields": ( - "files(id,name,mimeType,modifiedTime,owners,webViewLink)," - "nextPageToken" - ), - "orderBy": "modifiedTime desc", - } - if page_token: - params["pageToken"] = page_token - resp = httpx.get( - f"{_DRIVE_API}/files", - headers={"Authorization": f"Bearer {token}"}, - params=params, - timeout=30.0, - ) - resp.raise_for_status() - return resp.json() - - -def _gdrive_api_export( - token: str, file_id: str, mime_type: str -) -> str: - """Export a Google Workspace file as text.""" - resp = httpx.get( - f"{_DRIVE_API}/files/{file_id}/export", - headers={"Authorization": f"Bearer {token}"}, - params={"mimeType": mime_type}, - timeout=60.0, - ) - resp.raise_for_status() - return resp.text - - -def _parse_iso(dt_str: str) -> datetime: - if not dt_str: - return datetime.now(tz=timezone.utc) - try: - return datetime.fromisoformat(dt_str.replace("Z", "+00:00")) - except (ValueError, AttributeError): - return datetime.now(tz=timezone.utc) - - -@ConnectorRegistry.register("gdrive") -class GDriveConnector(BaseConnector): - """Google Drive connector — syncs documents via Drive API v3.""" - - connector_id = "gdrive" - display_name = "Google Drive" - auth_type = "oauth" - - def __init__(self, credentials_path: str = "") -> None: - self._credentials_path = credentials_path or _DEFAULT_CREDENTIALS_PATH - self._items_synced = 0 - - def _get_token(self) -> str: - tokens = load_tokens(self._credentials_path) - return tokens.get("token", "") if tokens else "" - - def is_connected(self) -> bool: - return bool(self._get_token()) - - def disconnect(self) -> None: - delete_tokens(self._credentials_path) - - def auth_url(self) -> str: - return build_google_auth_url( - client_id="openjarvis-drive", scopes=[_DRIVE_SCOPE] - ) - - def handle_callback(self, code: str) -> None: - save_tokens(self._credentials_path, {"token": code}) - - def sync( - self, - *, - since: Optional[datetime] = None, - cursor: Optional[str] = None, - ) -> Iterator[Document]: - token = self._get_token() - if not token: - return - - page_token = cursor - synced = 0 - - while True: - resp = _gdrive_api_list_files(token, page_token=page_token) - files = resp.get("files", []) - - for f in files: - modified = _parse_iso(f.get("modifiedTime", "")) - if since and modified < since: - continue - - file_id = f["id"] - mime = f.get("mimeType", "") - owners = f.get("owners", [{}]) - owner = owners[0] if owners else {} - - # Export Google Workspace files; skip binary files - export_mime = _EXPORT_MIMES.get(mime) - if export_mime: - content = _gdrive_api_export(token, file_id, export_mime) - else: - # Non-exportable file — store metadata only - content = f"[File: {f.get('name', '')}] ({mime})" - - synced += 1 - yield Document( - doc_id=f"gdrive:{file_id}", - source="gdrive", - doc_type="document", - content=content, - title=f.get("name", ""), - author=owner.get("displayName", owner.get("emailAddress", "")), - timestamp=modified, - url=f.get("webViewLink", ""), - metadata={"mime_type": mime}, - ) - - page_token = resp.get("nextPageToken") - if not page_token: - break - - self._items_synced = synced - - def sync_status(self) -> SyncStatus: - return SyncStatus(state="idle", items_synced=self._items_synced) - - def mcp_tools(self) -> List[ToolSpec]: - return [ - ToolSpec( - name="gdrive_search_files", - description="Search Google Drive files by name or content.", - parameters={ - "type": "object", - "properties": { - "query": {"type": "string", "description": "Search query"}, - }, - "required": ["query"], - }, - category="knowledge", - ), - ToolSpec( - name="gdrive_get_document", - description="Get the text content of a Google Drive document.", - parameters={ - "type": "object", - "properties": { - "file_id": {"type": "string", "description": "Drive file ID"}, - }, - "required": ["file_id"], - }, - category="knowledge", - ), - ToolSpec( - name="gdrive_list_recent", - description="List recently modified files in Google Drive.", - parameters={ - "type": "object", - "properties": { - "max_results": { - "type": "integer", - "description": "Max files to return", - "default": 20, - }, - }, - }, - category="knowledge", - ), - ] -``` - -- [ ] **Step 4: Run tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_gdrive.py -v` - -Expected: All 6 tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/openjarvis/connectors/gdrive.py tests/connectors/test_gdrive.py -git commit -m "feat: add Google Drive connector with file export and sync" -``` - ---- - -### Task 3: Google Calendar Connector - -**Files:** -- Create: `src/openjarvis/connectors/gcalendar.py` -- Create: `tests/connectors/test_gcalendar.py` - -- [ ] **Step 1: Write failing tests** - -Create `tests/connectors/test_gcalendar.py`: - -```python -"""Tests for Google Calendar connector (mocked API).""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import List -from unittest.mock import patch - -import pytest - -from openjarvis.connectors._stubs import Document -from openjarvis.core.registry import ConnectorRegistry - - -@pytest.fixture() -def connector(tmp_path: Path): - from openjarvis.connectors.gcalendar import GCalendarConnector - - return GCalendarConnector(credentials_path=str(tmp_path / "gcal.json")) - - -_CALENDARS_RESPONSE = { - "items": [ - {"id": "primary", "summary": "My Calendar"}, - ], -} - -_EVENTS_RESPONSE = { - "items": [ - { - "id": "evt1", - "summary": "Sprint Planning", - "description": "Review sprint goals and capacity.", - "start": {"dateTime": "2024-03-15T10:00:00Z"}, - "end": {"dateTime": "2024-03-15T11:00:00Z"}, - "attendees": [ - {"email": "alice@co.com", "displayName": "Alice"}, - {"email": "bob@co.com", "displayName": "Bob"}, - ], - "location": "Room 3", - "organizer": {"email": "alice@co.com", "displayName": "Alice"}, - "htmlLink": "https://calendar.google.com/event?eid=evt1", - }, - ], - "nextPageToken": None, -} - - -def test_not_connected(connector) -> None: - assert not connector.is_connected() - - -def test_auth_url(connector) -> None: - url = connector.auth_url() - assert "accounts.google.com" in url - assert "calendar.readonly" in url - - -@patch("openjarvis.connectors.gcalendar._gcal_api_events_list") -@patch("openjarvis.connectors.gcalendar._gcal_api_calendars_list") -def test_sync_yields_events( - mock_calendars, mock_events, connector, tmp_path: Path -) -> None: - creds = Path(connector._credentials_path) - creds.write_text(json.dumps({"token": "fake"}), encoding="utf-8") - - mock_calendars.return_value = _CALENDARS_RESPONSE - mock_events.return_value = _EVENTS_RESPONSE - - docs: List[Document] = list(connector.sync()) - - assert len(docs) == 1 - evt = docs[0] - assert evt.doc_id == "gcalendar:evt1" - assert evt.source == "gcalendar" - assert evt.doc_type == "event" - assert evt.title == "Sprint Planning" - assert "Sprint Planning" in evt.content - assert "alice@co.com" in [p for p in evt.participants] - assert "Room 3" in evt.content - - -def test_disconnect(connector, tmp_path: Path) -> None: - creds = Path(connector._credentials_path) - creds.write_text(json.dumps({"token": "fake"}), encoding="utf-8") - connector.disconnect() - assert not connector.is_connected() - - -def test_mcp_tools(connector) -> None: - tools = connector.mcp_tools() - names = {t.name for t in tools} - assert "calendar_get_events_today" in names - assert "calendar_search_events" in names - assert "calendar_next_meeting" in names - - -def test_registry() -> None: - from openjarvis.connectors.gcalendar import GCalendarConnector - - ConnectorRegistry.register_value("gcalendar", GCalendarConnector) - assert ConnectorRegistry.contains("gcalendar") -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_gcalendar.py -v` - -Expected: FAIL — `ModuleNotFoundError` - -- [ ] **Step 3: Implement GCalendarConnector** - -Create `src/openjarvis/connectors/gcalendar.py`: - -```python -"""Google Calendar connector — syncs events via Calendar API v3.""" - -from __future__ import annotations - -import logging -from datetime import datetime, timezone -from typing import Any, Dict, Iterator, List, Optional - -import httpx - -from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus -from openjarvis.connectors.oauth import ( - build_google_auth_url, - delete_tokens, - load_tokens, - save_tokens, -) -from openjarvis.core.config import DEFAULT_CONFIG_DIR -from openjarvis.core.registry import ConnectorRegistry -from openjarvis.tools._stubs import ToolSpec - -logger = logging.getLogger(__name__) - -_CAL_API = "https://www.googleapis.com/calendar/v3" -_CAL_SCOPE = "https://www.googleapis.com/auth/calendar.readonly" -_DEFAULT_CREDENTIALS_PATH = str( - DEFAULT_CONFIG_DIR / "connectors" / "gcalendar.json" -) - - -def _gcal_api_calendars_list(token: str) -> Dict[str, Any]: - resp = httpx.get( - f"{_CAL_API}/users/me/calendarList", - headers={"Authorization": f"Bearer {token}"}, - timeout=30.0, - ) - resp.raise_for_status() - return resp.json() - - -def _gcal_api_events_list( - token: str, - calendar_id: str, - *, - page_token: Optional[str] = None, - time_min: Optional[str] = None, -) -> Dict[str, Any]: - params: Dict[str, str] = { - "maxResults": "250", - "singleEvents": "true", - "orderBy": "startTime", - } - if page_token: - params["pageToken"] = page_token - if time_min: - params["timeMin"] = time_min - resp = httpx.get( - f"{_CAL_API}/calendars/{calendar_id}/events", - headers={"Authorization": f"Bearer {token}"}, - params=params, - timeout=30.0, - ) - resp.raise_for_status() - return resp.json() - - -def _parse_iso(dt_str: str) -> datetime: - if not dt_str: - return datetime.now(tz=timezone.utc) - try: - return datetime.fromisoformat(dt_str.replace("Z", "+00:00")) - except (ValueError, AttributeError): - return datetime.now(tz=timezone.utc) - - -def _format_event(event: Dict[str, Any]) -> str: - """Format a calendar event as human-readable text.""" - lines = [event.get("summary", "(No title)")] - - start = event.get("start", {}) - end = event.get("end", {}) - start_str = start.get("dateTime", start.get("date", "")) - end_str = end.get("dateTime", end.get("date", "")) - if start_str: - lines.append(f"When: {start_str} — {end_str}") - - location = event.get("location", "") - if location: - lines.append(f"Location: {location}") - - attendees = event.get("attendees", []) - if attendees: - names = [ - a.get("displayName", a.get("email", "")) - for a in attendees - ] - lines.append(f"Attendees: {', '.join(names)}") - - organizer = event.get("organizer", {}) - org_name = organizer.get("displayName", organizer.get("email", "")) - if org_name: - lines.append(f"Organizer: {org_name}") - - desc = event.get("description", "") - if desc: - lines.append(f"\n{desc}") - - return "\n".join(lines) - - -@ConnectorRegistry.register("gcalendar") -class GCalendarConnector(BaseConnector): - """Google Calendar connector — syncs events.""" - - connector_id = "gcalendar" - display_name = "Google Calendar" - auth_type = "oauth" - - def __init__(self, credentials_path: str = "") -> None: - self._credentials_path = ( - credentials_path or _DEFAULT_CREDENTIALS_PATH - ) - self._items_synced = 0 - - def _get_token(self) -> str: - tokens = load_tokens(self._credentials_path) - return tokens.get("token", "") if tokens else "" - - def is_connected(self) -> bool: - return bool(self._get_token()) - - def disconnect(self) -> None: - delete_tokens(self._credentials_path) - - def auth_url(self) -> str: - return build_google_auth_url( - client_id="openjarvis-calendar", scopes=[_CAL_SCOPE] - ) - - def handle_callback(self, code: str) -> None: - save_tokens(self._credentials_path, {"token": code}) - - def sync( - self, - *, - since: Optional[datetime] = None, - cursor: Optional[str] = None, - ) -> Iterator[Document]: - token = self._get_token() - if not token: - return - - time_min = since.isoformat() if since else None - cals = _gcal_api_calendars_list(token) - synced = 0 - - for cal in cals.get("items", []): - cal_id = cal.get("id", "primary") - page_token = cursor - - while True: - resp = _gcal_api_events_list( - token, cal_id, - page_token=page_token, - time_min=time_min, - ) - events = resp.get("items", []) - - for evt in events: - evt_id = evt.get("id", "") - start = evt.get("start", {}) - start_str = start.get( - "dateTime", start.get("date", "") - ) - timestamp = _parse_iso(start_str) - - attendees = evt.get("attendees", []) - participant_emails = [ - a.get("email", "") for a in attendees - ] - - synced += 1 - yield Document( - doc_id=f"gcalendar:{evt_id}", - source="gcalendar", - doc_type="event", - content=_format_event(evt), - title=evt.get("summary", ""), - author=evt.get("organizer", {}).get("email", ""), - participants=participant_emails, - timestamp=timestamp, - url=evt.get("htmlLink", ""), - metadata={ - "calendar_id": cal_id, - "location": evt.get("location", ""), - }, - ) - - page_token = resp.get("nextPageToken") - if not page_token: - break - - self._items_synced = synced - - def sync_status(self) -> SyncStatus: - return SyncStatus(state="idle", items_synced=self._items_synced) - - def mcp_tools(self) -> List[ToolSpec]: - return [ - ToolSpec( - name="calendar_get_events_today", - description="Get today's calendar events.", - parameters={"type": "object", "properties": {}}, - category="productivity", - ), - ToolSpec( - name="calendar_search_events", - description="Search calendar events by keyword.", - parameters={ - "type": "object", - "properties": { - "query": { - "type": "string", - "description": "Search query", - }, - }, - "required": ["query"], - }, - category="productivity", - ), - ToolSpec( - name="calendar_next_meeting", - description="Get the next upcoming meeting.", - parameters={"type": "object", "properties": {}}, - category="productivity", - ), - ] -``` - -- [ ] **Step 4: Run tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_gcalendar.py -v` - -Expected: All 6 tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/openjarvis/connectors/gcalendar.py tests/connectors/test_gcalendar.py -git commit -m "feat: add Google Calendar connector with event sync" -``` - ---- - -### Task 4: Google Contacts Connector - -**Files:** -- Create: `src/openjarvis/connectors/gcontacts.py` -- Create: `tests/connectors/test_gcontacts.py` - -- [ ] **Step 1: Write failing tests** - -Create `tests/connectors/test_gcontacts.py`: - -```python -"""Tests for Google Contacts connector (mocked API).""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import List -from unittest.mock import patch - -import pytest - -from openjarvis.connectors._stubs import Document -from openjarvis.core.registry import ConnectorRegistry - - -@pytest.fixture() -def connector(tmp_path: Path): - from openjarvis.connectors.gcontacts import GContactsConnector - - return GContactsConnector(credentials_path=str(tmp_path / "gcontacts.json")) - - -_CONNECTIONS_RESPONSE = { - "connections": [ - { - "resourceName": "people/c1", - "names": [{"displayName": "Alice Smith"}], - "emailAddresses": [{"value": "alice@co.com"}], - "phoneNumbers": [{"value": "+1-555-0100"}], - "organizations": [{"name": "Acme Corp", "title": "VP Engineering"}], - }, - { - "resourceName": "people/c2", - "names": [{"displayName": "Bob Jones"}], - "emailAddresses": [{"value": "bob@co.com"}], - "phoneNumbers": [], - "organizations": [{"name": "Acme Corp", "title": "Designer"}], - }, - ], - "nextPageToken": None, - "totalItems": 2, -} - - -def test_not_connected(connector) -> None: - assert not connector.is_connected() - - -def test_auth_url(connector) -> None: - url = connector.auth_url() - assert "accounts.google.com" in url - assert "contacts.readonly" in url - - -@patch("openjarvis.connectors.gcontacts._gcontacts_api_list") -def test_sync_yields_contacts(mock_list, connector, tmp_path: Path) -> None: - creds = Path(connector._credentials_path) - creds.write_text(json.dumps({"token": "fake"}), encoding="utf-8") - - mock_list.return_value = _CONNECTIONS_RESPONSE - - docs: List[Document] = list(connector.sync()) - - assert len(docs) == 2 - assert all(d.source == "gcontacts" for d in docs) - assert all(d.doc_type == "contact" for d in docs) - - alice = next(d for d in docs if "Alice" in d.title) - assert "alice@co.com" in alice.content - assert "VP Engineering" in alice.content - - -def test_disconnect(connector, tmp_path: Path) -> None: - creds = Path(connector._credentials_path) - creds.write_text(json.dumps({"token": "fake"}), encoding="utf-8") - connector.disconnect() - assert not connector.is_connected() - - -def test_mcp_tools(connector) -> None: - tools = connector.mcp_tools() - names = {t.name for t in tools} - assert "contacts_find" in names - assert "contacts_get_info" in names - - -def test_registry() -> None: - from openjarvis.connectors.gcontacts import GContactsConnector - - ConnectorRegistry.register_value("gcontacts", GContactsConnector) - assert ConnectorRegistry.contains("gcontacts") -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_gcontacts.py -v` - -Expected: FAIL — `ModuleNotFoundError` - -- [ ] **Step 3: Implement GContactsConnector** - -Create `src/openjarvis/connectors/gcontacts.py`: - -```python -"""Google Contacts connector — syncs contacts via People API.""" - -from __future__ import annotations - -import logging -from datetime import datetime, timezone -from typing import Any, Dict, Iterator, List, Optional - -import httpx - -from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus -from openjarvis.connectors.oauth import ( - build_google_auth_url, - delete_tokens, - load_tokens, - save_tokens, -) -from openjarvis.core.config import DEFAULT_CONFIG_DIR -from openjarvis.core.registry import ConnectorRegistry -from openjarvis.tools._stubs import ToolSpec - -logger = logging.getLogger(__name__) - -_PEOPLE_API = "https://people.googleapis.com/v1" -_CONTACTS_SCOPE = "https://www.googleapis.com/auth/contacts.readonly" -_DEFAULT_CREDENTIALS_PATH = str( - DEFAULT_CONFIG_DIR / "connectors" / "gcontacts.json" -) - -_PERSON_FIELDS = "names,emailAddresses,phoneNumbers,organizations" - - -def _gcontacts_api_list( - token: str, *, page_token: Optional[str] = None -) -> Dict[str, Any]: - """Call people.connections.list.""" - params: Dict[str, str] = { - "personFields": _PERSON_FIELDS, - "pageSize": "100", - } - if page_token: - params["pageToken"] = page_token - resp = httpx.get( - f"{_PEOPLE_API}/people/me/connections", - headers={"Authorization": f"Bearer {token}"}, - params=params, - timeout=30.0, - ) - resp.raise_for_status() - return resp.json() - - -def _format_contact(person: Dict[str, Any]) -> str: - """Format a contact as searchable text.""" - lines: List[str] = [] - - names = person.get("names", []) - if names: - lines.append(names[0].get("displayName", "")) - - for email in person.get("emailAddresses", []): - lines.append(email.get("value", "")) - - for phone in person.get("phoneNumbers", []): - lines.append(phone.get("value", "")) - - for org in person.get("organizations", []): - parts = [] - if org.get("name"): - parts.append(org["name"]) - if org.get("title"): - parts.append(org["title"]) - if parts: - lines.append(", ".join(parts)) - - return "\n".join(lines) - - -@ConnectorRegistry.register("gcontacts") -class GContactsConnector(BaseConnector): - """Google Contacts connector — syncs contacts via People API.""" - - connector_id = "gcontacts" - display_name = "Google Contacts" - auth_type = "oauth" - - def __init__(self, credentials_path: str = "") -> None: - self._credentials_path = ( - credentials_path or _DEFAULT_CREDENTIALS_PATH - ) - self._items_synced = 0 - - def _get_token(self) -> str: - tokens = load_tokens(self._credentials_path) - return tokens.get("token", "") if tokens else "" - - def is_connected(self) -> bool: - return bool(self._get_token()) - - def disconnect(self) -> None: - delete_tokens(self._credentials_path) - - def auth_url(self) -> str: - return build_google_auth_url( - client_id="openjarvis-contacts", scopes=[_CONTACTS_SCOPE] - ) - - def handle_callback(self, code: str) -> None: - save_tokens(self._credentials_path, {"token": code}) - - def sync( - self, - *, - since: Optional[datetime] = None, - cursor: Optional[str] = None, - ) -> Iterator[Document]: - token = self._get_token() - if not token: - return - - page_token = cursor - synced = 0 - - while True: - resp = _gcontacts_api_list(token, page_token=page_token) - people = resp.get("connections", []) - - for person in people: - resource = person.get("resourceName", "") - names = person.get("names", []) - display_name = names[0].get("displayName", "") if names else "" - emails = person.get("emailAddresses", []) - primary_email = emails[0].get("value", "") if emails else "" - - synced += 1 - yield Document( - doc_id=f"gcontacts:{resource}", - source="gcontacts", - doc_type="contact", - content=_format_contact(person), - title=display_name, - author=primary_email, - participants=[primary_email] if primary_email else [], - timestamp=datetime.now(tz=timezone.utc), - metadata={"resource_name": resource}, - ) - - page_token = resp.get("nextPageToken") - if not page_token: - break - - self._items_synced = synced - - def sync_status(self) -> SyncStatus: - return SyncStatus(state="idle", items_synced=self._items_synced) - - def mcp_tools(self) -> List[ToolSpec]: - return [ - ToolSpec( - name="contacts_find", - description="Find a contact by name or email.", - parameters={ - "type": "object", - "properties": { - "query": { - "type": "string", - "description": "Name or email to search", - }, - }, - "required": ["query"], - }, - category="productivity", - ), - ToolSpec( - name="contacts_get_info", - description="Get full details for a contact.", - parameters={ - "type": "object", - "properties": { - "name": { - "type": "string", - "description": "Contact name", - }, - }, - "required": ["name"], - }, - category="productivity", - ), - ] -``` - -- [ ] **Step 4: Run tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_gcontacts.py -v` - -Expected: All 6 tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/openjarvis/connectors/gcontacts.py tests/connectors/test_gcontacts.py -git commit -m "feat: add Google Contacts connector with People API sync" -``` - ---- - -### Task 5: iMessage Connector (macOS Local SQLite) - -**Files:** -- Create: `src/openjarvis/connectors/imessage.py` -- Create: `tests/connectors/test_imessage.py` - -- [ ] **Step 1: Write failing tests** - -Create `tests/connectors/test_imessage.py`: - -```python -"""Tests for iMessage connector (uses a temp SQLite DB mimicking chat.db).""" - -from __future__ import annotations - -import sqlite3 -from pathlib import Path -from typing import List - -import pytest - -from openjarvis.connectors._stubs import Document -from openjarvis.core.registry import ConnectorRegistry - - -def _create_fake_chat_db(db_path: Path) -> None: - """Create a minimal iMessage chat.db schema with test data.""" - conn = sqlite3.connect(str(db_path)) - conn.executescript(""" - CREATE TABLE handle ( - ROWID INTEGER PRIMARY KEY, - id TEXT NOT NULL - ); - CREATE TABLE chat ( - ROWID INTEGER PRIMARY KEY, - chat_identifier TEXT NOT NULL, - display_name TEXT - ); - CREATE TABLE message ( - ROWID INTEGER PRIMARY KEY, - text TEXT, - handle_id INTEGER, - date INTEGER, - is_from_me INTEGER DEFAULT 0, - cache_roomnames TEXT - ); - CREATE TABLE chat_message_join ( - chat_id INTEGER, - message_id INTEGER - ); - - INSERT INTO handle VALUES (1, '+15550100'); - INSERT INTO handle VALUES (2, 'alice@icloud.com'); - - INSERT INTO chat VALUES (1, '+15550100', 'Alice'); - INSERT INTO chat VALUES (2, 'chat123', 'Team Group'); - - INSERT INTO message VALUES (1, 'Hey, are we meeting tomorrow?', 1, 700000000000000000, 0, NULL); - INSERT INTO message VALUES (2, 'Yes at 3pm!', 1, 700000060000000000, 1, NULL); - INSERT INTO message VALUES (3, 'Group message about project', 2, 700000120000000000, 0, 'chat123'); - - INSERT INTO chat_message_join VALUES (1, 1); - INSERT INTO chat_message_join VALUES (1, 2); - INSERT INTO chat_message_join VALUES (2, 3); - """) - conn.commit() - conn.close() - - -@pytest.fixture() -def chat_db(tmp_path: Path) -> Path: - db_path = tmp_path / "chat.db" - _create_fake_chat_db(db_path) - return db_path - - -@pytest.fixture() -def connector(chat_db: Path): - from openjarvis.connectors.imessage import IMessageConnector - - return IMessageConnector(db_path=str(chat_db)) - - -def test_is_connected(connector) -> None: - assert connector.is_connected() - - -def test_not_connected_missing_db() -> None: - from openjarvis.connectors.imessage import IMessageConnector - - conn = IMessageConnector(db_path="/nonexistent/chat.db") - assert not conn.is_connected() - - -def test_sync_yields_messages(connector) -> None: - docs: List[Document] = list(connector.sync()) - assert len(docs) == 3 - assert all(d.source == "imessage" for d in docs) - assert all(d.doc_type == "message" for d in docs) - - -def test_sync_message_content(connector) -> None: - docs: List[Document] = list(connector.sync()) - texts = {d.content for d in docs} - assert "Hey, are we meeting tomorrow?" in texts - assert "Yes at 3pm!" in texts - - -def test_sync_sets_author(connector) -> None: - docs: List[Document] = list(connector.sync()) - msg1 = next(d for d in docs if "meeting tomorrow" in d.content) - assert msg1.author == "+15550100" - - -def test_disconnect(connector) -> None: - connector.disconnect() - assert not connector.is_connected() - - -def test_mcp_tools(connector) -> None: - tools = connector.mcp_tools() - names = {t.name for t in tools} - assert "imessage_search_messages" in names - assert "imessage_get_conversation" in names - - -def test_registry() -> None: - from openjarvis.connectors.imessage import IMessageConnector - - ConnectorRegistry.register_value("imessage", IMessageConnector) - assert ConnectorRegistry.contains("imessage") -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_imessage.py -v` - -Expected: FAIL — `ModuleNotFoundError` - -- [ ] **Step 3: Implement IMessageConnector** - -Create `src/openjarvis/connectors/imessage.py`: - -```python -"""iMessage connector — reads from macOS ~/Library/Messages/chat.db.""" - -from __future__ import annotations - -import logging -import sqlite3 -from datetime import datetime, timezone -from pathlib import Path -from typing import Any, Dict, Iterator, List, Optional - -from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus -from openjarvis.core.registry import ConnectorRegistry -from openjarvis.tools._stubs import ToolSpec - -logger = logging.getLogger(__name__) - -# macOS iMessage stores timestamps as nanoseconds since 2001-01-01 -_APPLE_EPOCH = datetime(2001, 1, 1, tzinfo=timezone.utc) -_NS_FACTOR = 1_000_000_000 - -_DEFAULT_DB_PATH = str( - Path.home() / "Library" / "Messages" / "chat.db" -) - - -def _apple_ts_to_datetime(apple_ts: int) -> datetime: - """Convert Apple's nanosecond timestamp to datetime.""" - if apple_ts == 0: - return datetime.now(tz=timezone.utc) - try: - seconds = apple_ts / _NS_FACTOR - return datetime( - 2001, 1, 1, tzinfo=timezone.utc - ) + __import__("datetime").timedelta(seconds=seconds) - except (ValueError, OverflowError): - return datetime.now(tz=timezone.utc) - - -@ConnectorRegistry.register("imessage") -class IMessageConnector(BaseConnector): - """iMessage connector — reads from the local macOS Messages database. - - Requires Full Disk Access permission on macOS. - """ - - connector_id = "imessage" - display_name = "iMessage" - auth_type = "local" - - def __init__(self, db_path: str = "") -> None: - self._db_path = db_path or _DEFAULT_DB_PATH - self._connected = Path(self._db_path).exists() - self._items_synced = 0 - - def is_connected(self) -> bool: - return self._connected and Path(self._db_path).exists() - - def disconnect(self) -> None: - self._connected = False - - def sync( - self, - *, - since: Optional[datetime] = None, - cursor: Optional[str] = None, - ) -> Iterator[Document]: - if not self.is_connected(): - return - - conn = sqlite3.connect( - f"file:{self._db_path}?mode=ro", uri=True - ) - conn.row_factory = sqlite3.Row - - # Build handle ID → identifier map - handle_map: Dict[int, str] = {} - for row in conn.execute("SELECT ROWID, id FROM handle"): - handle_map[row["ROWID"]] = row["id"] - - # Build message → chat map - msg_chat: Dict[int, int] = {} - for row in conn.execute( - "SELECT chat_id, message_id FROM chat_message_join" - ): - msg_chat[row["message_id"]] = row["chat_id"] - - # Build chat map - chat_map: Dict[int, Dict[str, str]] = {} - for row in conn.execute( - "SELECT ROWID, chat_identifier, display_name FROM chat" - ): - chat_map[row["ROWID"]] = { - "identifier": row["chat_identifier"], - "display_name": row["display_name"] or "", - } - - # Query messages - sql = ( - "SELECT ROWID, text, handle_id, date, is_from_me," - " cache_roomnames FROM message WHERE text IS NOT NULL" - ) - if since: - # Convert since to Apple timestamp - delta = since - _APPLE_EPOCH - apple_ts = int(delta.total_seconds() * _NS_FACTOR) - sql += f" AND date >= {apple_ts}" - - sql += " ORDER BY date ASC" - synced = 0 - - for row in conn.execute(sql): - msg_id = row["ROWID"] - text = row["text"] or "" - handle_id = row["handle_id"] - is_from_me = row["is_from_me"] - - sender = "me" if is_from_me else handle_map.get(handle_id, "") - timestamp = _apple_ts_to_datetime(row["date"]) - - # Determine chat context - chat_id = msg_chat.get(msg_id, 0) - chat_info = chat_map.get(chat_id, {}) - chat_name = ( - chat_info.get("display_name") - or chat_info.get("identifier", "") - ) - - synced += 1 - yield Document( - doc_id=f"imessage:{msg_id}", - source="imessage", - doc_type="message", - content=text, - title=chat_name, - author=sender, - participants=[sender], - timestamp=timestamp, - thread_id=chat_info.get("identifier"), - metadata={ - "chat_name": chat_name, - "is_from_me": bool(is_from_me), - }, - ) - - conn.close() - self._items_synced = synced - - def sync_status(self) -> SyncStatus: - return SyncStatus(state="idle", items_synced=self._items_synced) - - def mcp_tools(self) -> List[ToolSpec]: - return [ - ToolSpec( - name="imessage_search_messages", - description="Search iMessage history by keyword.", - parameters={ - "type": "object", - "properties": { - "query": { - "type": "string", - "description": "Search query", - }, - }, - "required": ["query"], - }, - category="communication", - ), - ToolSpec( - name="imessage_get_conversation", - description="Get messages from a specific iMessage conversation.", - parameters={ - "type": "object", - "properties": { - "contact": { - "type": "string", - "description": "Phone number or email", - }, - "max_results": { - "type": "integer", - "description": "Max messages", - "default": 50, - }, - }, - "required": ["contact"], - }, - category="communication", - ), - ] -``` - -- [ ] **Step 4: Run tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_imessage.py -v` - -Expected: All 8 tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/openjarvis/connectors/imessage.py tests/connectors/test_imessage.py -git commit -m "feat: add iMessage connector reading from macOS Messages database" -``` - ---- - -### Task 6: Wire Up Auto-Registration + Full Test Suite - -**Files:** -- Modify: `src/openjarvis/connectors/__init__.py` - -- [ ] **Step 1: Update __init__.py with all new connectors** - -Add auto-imports for all 5 new connectors at the bottom of `src/openjarvis/connectors/__init__.py`: - -```python -try: - import openjarvis.connectors.slack_connector # noqa: F401 -except ImportError: - pass - -try: - import openjarvis.connectors.gdrive # noqa: F401 -except ImportError: - pass - -try: - import openjarvis.connectors.gcalendar # noqa: F401 -except ImportError: - pass - -try: - import openjarvis.connectors.gcontacts # noqa: F401 -except ImportError: - pass - -try: - import openjarvis.connectors.imessage # noqa: F401 -except ImportError: - pass -``` - -- [ ] **Step 2: Run ALL connector tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/ tests/tools/test_knowledge_search.py tests/cli/test_connect.py -v` - -Expected: All tests PASS (Phase 1 tests + 5 new connector test files). - -- [ ] **Step 3: Run linter on all new files** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run ruff check src/openjarvis/connectors/ tests/connectors/` - -Expected: No errors. - -- [ ] **Step 4: Commit** - -```bash -git add src/openjarvis/connectors/__init__.py -git commit -m "feat: auto-register all Phase 2A connectors (Slack, Drive, Calendar, Contacts, iMessage)" -``` - ---- - -## Post-Plan Notes - -**What this plan produces:** 5 additional connectors bringing the total to 9 (Gmail, Obsidian, Notion, Granola, Slack, Google Drive, Google Calendar, Google Contacts, iMessage). All follow the established BaseConnector pattern with mocked tests. - -**What comes next:** -- **Phase 2B:** Desktop setup wizard UI (Tauri/TypeScript/React) — the visual onboarding experience -- **Phase 3:** ColBERTv2 persistence + DeepResearchAgent -- **Phase 4:** Channel plugins (iMessage/WhatsApp/Slack for talking to the agent) -- **Phase 5:** Incremental sync, attachment store, settings page diff --git a/docs/superpowers/plans/2026-03-25-deep-research-phase3.md b/docs/superpowers/plans/2026-03-25-deep-research-phase3.md deleted file mode 100644 index f025ee0a..00000000 --- a/docs/superpowers/plans/2026-03-25-deep-research-phase3.md +++ /dev/null @@ -1,1270 +0,0 @@ -# Deep Research Phase 3: Two-Stage Retrieval + Deep Research Agent - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add ColBERTv2 reranking on top of BM25 retrieval and build the DeepResearchAgent — a multi-hop tool-using agent that searches your personal knowledge base across sources, synthesizes findings, and produces cited research reports. - -**Architecture:** A new `TwoStageRetriever` composes BM25 recall (KnowledgeStore FTS5) with ColBERTv2 semantic reranking. The `knowledge_search` tool is upgraded to use it. A `DeepResearchAgent` extends `ToolUsingAgent` with a research-oriented system prompt, multi-hop loop, and structured report output with cross-platform citations. ColBERT persistence to disk is added via memory-mapped tensors for large knowledge bases. - -**Tech Stack:** Python 3.10+, sqlite3 (BM25/FTS5), torch + colbert-ai (reranking, optional), pytest - -**Spec:** `docs/superpowers/specs/2026-03-25-deep-research-setup-design.md` — Sections 7, 8, Phase 3 - -**Depends on:** Phase 1 complete (KnowledgeStore, knowledge_search tool, IngestionPipeline, SemanticChunker) - ---- - -## File Structure - -``` -src/openjarvis/connectors/ -├── retriever.py # TwoStageRetriever: BM25 recall → ColBERT rerank -├── store.py # (modify) Add store_embedding/load_embedding methods -├── pipeline.py # (modify) Dual-write to ColBERT index during ingest - -src/openjarvis/agents/ -├── deep_research.py # DeepResearchAgent with multi-hop loop + citations - -src/openjarvis/tools/ -├── knowledge_search.py # (modify) Upgrade to use TwoStageRetriever - -tests/connectors/ -├── test_retriever.py # TwoStageRetriever tests - -tests/agents/ -├── test_deep_research.py # DeepResearchAgent tests -``` - ---- - -### Task 1: TwoStageRetriever (BM25 Recall → ColBERT Rerank) - -**Files:** -- Create: `src/openjarvis/connectors/retriever.py` -- Create: `tests/connectors/test_retriever.py` - -- [ ] **Step 1: Write failing tests** - -Create `tests/connectors/test_retriever.py`: - -```python -"""Tests for TwoStageRetriever — BM25 recall + ColBERT rerank.""" - -from __future__ import annotations - -from pathlib import Path -from unittest.mock import MagicMock, patch - -import pytest - -from openjarvis.connectors.retriever import TwoStageRetriever -from openjarvis.connectors.store import KnowledgeStore -from openjarvis.tools.storage._stubs import RetrievalResult - - -@pytest.fixture -def store(tmp_path: Path) -> KnowledgeStore: - s = KnowledgeStore(db_path=str(tmp_path / "retriever_test.db")) - s.store( - content="Kubernetes migration proposal with cost analysis", - source="gdrive", - doc_type="document", - author="sarah", - title="K8s Proposal", - ) - s.store( - content="Discussion about Kubernetes deployment timeline", - source="slack", - doc_type="message", - author="mike", - title="#infrastructure", - ) - s.store( - content="Quarterly budget review for cloud services", - source="gmail", - doc_type="email", - author="alice", - title="Re: Budget Q3", - ) - s.store( - content="Meeting notes about project planning", - source="granola", - doc_type="document", - author="bob", - title="Sprint Planning", - ) - return s - - -@pytest.fixture -def retriever(store: KnowledgeStore) -> TwoStageRetriever: - return TwoStageRetriever(store=store) - - -def test_retrieve_returns_results(retriever: TwoStageRetriever) -> None: - results = retriever.retrieve("Kubernetes migration") - assert len(results) > 0 - assert all(isinstance(r, RetrievalResult) for r in results) - - -def test_retrieve_respects_top_k(retriever: TwoStageRetriever) -> None: - results = retriever.retrieve("cloud", top_k=2) - assert len(results) <= 2 - - -def test_retrieve_with_source_filter( - retriever: TwoStageRetriever, -) -> None: - results = retriever.retrieve("Kubernetes", source="gdrive") - assert len(results) >= 1 - assert all( - r.metadata.get("source") == "gdrive" for r in results - ) - - -def test_retrieve_with_author_filter( - retriever: TwoStageRetriever, -) -> None: - results = retriever.retrieve("budget", author="alice") - assert len(results) >= 1 - assert all( - r.metadata.get("author") == "alice" for r in results - ) - - -def test_retrieve_bm25_only_when_no_colbert( - retriever: TwoStageRetriever, -) -> None: - """Without ColBERT, falls back to BM25-only results.""" - results = retriever.retrieve("planning") - assert len(results) > 0 - # Results should still have scores - assert all(r.score >= 0 for r in results) - - -def test_retrieve_with_colbert_reranking( - store: KnowledgeStore, -) -> None: - """With a mock ColBERT reranker, results are reranked.""" - mock_reranker = MagicMock() - mock_reranker.rerank.return_value = [ - RetrievalResult( - content="Kubernetes migration proposal with cost analysis", - score=0.95, - source="gdrive", - metadata={"source": "gdrive", "author": "sarah"}, - ), - ] - - retriever = TwoStageRetriever( - store=store, reranker=mock_reranker - ) - results = retriever.retrieve("Kubernetes", top_k=5) - assert len(results) >= 1 - mock_reranker.rerank.assert_called_once() - - -def test_retrieve_no_results(retriever: TwoStageRetriever) -> None: - results = retriever.retrieve("xyznonexistent999") - assert len(results) == 0 - - -def test_retrieve_recall_k_larger_than_top_k( - retriever: TwoStageRetriever, -) -> None: - """BM25 recall fetches more candidates than final top_k.""" - results = retriever.retrieve("Kubernetes", top_k=1) - assert len(results) <= 1 -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_retriever.py -v` - -Expected: FAIL — `ModuleNotFoundError` - -- [ ] **Step 3: Implement TwoStageRetriever** - -Create `src/openjarvis/connectors/retriever.py`: - -```python -"""Two-stage retriever: BM25 recall from KnowledgeStore → optional ColBERT rerank. - -Stage 1 (BM25): Fast keyword recall via SQLite FTS5. Returns top-N candidates. -Stage 2 (ColBERT): Semantic reranking of candidates via token-level MaxSim scoring. - Falls back to BM25-only if ColBERT is not available. -""" - -from __future__ import annotations - -import logging -from abc import ABC, abstractmethod -from typing import Any, List, Optional - -from openjarvis.connectors.store import KnowledgeStore -from openjarvis.tools.storage._stubs import RetrievalResult - -logger = logging.getLogger(__name__) - - -class Reranker(ABC): - """Abstract base for reranking BM25 candidates.""" - - @abstractmethod - def rerank( - self, - query: str, - candidates: List[RetrievalResult], - *, - top_k: int = 10, - ) -> List[RetrievalResult]: - """Rerank candidates and return top-k by semantic relevance.""" - - -class ColBERTReranker(Reranker): - """Reranker using ColBERTv2 late-interaction MaxSim scoring. - - Lazily loads the ColBERT checkpoint on first use. - """ - - def __init__( - self, - *, - checkpoint: str = "colbert-ir/colbertv2.0", - device: str = "cpu", - ) -> None: - self._checkpoint_name = checkpoint - self._device = device - self._loaded = False - self._checkpoint = None - - def _load(self) -> None: - if self._loaded: - return - try: - from colbert.modeling.checkpoint import Checkpoint - - self._checkpoint = Checkpoint( - self._checkpoint_name, colbert_config=None - ) - self._loaded = True - logger.info("ColBERT checkpoint loaded: %s", self._checkpoint_name) - except ImportError: - logger.warning("colbert-ai not installed, reranking disabled") - self._loaded = True # Don't retry - except Exception as exc: - logger.warning("ColBERT load failed: %s", exc) - self._loaded = True - - def _encode(self, text: str) -> Any: - if self._checkpoint is None: - return None - return self._checkpoint.queryFromText([text])[0] - - @staticmethod - def _maxsim(query_embs: Any, doc_embs: Any) -> float: - import torch - - # query_embs: (Q, D), doc_embs: (T, D) - sim = torch.nn.functional.cosine_similarity( - query_embs.unsqueeze(1), doc_embs.unsqueeze(0), dim=2 - ) - return sim.max(dim=1).values.sum().item() - - def rerank( - self, - query: str, - candidates: List[RetrievalResult], - *, - top_k: int = 10, - ) -> List[RetrievalResult]: - self._load() - if self._checkpoint is None: - return candidates[:top_k] - - query_embs = self._encode(query) - if query_embs is None: - return candidates[:top_k] - - scored = [] - for r in candidates: - doc_embs = self._encode(r.content) - if doc_embs is None: - scored.append((r, r.score)) - else: - score = self._maxsim(query_embs, doc_embs) - scored.append((r, score)) - - scored.sort(key=lambda x: x[1], reverse=True) - results = [] - for r, score in scored[:top_k]: - results.append( - RetrievalResult( - content=r.content, - score=score, - source=r.source, - metadata=r.metadata, - ) - ) - return results - - -class TwoStageRetriever: - """Composes BM25 recall with optional semantic reranking. - - Parameters - ---------- - store: - KnowledgeStore for BM25 recall via FTS5. - reranker: - Optional Reranker (e.g., ColBERTReranker). If None, BM25-only. - recall_k: - Number of BM25 candidates to fetch for reranking (default: 100). - """ - - def __init__( - self, - store: KnowledgeStore, - reranker: Optional[Reranker] = None, - *, - recall_k: int = 100, - ) -> None: - self._store = store - self._reranker = reranker - self._recall_k = recall_k - - def retrieve( - self, - query: str, - *, - top_k: int = 10, - source: str = "", - doc_type: str = "", - author: str = "", - since: str = "", - until: str = "", - ) -> List[RetrievalResult]: - """Two-stage retrieval: BM25 recall → optional rerank. - - Falls back to BM25-only if no reranker is configured. - """ - # Stage 1: BM25 recall - recall_n = max(self._recall_k, top_k * 3) - candidates = self._store.retrieve( - query, - top_k=recall_n, - source=source, - doc_type=doc_type, - author=author, - since=since, - until=until, - ) - - if not candidates: - return [] - - # Stage 2: Rerank (or just truncate) - if self._reranker is not None and len(candidates) > top_k: - return self._reranker.rerank( - query, candidates, top_k=top_k - ) - - return candidates[:top_k] -``` - -- [ ] **Step 4: Run tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_retriever.py -v` - -Expected: All 8 tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/openjarvis/connectors/retriever.py tests/connectors/test_retriever.py -git commit -m "feat: add TwoStageRetriever with BM25 recall and pluggable ColBERT reranking" -``` - ---- - -### Task 2: Upgrade knowledge_search to Use TwoStageRetriever - -**Files:** -- Modify: `src/openjarvis/tools/knowledge_search.py` -- Modify: `tests/tools/test_knowledge_search.py` - -- [ ] **Step 1: Write new test for two-stage retrieval** - -Add to `tests/tools/test_knowledge_search.py`: - -```python -def test_tool_uses_two_stage_retriever(tmp_path: Path) -> None: - """When initialized with a TwoStageRetriever, uses it.""" - from openjarvis.connectors.retriever import TwoStageRetriever - - store = KnowledgeStore(db_path=str(tmp_path / "ts_test.db")) - store.store( - content="Deep learning research paper", - source="gdrive", - doc_type="document", - ) - retriever = TwoStageRetriever(store=store) - tool = KnowledgeSearchTool(store=store, retriever=retriever) - result = tool.execute(query="deep learning") - assert result.success - assert result.metadata["num_results"] > 0 -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/tools/test_knowledge_search.py::test_tool_uses_two_stage_retriever -v` - -Expected: FAIL — `TypeError: __init__() got an unexpected keyword argument 'retriever'` - -- [ ] **Step 3: Update KnowledgeSearchTool** - -Modify `src/openjarvis/tools/knowledge_search.py` — add optional `retriever` parameter: - -In `__init__`, add `retriever` parameter: - -```python -def __init__( - self, - store: Optional[KnowledgeStore] = None, - retriever: Optional[Any] = None, -) -> None: - self._store = store - self._retriever = retriever -``` - -In `execute`, use retriever if available, else fall back to store: - -```python -# In execute(), replace the store.retrieve() call with: -if self._retriever is not None: - results = self._retriever.retrieve( - query, - top_k=top_k, - source=params.get("source", ""), - doc_type=params.get("doc_type", ""), - author=params.get("author", ""), - since=params.get("since", ""), - until=params.get("until", ""), - ) -elif self._store is not None: - results = self._store.retrieve( - query, - top_k=top_k, - source=params.get("source", ""), - doc_type=params.get("doc_type", ""), - author=params.get("author", ""), - since=params.get("since", ""), - until=params.get("until", ""), - ) -else: - return ToolResult( - tool_name="knowledge_search", - content="No knowledge store configured.", - success=False, - ) -``` - -Also update the no-store check at the top of `execute()` to allow retriever-only mode: - -```python -if self._store is None and self._retriever is None: - return ToolResult( - tool_name="knowledge_search", - content="No knowledge store configured. Run 'jarvis connect' to set up data sources.", - success=False, - ) -``` - -- [ ] **Step 4: Run all knowledge_search tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/tools/test_knowledge_search.py -v` - -Expected: All tests PASS (existing + new). - -- [ ] **Step 5: Commit** - -```bash -git add src/openjarvis/tools/knowledge_search.py tests/tools/test_knowledge_search.py -git commit -m "feat: upgrade knowledge_search to support TwoStageRetriever" -``` - ---- - -### Task 3: DeepResearchAgent - -**Files:** -- Create: `src/openjarvis/agents/deep_research.py` -- Create: `tests/agents/test_deep_research.py` - -- [ ] **Step 1: Write failing tests** - -Create `tests/agents/test_deep_research.py`: - -```python -"""Tests for DeepResearchAgent — multi-hop research with citations.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any, Dict, List, Optional -from unittest.mock import MagicMock - -import pytest - -from openjarvis.agents.deep_research import DeepResearchAgent -from openjarvis.connectors.store import KnowledgeStore -from openjarvis.core.registry import AgentRegistry -from openjarvis.core.types import ToolResult -from openjarvis.tools._stubs import BaseTool, ToolSpec -from openjarvis.tools.knowledge_search import KnowledgeSearchTool - - -@pytest.fixture -def store(tmp_path: Path) -> KnowledgeStore: - s = KnowledgeStore(db_path=str(tmp_path / "dr_test.db")) - s.store( - content="Kubernetes migration was proposed by Sarah in March", - source="slack", - doc_type="message", - author="sarah", - title="#infrastructure", - url="https://slack.com/archives/C001/p123", - ) - s.store( - content="Cost analysis shows 40% increase during transition", - source="gdrive", - doc_type="document", - author="sarah", - title="K8s Cost Analysis", - url="https://drive.google.com/d/doc1", - ) - s.store( - content="Migration approved by engineering leads on March 8", - source="gmail", - doc_type="email", - author="mike", - title="Re: K8s migration approved", - url="https://mail.google.com/mail/u/0/#inbox/msg1", - ) - return s - - -@pytest.fixture -def mock_engine(): - engine = MagicMock() - engine.engine_id = "mock" - engine.health.return_value = True - return engine - - -def _make_engine_response( - content: str, tool_calls: list | None = None -) -> dict: - result: Dict[str, Any] = { - "content": content, - "usage": { - "prompt_tokens": 50, - "completion_tokens": 50, - "total_tokens": 100, - }, - "model": "test-model", - "finish_reason": "stop", - } - if tool_calls: - result["tool_calls"] = tool_calls - result["finish_reason"] = "tool_calls" - return result - - -def test_agent_registration() -> None: - import openjarvis.agents.deep_research # noqa: F401 - - AgentRegistry.register_value( - "deep_research", DeepResearchAgent - ) - assert AgentRegistry.contains("deep_research") - - -def test_agent_produces_result( - mock_engine: MagicMock, store: KnowledgeStore -) -> None: - """Agent returns an AgentResult with content.""" - mock_engine.generate.return_value = _make_engine_response( - "Based on my research, the Kubernetes migration was " - "proposed by Sarah.\n\n**Sources:**\n" - "1. [slack] #infrastructure" - ) - - ks_tool = KnowledgeSearchTool(store=store) - agent = DeepResearchAgent( - engine=mock_engine, - model="test-model", - tools=[ks_tool], - max_turns=5, - ) - - result = agent.run("What was the K8s migration context?") - assert result.content - assert result.turns >= 1 - - -def test_agent_uses_knowledge_search( - mock_engine: MagicMock, store: KnowledgeStore -) -> None: - """Agent calls knowledge_search tool during research.""" - # First call: agent wants to search - call1 = _make_engine_response( - "", - tool_calls=[ - { - "id": "call_1", - "type": "function", - "function": { - "name": "knowledge_search", - "arguments": json.dumps( - {"query": "Kubernetes migration"} - ), - }, - } - ], - ) - # Second call: agent synthesizes - call2 = _make_engine_response( - "The Kubernetes migration was proposed by Sarah in March.\n\n" - "**Sources:**\n" - "1. [slack] #infrastructure — sarah\n" - "2. [gdrive] K8s Cost Analysis — sarah" - ) - mock_engine.generate.side_effect = [call1, call2] - - ks_tool = KnowledgeSearchTool(store=store) - agent = DeepResearchAgent( - engine=mock_engine, - model="test-model", - tools=[ks_tool], - max_turns=5, - ) - - result = agent.run("What was the K8s migration context?") - assert result.content - assert result.turns >= 1 - assert len(result.tool_results) >= 1 - assert result.tool_results[0].tool_name == "knowledge_search" - assert result.tool_results[0].success - - -def test_agent_respects_max_turns( - mock_engine: MagicMock, store: KnowledgeStore -) -> None: - """Agent stops after max_turns even if not done.""" - # Always return a tool call (infinite loop without max_turns) - mock_engine.generate.return_value = _make_engine_response( - "", - tool_calls=[ - { - "id": "call_n", - "type": "function", - "function": { - "name": "knowledge_search", - "arguments": json.dumps( - {"query": "anything"} - ), - }, - } - ], - ) - - ks_tool = KnowledgeSearchTool(store=store) - agent = DeepResearchAgent( - engine=mock_engine, - model="test-model", - tools=[ks_tool], - max_turns=3, - ) - - result = agent.run("Research something") - assert result.turns <= 3 - - -def test_agent_system_prompt_mentions_research( - mock_engine: MagicMock, store: KnowledgeStore -) -> None: - """System prompt should mention research and citations.""" - mock_engine.generate.return_value = _make_engine_response( - "Final answer here." - ) - - ks_tool = KnowledgeSearchTool(store=store) - agent = DeepResearchAgent( - engine=mock_engine, - model="test-model", - tools=[ks_tool], - ) - - agent.run("test") - - # Check the system message passed to generate - call_args = mock_engine.generate.call_args - messages = call_args[0][0] # first positional arg - system_msg = messages[0] - assert "research" in system_msg.content.lower() - assert "source" in system_msg.content.lower() - - -def test_agent_defaults( - mock_engine: MagicMock, store: KnowledgeStore -) -> None: - """Check default max_turns and agent_id.""" - ks_tool = KnowledgeSearchTool(store=store) - agent = DeepResearchAgent( - engine=mock_engine, - model="test-model", - tools=[ks_tool], - ) - assert agent.agent_id == "deep_research" - assert agent._max_turns == 5 -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/agents/test_deep_research.py -v` - -Expected: FAIL — `ModuleNotFoundError` - -- [ ] **Step 3: Implement DeepResearchAgent** - -Create `src/openjarvis/agents/deep_research.py`: - -```python -"""DeepResearchAgent — multi-hop research over personal knowledge base. - -Extends ToolUsingAgent with: -- Research-oriented system prompt emphasizing citations and sources -- Multi-hop retrieval loop (up to max_turns subqueries) -- Structured report output with cross-platform source attribution -""" - -from __future__ import annotations - -import json -import logging -from typing import Any, Dict, List, Optional - -from openjarvis.agents._stubs import ( - AgentContext, - AgentResult, - ToolUsingAgent, -) -from openjarvis.core.events import EventBus -from openjarvis.core.registry import AgentRegistry -from openjarvis.core.types import Message, Role, ToolCall, ToolResult -from openjarvis.engine._stubs import InferenceEngine -from openjarvis.tools._stubs import BaseTool, build_tool_descriptions - -logger = logging.getLogger(__name__) - -_SYSTEM_PROMPT = """\ -You are a Deep Research agent with access to a personal knowledge base \ -containing emails, messages, documents, calendar events, contacts, and \ -notes from multiple sources (Gmail, Slack, Google Drive, Notion, \ -Granola, Obsidian, and more). - -Your job is to answer the user's question by searching across these \ -sources, synthesizing information, and producing a comprehensive \ -research report with citations. - -## How to Research - -1. Use the `knowledge_search` tool to find relevant information. \ -You can filter by source, doc_type, author, and date range. -2. Use the `think` tool to reason about what you've found and plan \ -your next search. -3. Make multiple searches to cross-reference across sources. \ -A question about a decision might involve Slack threads, email chains, \ -documents, and calendar events. -4. When you have enough information, synthesize a clear answer. - -## Output Format - -Structure your response as: - -1. A narrative answer addressing the user's question -2. Inline citations in the format [source] title — author -3. A "Sources" section at the end listing all referenced items with \ -their platform, title, author, and URL - -## Important - -- Always cite your sources — never present information without \ -attribution -- Cross-reference across platforms when possible -- If you can't find enough information, say so clearly -- Prefer recent information over older content when relevant - -{tool_descriptions}""" - - -@AgentRegistry.register("deep_research") -class DeepResearchAgent(ToolUsingAgent): - """Multi-hop research agent for personal knowledge base queries. - - Searches across indexed personal data (emails, messages, documents, - calendar events, contacts, notes) using the knowledge_search tool, - performs multi-hop retrieval to cross-reference sources, and - synthesizes findings into a cited research report. - - Parameters - ---------- - engine: - Inference engine for LLM calls. - model: - Model name/ID. - tools: - List of tools (should include KnowledgeSearchTool at minimum). - max_turns: - Maximum research hops (default: 5). - """ - - agent_id = "deep_research" - _default_max_turns = 5 - _default_temperature = 0.3 - _default_max_tokens = 4096 - - def __init__( - self, - engine: InferenceEngine, - model: str, - *, - tools: Optional[List[BaseTool]] = None, - bus: Optional[EventBus] = None, - max_turns: Optional[int] = None, - temperature: Optional[float] = None, - max_tokens: Optional[int] = None, - interactive: bool = False, - confirm_callback: Any = None, - ) -> None: - super().__init__( - engine, - model, - tools=tools, - bus=bus, - max_turns=max_turns or self._default_max_turns, - temperature=temperature or self._default_temperature, - max_tokens=max_tokens or self._default_max_tokens, - interactive=interactive, - confirm_callback=confirm_callback, - ) - - def run( - self, - input: str, - context: Optional[AgentContext] = None, - **kwargs: Any, - ) -> AgentResult: - """Execute a multi-hop research query. - - The agent loops through: search → think → search → ... → synthesize, - up to max_turns iterations. - """ - self._emit_turn_start(input) - - # Build system prompt with tool descriptions - tool_desc = build_tool_descriptions(self._tools) - system_prompt = _SYSTEM_PROMPT.format( - tool_descriptions=tool_desc - ) - - # Assemble initial messages - messages = self._build_messages( - input, context, system_prompt=system_prompt - ) - - all_tool_results: List[ToolResult] = [] - turns = 0 - total_usage: Dict[str, int] = { - "prompt_tokens": 0, - "completion_tokens": 0, - "total_tokens": 0, - } - - for _turn in range(self._max_turns): - turns += 1 - - # Generate response - result = self._generate(messages) - usage = result.get("usage", {}) - for k in total_usage: - total_usage[k] += usage.get(k, 0) - - content = result.get("content", "") - tool_calls_raw = result.get("tool_calls", []) - - # No tool calls → final answer - if not tool_calls_raw: - final_content = self._strip_think_tags(content) - self._emit_turn_end(turns=turns) - return AgentResult( - content=final_content, - tool_results=all_tool_results, - turns=turns, - metadata={ - **total_usage, - "sources": self._extract_sources( - all_tool_results - ), - }, - ) - - # Append assistant message - messages.append( - Message(role=Role.ASSISTANT, content=content) - ) - - # Execute tool calls - for tc_raw in tool_calls_raw: - fn = tc_raw.get("function", {}) - tool_call = ToolCall( - id=tc_raw.get("id", f"dr_{turns}"), - name=fn.get("name", ""), - arguments=fn.get("arguments", "{}"), - ) - - tool_result = self._executor.execute(tool_call) - all_tool_results.append(tool_result) - - # Add tool result as message - messages.append( - Message( - role=Role.TOOL, - content=tool_result.content, - tool_call_id=tool_call.id, - name=tool_call.name, - ) - ) - - # Max turns exceeded — return what we have - self._emit_turn_end(turns=turns) - return self._max_turns_result( - all_tool_results, - turns, - metadata={ - **total_usage, - "sources": self._extract_sources(all_tool_results), - }, - ) - - @staticmethod - def _extract_sources( - tool_results: List[ToolResult], - ) -> List[str]: - """Extract source references from tool results for metadata.""" - sources: List[str] = [] - for tr in tool_results: - if tr.tool_name == "knowledge_search" and tr.success: - n = tr.metadata.get("num_results", 0) - if n > 0: - sources.append( - f"knowledge_search: {n} results" - ) - return sources -``` - -- [ ] **Step 4: Run tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/agents/test_deep_research.py -v` - -Expected: All 6 tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/openjarvis/agents/deep_research.py tests/agents/test_deep_research.py -git commit -m "feat: add DeepResearchAgent with multi-hop retrieval and cited reports" -``` - ---- - -### Task 4: End-to-End Integration Test - -**Files:** -- Create: `tests/agents/test_deep_research_integration.py` - -- [ ] **Step 1: Write integration test** - -Create `tests/agents/test_deep_research_integration.py`: - -```python -"""Integration test — full pipeline from connector to Deep Research agent.""" - -from __future__ import annotations - -import json -from pathlib import Path -from unittest.mock import MagicMock - -import pytest - -from openjarvis.agents.deep_research import DeepResearchAgent -from openjarvis.connectors._stubs import Document -from openjarvis.connectors.pipeline import IngestionPipeline -from openjarvis.connectors.retriever import TwoStageRetriever -from openjarvis.connectors.store import KnowledgeStore -from openjarvis.tools.knowledge_search import KnowledgeSearchTool - - -@pytest.fixture -def populated_store(tmp_path: Path) -> KnowledgeStore: - """KnowledgeStore with realistic multi-source data.""" - store = KnowledgeStore( - db_path=str(tmp_path / "integration.db") - ) - pipeline = IngestionPipeline(store=store, max_tokens=256) - - docs = [ - Document( - doc_id="slack:c001:t1", - source="slack", - doc_type="message", - content=( - "Sarah: We should migrate to Kubernetes. " - "The current Docker Swarm setup can't handle our scale." - ), - title="#infrastructure", - author="sarah", - ), - Document( - doc_id="gmail:msg1", - source="gmail", - doc_type="email", - content=( - "Hi team, I've completed the cost analysis for " - "the K8s migration. Estimated 40% increase in " - "cloud spend during the transition period, but " - "20% reduction long-term." - ), - title="K8s Cost Analysis", - author="mike", - ), - Document( - doc_id="gdrive:doc1", - source="gdrive", - doc_type="document", - content=( - "# Kubernetes Migration Proposal\n\n" - "## Timeline\n" - "Six-week migration window starting April 1st.\n\n" - "## Team\n" - "Sarah (lead), Mike (infra), Bob (testing)\n\n" - "## Risks\n" - "Downtime during cutover, learning curve for team." - ), - title="K8s Migration Proposal v2", - author="sarah", - ), - Document( - doc_id="gcalendar:evt1", - source="gcalendar", - doc_type="event", - content=( - "Infrastructure Sync\n" - "When: March 5, 2024 10:00 AM\n" - "Attendees: Sarah, Mike, Bob\n" - "Agenda: Review K8s migration proposal" - ), - title="Infrastructure Sync", - author="sarah", - ), - Document( - doc_id="granola:not1", - source="granola", - doc_type="document", - content=( - "## Summary\n" - "Discussed K8s migration timeline. Sarah presented " - "the proposal. Mike raised cost concerns. " - "Decision: proceed with 6-week plan.\n\n" - "## Transcript\n" - "**sarah:** Let me walk through the timeline.\n" - "**mike:** What about the cost increase?\n" - "**sarah:** Short-term 40% increase, long-term savings." - ), - title="Infrastructure Sync Notes", - author="sarah", - ), - ] - - pipeline.ingest(docs) - return store - - -@pytest.fixture -def mock_engine(): - engine = MagicMock() - engine.engine_id = "mock" - engine.health.return_value = True - return engine - - -def test_full_research_pipeline( - populated_store: KnowledgeStore, - mock_engine: MagicMock, -) -> None: - """Full pipeline: multi-source data → search → agent → cited report.""" - # Agent searches, then synthesizes - search_call = { - "content": "", - "usage": { - "prompt_tokens": 100, - "completion_tokens": 50, - "total_tokens": 150, - }, - "model": "test", - "finish_reason": "tool_calls", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "knowledge_search", - "arguments": json.dumps( - {"query": "Kubernetes migration"} - ), - }, - } - ], - } - - final_answer = { - "content": ( - "## Kubernetes Migration — Decision Context\n\n" - "The migration to Kubernetes was proposed by Sarah " - "in the #infrastructure Slack channel " - "[slack] #infrastructure — sarah.\n\n" - "Mike completed a cost analysis showing a 40% " - "short-term increase [gmail] K8s Cost Analysis — mike.\n\n" - "The proposal document outlines a six-week timeline " - "[gdrive] K8s Migration Proposal v2 — sarah.\n\n" - "The team discussed and approved the plan during " - "the March 5th Infrastructure Sync " - "[granola] Infrastructure Sync Notes — sarah.\n\n" - "**Sources:**\n" - "1. [slack] #infrastructure — sarah\n" - "2. [gmail] K8s Cost Analysis — mike\n" - "3. [gdrive] K8s Migration Proposal v2 — sarah\n" - "4. [gcalendar] Infrastructure Sync — sarah\n" - "5. [granola] Infrastructure Sync Notes — sarah" - ), - "usage": { - "prompt_tokens": 500, - "completion_tokens": 200, - "total_tokens": 700, - }, - "model": "test", - "finish_reason": "stop", - } - - mock_engine.generate.side_effect = [search_call, final_answer] - - retriever = TwoStageRetriever(store=populated_store) - ks_tool = KnowledgeSearchTool( - store=populated_store, retriever=retriever - ) - - agent = DeepResearchAgent( - engine=mock_engine, - model="test-model", - tools=[ks_tool], - max_turns=5, - ) - - result = agent.run( - "What was the context around the Kubernetes migration decision?" - ) - - # Verify agent produced a result - assert result.content - assert "Kubernetes" in result.content - assert result.turns >= 1 - - # Verify tool was called - assert len(result.tool_results) >= 1 - assert result.tool_results[0].tool_name == "knowledge_search" - assert result.tool_results[0].success - - # Verify knowledge_search found cross-platform results - search_result = result.tool_results[0] - assert search_result.metadata.get("num_results", 0) > 0 - - -def test_search_finds_cross_platform_data( - populated_store: KnowledgeStore, -) -> None: - """Verify the store has data from multiple sources.""" - retriever = TwoStageRetriever(store=populated_store) - tool = KnowledgeSearchTool( - store=populated_store, retriever=retriever - ) - - result = tool.execute(query="Kubernetes migration") - assert result.success - assert result.metadata["num_results"] > 0 - - # Should find results from multiple sources - content = result.content - sources_found = set() - for src in ["slack", "gmail", "gdrive", "granola"]: - if f"[{src}]" in content: - sources_found.add(src) - assert len(sources_found) >= 2, ( - f"Expected cross-platform results, found: {sources_found}" - ) -``` - -- [ ] **Step 2: Run integration test** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/agents/test_deep_research_integration.py -v` - -Expected: All 2 tests PASS. - -- [ ] **Step 3: Run full test suite** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/ tests/agents/test_deep_research.py tests/agents/test_deep_research_integration.py tests/tools/test_knowledge_search.py -v` - -Expected: All tests PASS. - -- [ ] **Step 4: Run linter** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run ruff check src/openjarvis/connectors/retriever.py src/openjarvis/agents/deep_research.py src/openjarvis/tools/knowledge_search.py tests/connectors/test_retriever.py tests/agents/` - -Expected: No errors. - -- [ ] **Step 5: Commit** - -```bash -git add tests/agents/test_deep_research_integration.py -git commit -m "feat: add end-to-end integration test for Deep Research pipeline" -``` - ---- - -## Post-Plan Notes - -**What this plan produces:** -- `TwoStageRetriever` — composable BM25 recall + pluggable ColBERT reranking -- `ColBERTReranker` — lazy-loading reranker with MaxSim scoring (optional dependency) -- `KnowledgeSearchTool` upgraded to use two-stage retrieval -- `DeepResearchAgent` — multi-hop research agent with cross-platform citations -- Full integration test proving the pipeline end-to-end - -**ColBERT is optional:** The system works with BM25-only when ColBERT dependencies (torch, colbert-ai) aren't installed. ColBERT adds semantic reranking quality when available. - -**What comes next:** -- **Phase 4:** ChannelAgent + iMessage/WhatsApp/Slack plugins for chatting with the agent -- **Phase 5:** Incremental sync, attachment store, polish -- **Phase 2B:** Desktop wizard UI diff --git a/docs/superpowers/plans/2026-03-26-deep-research-agent-v2.md b/docs/superpowers/plans/2026-03-26-deep-research-agent-v2.md deleted file mode 100644 index 5058b85b..00000000 --- a/docs/superpowers/plans/2026-03-26-deep-research-agent-v2.md +++ /dev/null @@ -1,798 +0,0 @@ -# Deep Research Agent v2 Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `knowledge_sql` and `scan_chunks` tools to the DeepResearchAgent, rewrite the system prompt with query strategies, wire in the `think` tool, and validate with 4 test queries. - -**Architecture:** Two new tool files following the existing `BaseTool`/`ToolSpec`/`ToolResult` pattern. The `knowledge_sql` tool runs read-only SQL against `KnowledgeStore._conn`. The `scan_chunks` tool pulls chunks from the store, batches them, and calls the LM to extract findings. The system prompt teaches the model when to use each tool. Everything wired together in `_launch_chat`. - -**Tech Stack:** Python 3.10+, sqlite3, pytest - ---- - -### Task 1: Create `knowledge_sql` tool - -**Files:** -- Create: `src/openjarvis/tools/knowledge_sql.py` -- Create: `tests/tools/test_knowledge_sql.py` - -- [ ] **Step 1: Write the test file** - -Create `tests/tools/__init__.py` if it doesn't exist, then create `tests/tools/test_knowledge_sql.py`: - -```python -"""Tests for KnowledgeSQLTool.""" - -from __future__ import annotations - -import tempfile -from pathlib import Path - -import pytest - -from openjarvis.connectors.store import KnowledgeStore -from openjarvis.core.registry import ToolRegistry - - -@pytest.fixture() -def store(tmp_path: Path) -> KnowledgeStore: - ks = KnowledgeStore(str(tmp_path / "test.db")) - ks.store("Hello from Alice", source="imessage", author="Alice", doc_type="message") - ks.store("Hello from Alice again", source="imessage", author="Alice", doc_type="message") - ks.store("Meeting notes Q1", source="granola", author="Bob", doc_type="document") - ks.store("Email about Spain trip", source="gmail", author="Carol", doc_type="email") - return ks - - -def test_select_count(store: KnowledgeStore) -> None: - from openjarvis.tools.knowledge_sql import KnowledgeSQLTool - tool = KnowledgeSQLTool(store=store) - result = tool.execute(query="SELECT COUNT(*) as total FROM knowledge_chunks") - assert result.success - assert "4" in result.content - - -def test_group_by_author(store: KnowledgeStore) -> None: - from openjarvis.tools.knowledge_sql import KnowledgeSQLTool - tool = KnowledgeSQLTool(store=store) - result = tool.execute( - query="SELECT author, COUNT(*) as n FROM knowledge_chunks GROUP BY author ORDER BY n DESC" - ) - assert result.success - assert "Alice" in result.content - assert "2" in result.content - - -def test_rejects_non_select(store: KnowledgeStore) -> None: - from openjarvis.tools.knowledge_sql import KnowledgeSQLTool - tool = KnowledgeSQLTool(store=store) - result = tool.execute(query="DELETE FROM knowledge_chunks") - assert not result.success - assert "read-only" in result.content.lower() or "SELECT" in result.content - - -def test_rejects_drop(store: KnowledgeStore) -> None: - from openjarvis.tools.knowledge_sql import KnowledgeSQLTool - tool = KnowledgeSQLTool(store=store) - result = tool.execute(query="DROP TABLE knowledge_chunks") - assert not result.success - - -def test_handles_bad_sql(store: KnowledgeStore) -> None: - from openjarvis.tools.knowledge_sql import KnowledgeSQLTool - tool = KnowledgeSQLTool(store=store) - result = tool.execute(query="SELECT * FROM nonexistent_table") - assert not result.success - - -def test_filter_by_source(store: KnowledgeStore) -> None: - from openjarvis.tools.knowledge_sql import KnowledgeSQLTool - tool = KnowledgeSQLTool(store=store) - result = tool.execute( - query="SELECT title, author FROM knowledge_chunks WHERE source = 'gmail'" - ) - assert result.success - assert "Carol" in result.content - - -def test_registered() -> None: - from openjarvis.tools.knowledge_sql import KnowledgeSQLTool - ToolRegistry.register_value("knowledge_sql", KnowledgeSQLTool) - assert ToolRegistry.contains("knowledge_sql") -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/tools/test_knowledge_sql.py -v --tb=short -``` - -Expected: FAIL — `ModuleNotFoundError` - -- [ ] **Step 3: Create the tool** - -Create `src/openjarvis/tools/knowledge_sql.py`: - -```python -"""KnowledgeSQLTool — read-only SQL queries against the KnowledgeStore. - -Allows agents to run SELECT queries for aggregation, counting, ranking, -and filtering operations that BM25 search cannot handle. -""" - -from __future__ import annotations - -import sqlite3 -from typing import Any, Optional - -from openjarvis.connectors.store import KnowledgeStore -from openjarvis.core.registry import ToolRegistry -from openjarvis.core.types import ToolResult -from openjarvis.tools._stubs import BaseTool, ToolSpec - -_MAX_ROWS = 50 - -_SCHEMA_DESCRIPTION = ( - "Table: knowledge_chunks\n" - "Columns: id, content, source, doc_type, doc_id, title, author, " - "participants, timestamp, thread_id, url, metadata, chunk_index" -) - - -@ToolRegistry.register("knowledge_sql") -class KnowledgeSQLTool(BaseTool): - """Run read-only SQL against the knowledge store for aggregation queries.""" - - tool_id = "knowledge_sql" - - def __init__(self, store: Optional[KnowledgeStore] = None) -> None: - self._store = store - - @property - def spec(self) -> ToolSpec: - return ToolSpec( - name="knowledge_sql", - description=( - "Run a read-only SQL SELECT query against the knowledge_chunks table. " - "Use for counting, ranking, aggregation, and filtering. " - f"{_SCHEMA_DESCRIPTION}" - ), - parameters={ - "type": "object", - "properties": { - "query": { - "type": "string", - "description": ( - "SQL SELECT query. Only SELECT statements allowed. " - "Example: SELECT author, COUNT(*) as n FROM knowledge_chunks " - "WHERE source='imessage' GROUP BY author ORDER BY n DESC LIMIT 10" - ), - }, - }, - "required": ["query"], - }, - category="knowledge", - ) - - def execute(self, **params: Any) -> ToolResult: - if self._store is None: - return ToolResult( - tool_name="knowledge_sql", - content="No knowledge store configured.", - success=False, - ) - - query: str = params.get("query", "").strip() - if not query: - return ToolResult( - tool_name="knowledge_sql", - content="No query provided.", - success=False, - ) - - # Read-only enforcement - normalized = query.lstrip().upper() - if not normalized.startswith("SELECT"): - return ToolResult( - tool_name="knowledge_sql", - content="Only SELECT queries are allowed (read-only).", - success=False, - ) - - # Block dangerous keywords even in SELECT subqueries - for forbidden in ("DROP", "DELETE", "INSERT", "UPDATE", "ALTER", "CREATE", "ATTACH"): - if forbidden in normalized: - return ToolResult( - tool_name="knowledge_sql", - content=f"Query contains forbidden keyword: {forbidden}. Only SELECT queries allowed.", - success=False, - ) - - try: - rows = self._store._conn.execute(query).fetchmany(_MAX_ROWS) - except sqlite3.OperationalError as exc: - return ToolResult( - tool_name="knowledge_sql", - content=f"SQL error: {exc}", - success=False, - ) - - if not rows: - return ToolResult( - tool_name="knowledge_sql", - content="Query returned no results.", - success=True, - metadata={"num_rows": 0}, - ) - - # Format as table - columns = rows[0].keys() - lines = [" | ".join(columns)] - lines.append(" | ".join("---" for _ in columns)) - for row in rows: - lines.append(" | ".join(str(row[c]) for c in columns)) - - return ToolResult( - tool_name="knowledge_sql", - content="\n".join(lines), - success=True, - metadata={"num_rows": len(rows)}, - ) - - -__all__ = ["KnowledgeSQLTool"] -``` - -- [ ] **Step 4: Ensure tests/tools/__init__.py exists** - -```bash -touch tests/tools/__init__.py -``` - -- [ ] **Step 5: Run tests** - -```bash -uv run pytest tests/tools/test_knowledge_sql.py -v --tb=short -``` - -Expected: 7/7 PASS. - -- [ ] **Step 6: Commit** - -```bash -git add src/openjarvis/tools/knowledge_sql.py tests/tools/__init__.py tests/tools/test_knowledge_sql.py -git commit -m "feat: add knowledge_sql tool for read-only SQL aggregation queries" -``` - ---- - -### Task 2: Create `scan_chunks` tool - -**Files:** -- Create: `src/openjarvis/tools/scan_chunks.py` -- Create: `tests/tools/test_scan_chunks.py` - -- [ ] **Step 1: Write the test file** - -Create `tests/tools/test_scan_chunks.py`: - -```python -"""Tests for ScanChunksTool.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any, Dict -from unittest.mock import MagicMock - -import pytest - -from openjarvis.connectors.store import KnowledgeStore -from openjarvis.core.registry import ToolRegistry - - -@pytest.fixture() -def store(tmp_path: Path) -> KnowledgeStore: - ks = KnowledgeStore(str(tmp_path / "test.db")) - ks.store("Met with Sequoia about Series A", source="granola", doc_type="document") - ks.store("Fundraising discussion with a]16z", source="granola", doc_type="document") - ks.store("Weekly standup notes", source="granola", doc_type="document") - ks.store("Trip to Spain with family", source="imessage", doc_type="message") - return ks - - -def _fake_engine() -> MagicMock: - """Mock engine that echoes back a summary.""" - engine = MagicMock() - engine.generate.return_value = { - "content": "Found: Sequoia Series A discussion, a16z fundraising", - "usage": {}, - } - return engine - - -def test_scan_finds_semantic_matches(store: KnowledgeStore) -> None: - from openjarvis.tools.scan_chunks import ScanChunksTool - engine = _fake_engine() - tool = ScanChunksTool(store=store, engine=engine, model="test") - result = tool.execute(question="Which VCs have I spoken with?") - assert result.success - assert "Sequoia" in result.content or "Found" in result.content - # Engine should have been called with chunks - assert engine.generate.called - - -def test_scan_respects_source_filter(store: KnowledgeStore) -> None: - from openjarvis.tools.scan_chunks import ScanChunksTool - engine = _fake_engine() - tool = ScanChunksTool(store=store, engine=engine, model="test") - result = tool.execute(question="What trips?", source="imessage") - assert result.success - # Should only have passed imessage chunks to the engine - call_args = engine.generate.call_args - messages = call_args[0][0] if call_args[0] else call_args[1].get("messages", []) - # At least one call should contain "Spain" - all_content = str(messages) - assert "Spain" in all_content - - -def test_scan_empty_store(tmp_path: Path) -> None: - from openjarvis.tools.scan_chunks import ScanChunksTool - ks = KnowledgeStore(str(tmp_path / "empty.db")) - engine = _fake_engine() - tool = ScanChunksTool(store=ks, engine=engine, model="test") - result = tool.execute(question="Anything?") - assert result.success - assert "no chunks" in result.content.lower() or result.content == "" - - -def test_registered() -> None: - from openjarvis.tools.scan_chunks import ScanChunksTool - ToolRegistry.register_value("scan_chunks", ScanChunksTool) - assert ToolRegistry.contains("scan_chunks") -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/tools/test_scan_chunks.py -v --tb=short -``` - -Expected: FAIL — `ModuleNotFoundError` - -- [ ] **Step 3: Create the tool** - -Create `src/openjarvis/tools/scan_chunks.py`: - -```python -"""ScanChunksTool — semantic grep via LM-powered chunk scanning. - -Pulls chunks from the KnowledgeStore by filter, batches them, and asks the -LM to extract information relevant to a question. Catches semantic matches -that keyword-based BM25 search misses. -""" - -from __future__ import annotations - -from typing import Any, List, Optional - -from openjarvis.connectors.store import KnowledgeStore -from openjarvis.core.registry import ToolRegistry -from openjarvis.core.types import Message, Role, ToolResult -from openjarvis.engine._stubs import InferenceEngine -from openjarvis.tools._stubs import BaseTool, ToolSpec - -_DEFAULT_MAX_CHUNKS = 200 -_DEFAULT_BATCH_SIZE = 20 - - -@ToolRegistry.register("scan_chunks") -class ScanChunksTool(BaseTool): - """Semantic grep — feeds chunks to the LM to find information BM25 misses.""" - - tool_id = "scan_chunks" - - def __init__( - self, - store: Optional[KnowledgeStore] = None, - engine: Optional[InferenceEngine] = None, - model: str = "", - ) -> None: - self._store = store - self._engine = engine - self._model = model - - @property - def spec(self) -> ToolSpec: - return ToolSpec( - name="scan_chunks", - description=( - "Semantic search — feeds chunks from the knowledge store to an LM " - "that reads the actual text looking for relevant information. " - "Use when keyword search (knowledge_search) misses semantic matches " - "(e.g. searching for 'VCs' when text says 'fundraising round'). " - "Slower but catches what BM25 misses. " - "Filters: source, doc_type, since, until." - ), - parameters={ - "type": "object", - "properties": { - "question": { - "type": "string", - "description": "What to look for in the chunks.", - }, - "source": { - "type": "string", - "description": "Filter by source (e.g. 'granola', 'gmail').", - }, - "doc_type": { - "type": "string", - "description": "Filter by doc type (e.g. 'document', 'email').", - }, - "since": { - "type": "string", - "description": "Only chunks after this ISO timestamp.", - }, - "until": { - "type": "string", - "description": "Only chunks before this ISO timestamp.", - }, - "max_chunks": { - "type": "integer", - "description": f"Max chunks to scan (default {_DEFAULT_MAX_CHUNKS}).", - }, - }, - "required": ["question"], - }, - category="knowledge", - ) - - def execute(self, **params: Any) -> ToolResult: - if self._store is None or self._engine is None: - return ToolResult( - tool_name="scan_chunks", - content="Scan tool not configured (missing store or engine).", - success=False, - ) - - question: str = params.get("question", "") - if not question: - return ToolResult( - tool_name="scan_chunks", - content="No question provided.", - success=False, - ) - - source: str = params.get("source", "") - doc_type: str = params.get("doc_type", "") - since: str = params.get("since", "") - until: str = params.get("until", "") - max_chunks: int = int(params.get("max_chunks", _DEFAULT_MAX_CHUNKS)) - batch_size: int = _DEFAULT_BATCH_SIZE - - # Pull chunks matching filters - where_clauses: List[str] = [] - sql_params: List[Any] = [] - - if source: - where_clauses.append("source = ?") - sql_params.append(source) - if doc_type: - where_clauses.append("doc_type = ?") - sql_params.append(doc_type) - if since: - where_clauses.append("timestamp >= ?") - sql_params.append(since) - if until: - where_clauses.append("timestamp <= ?") - sql_params.append(until) - - where = "" - if where_clauses: - where = "WHERE " + " AND ".join(where_clauses) - - sql = f"SELECT content, source, title, author FROM knowledge_chunks {where} LIMIT ?" - sql_params.append(max_chunks) - - rows = self._store._conn.execute(sql, sql_params).fetchall() - - if not rows: - return ToolResult( - tool_name="scan_chunks", - content="No chunks found matching filters.", - success=True, - metadata={"chunks_scanned": 0}, - ) - - # Batch and scan - findings: List[str] = [] - for i in range(0, len(rows), batch_size): - batch = rows[i : i + batch_size] - batch_text = "\n\n---\n\n".join( - f"[{row['source']}] {row['title']} by {row['author']}:\n{row['content']}" - for row in batch - ) - - messages = [ - Message( - role=Role.USER, - content=( - f"/no_think\n" - f"Extract any information relevant to this question: {question}\n\n" - f"If nothing is relevant, reply with exactly: NOTHING_RELEVANT\n\n" - f"Chunks:\n{batch_text}" - ), - ), - ] - - result = self._engine.generate( - messages, model=self._model, max_tokens=1024 - ) - content = result.get("content", "").strip() - if content and "NOTHING_RELEVANT" not in content: - findings.append(content) - - if not findings: - return ToolResult( - tool_name="scan_chunks", - content=f"Scanned {len(rows)} chunks — no relevant information found.", - success=True, - metadata={"chunks_scanned": len(rows)}, - ) - - return ToolResult( - tool_name="scan_chunks", - content="\n\n".join(findings), - success=True, - metadata={"chunks_scanned": len(rows), "batches_with_findings": len(findings)}, - ) - - -__all__ = ["ScanChunksTool"] -``` - -- [ ] **Step 4: Run tests** - -```bash -uv run pytest tests/tools/test_scan_chunks.py -v --tb=short -``` - -Expected: 4/4 PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/openjarvis/tools/scan_chunks.py tests/tools/test_scan_chunks.py -git commit -m "feat: add scan_chunks tool for LM-powered semantic grep" -``` - ---- - -### Task 3: Rewrite system prompt + wire all tools + tune loop guard - -**Files:** -- Modify: `src/openjarvis/agents/deep_research.py` (system prompt + max_turns) -- Modify: `src/openjarvis/cli/deep_research_setup_cmd.py` (`_launch_chat` function) - -- [ ] **Step 1: Replace `DEEP_RESEARCH_SYSTEM_PROMPT` in `deep_research.py`** - -Replace the entire `DEEP_RESEARCH_SYSTEM_PROMPT` string (from line 33 to line 55) with: - -```python -DEEP_RESEARCH_SYSTEM_PROMPT = """\ -/no_think -You are a deep research agent with access to a personal knowledge base \ -containing emails, messages, meeting notes, documents, and notes. - -## Your Tools - -- **knowledge_search**: BM25 keyword search. Best for finding specific topics, \ -names, or phrases. Use filters (source, author, since, until) to narrow results. - -- **knowledge_sql**: Run read-only SQL against the knowledge_chunks table. \ -Schema: knowledge_chunks(id, content, source, doc_type, doc_id, title, \ -author, participants, timestamp, thread_id, url, metadata, chunk_index). \ -Best for counting, ranking, and aggregation. \ -Example: SELECT author, COUNT(*) as n FROM knowledge_chunks \ -WHERE source='imessage' GROUP BY author ORDER BY n DESC LIMIT 10 - -- **scan_chunks**: Semantic search — feeds chunks to an LM that reads the \ -actual text looking for relevant information. Use when keyword search returns \ -nothing useful or when you need semantic matching (e.g. searching for 'VCs' \ -when text says 'fundraising round'). Slower but catches what BM25 misses. - -- **think**: Reasoning scratchpad. Use between searches to plan your next \ -query, evaluate findings so far, and identify gaps. - -## Strategy - -1. Start with **think** to plan your approach — which tools suit this query? -2. For "who/what/how many" queries → use **knowledge_sql** with GROUP BY -3. For specific topics or names → use **knowledge_search** -4. If keyword search returns nothing useful → try **scan_chunks** with filters -5. Cross-reference across sources (emails, messages, meeting notes) -6. After gathering evidence → write a cited narrative report - -## Citation Format - -Cite sources as: [source] title -- author -End with a Sources section listing all referenced items. - -## Rules - -- Always cite your sources. Never present information without attribution. -- Make at least two searches to cross-reference across different sources. -- If a search returns no results, try a different tool or rephrase the query. -- Prefer specificity: filter by source, author, or date when appropriate. -- Your final answer should be a coherent narrative, not a list of raw results.""" -``` - -- [ ] **Step 2: Update `_default_max_turns` from 5 to 8** - -In `deep_research.py`, change: - -```python -_default_max_turns = 8 -``` - -- [ ] **Step 3: Update `_launch_chat` in `deep_research_setup_cmd.py` to wire all 4 tools** - -Replace the tool setup section in `_launch_chat` (around lines 293-303): - -```python -def _launch_chat(store: KnowledgeStore, console: Console) -> None: - """Start an interactive Deep Research chat session.""" - from openjarvis.agents.deep_research import DeepResearchAgent - from openjarvis.connectors.retriever import TwoStageRetriever - from openjarvis.engine.ollama import OllamaEngine - from openjarvis.tools.knowledge_search import KnowledgeSearchTool - from openjarvis.tools.knowledge_sql import KnowledgeSQLTool - from openjarvis.tools.scan_chunks import ScanChunksTool - from openjarvis.tools.think import ThinkTool - - console.print("\n[bold]Setting up Deep Research agent...[/bold]") - - # Engine - engine = OllamaEngine() - if not engine.health(): - console.print( - "[red]Ollama is not running.[/red] Start it with: " - "[bold]ollama serve[/bold]" - ) - return - - models = engine.list_models() - if _OLLAMA_MODEL not in models and f"{_OLLAMA_MODEL}:latest" not in models: - base_name = _OLLAMA_MODEL.split(":")[0] - matching = [m for m in models if m.startswith(base_name)] - if not matching: - console.print( - f"[yellow]Model {_OLLAMA_MODEL} not found.[/yellow] " - f"Pull it with: [bold]ollama pull {_OLLAMA_MODEL}[/bold]" - ) - return - - # Tools - retriever = TwoStageRetriever(store) - tools = [ - KnowledgeSearchTool(retriever=retriever), - KnowledgeSQLTool(store=store), - ScanChunksTool(store=store, engine=engine, model=_OLLAMA_MODEL), - ThinkTool(), - ] - - # Agent - agent = DeepResearchAgent( - engine=engine, - model=_OLLAMA_MODEL, - tools=tools, - interactive=True, - ) - - console.print( - f"[green]Ready![/green] Using [bold]{_OLLAMA_MODEL}[/bold] via Ollama.\n" - "Tools: knowledge_search, knowledge_sql, scan_chunks, think\n" - "Type your research question. Type [bold]/quit[/bold] to exit.\n" - ) - - # REPL - while True: - try: - query = console.input("[bold blue]research>[/bold blue] ").strip() - except (EOFError, KeyboardInterrupt): - break - - if not query: - continue - if query.lower() in ("/quit", "/exit", "quit", "exit"): - break - - try: - result = agent.run(query) - console.print(f"\n{result.content}\n") - if result.metadata and result.metadata.get("sources"): - console.print("[dim]Sources:[/dim]") - for s in result.metadata["sources"]: - console.print(f" [dim]- {s}[/dim]") - console.print() - except Exception as exc: # noqa: BLE001 - console.print(f"[red]Error: {exc}[/red]\n") -``` - -- [ ] **Step 4: Run all tests** - -```bash -uv run pytest tests/agents/test_deep_research.py tests/agents/test_deep_research_integration.py tests/tools/test_knowledge_sql.py tests/tools/test_scan_chunks.py -v --tb=short -``` - -Expected: All PASS. - -- [ ] **Step 5: Lint** - -```bash -uv run ruff check src/openjarvis/agents/deep_research.py src/openjarvis/cli/deep_research_setup_cmd.py src/openjarvis/tools/knowledge_sql.py src/openjarvis/tools/scan_chunks.py -``` - -- [ ] **Step 6: Commit** - -```bash -git add src/openjarvis/agents/deep_research.py src/openjarvis/cli/deep_research_setup_cmd.py -git commit -m "feat: rewrite DeepResearch system prompt, wire SQL + scan + think tools, increase max_turns to 8" -``` - ---- - -### Task 4: Evaluate with 4 test queries - -**Files:** None (manual testing) - -- [ ] **Step 1: Run the 4 test queries** - -```python -import time -from openjarvis.agents.deep_research import DeepResearchAgent -from openjarvis.connectors.retriever import TwoStageRetriever -from openjarvis.connectors.store import KnowledgeStore -from openjarvis.engine.ollama import OllamaEngine -from openjarvis.tools.knowledge_search import KnowledgeSearchTool -from openjarvis.tools.knowledge_sql import KnowledgeSQLTool -from openjarvis.tools.scan_chunks import ScanChunksTool -from openjarvis.tools.think import ThinkTool - -store = KnowledgeStore() -engine = OllamaEngine() -retriever = TwoStageRetriever(store) - -tools = [ - KnowledgeSearchTool(retriever=retriever), - KnowledgeSQLTool(store=store), - ScanChunksTool(store=store, engine=engine, model="qwen3.5:9b"), - ThinkTool(), -] - -agent = DeepResearchAgent(engine=engine, model="qwen3.5:9b", tools=tools, max_turns=8) - -queries = [ - "When was my most recent trip to Spain?", - "Which VCs have I spoken with since 2023?", - "Who are the 10 people I have spoken with the most over text?", - "What meetings take up most of my time based on my calendar and meeting logs?", -] - -for i, query in enumerate(queries, 1): - print(f"QUERY {i}: {query}") - t0 = time.time() - result = agent.run(query) - print(result.content[:3000]) - print(f"[Turns: {result.turns}, Tools: {len(result.tool_results)}, Time: {time.time()-t0:.0f}s]\n") -``` - -- [ ] **Step 2: Compare to v1 results** - -v1 returned empty or "no data found" for all 4 queries. Success = at least 3 of 4 produce substantive answers. - -- [ ] **Step 3: Push** - -```bash -git push origin feat/deep-research-setup -``` diff --git a/docs/superpowers/plans/2026-03-26-deep-research-phase2b-api.md b/docs/superpowers/plans/2026-03-26-deep-research-phase2b-api.md deleted file mode 100644 index da4b7802..00000000 --- a/docs/superpowers/plans/2026-03-26-deep-research-phase2b-api.md +++ /dev/null @@ -1,361 +0,0 @@ -# Deep Research Phase 2B-i: Connector Management API Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add FastAPI endpoints for connector management so the desktop wizard and CLI can list sources, initiate OAuth, check sync status, and trigger syncs — all via HTTP. - -**Architecture:** A new `connectors` API router exposes endpoints for listing available connectors, initiating auth flows, checking connection status, triggering syncs, and streaming sync progress. The router wraps the existing `ConnectorRegistry`, `SyncEngine`, and `KnowledgeStore` from Phase 1. An OAuth callback server handles the browser redirect during auth flows. - -**Tech Stack:** Python 3.10+, FastAPI (existing server infrastructure), pytest + httpx (test client) - -**Spec:** `docs/superpowers/specs/2026-03-25-deep-research-setup-design.md` — Sections 4, 10 - -**Depends on:** Phase 1 (ConnectorRegistry, BaseConnector, SyncEngine, KnowledgeStore) - ---- - -## File Structure - -``` -src/openjarvis/server/ -├── connectors_router.py # FastAPI router: /v1/connectors/* endpoints - -tests/server/ -├── test_connectors_router.py # API endpoint tests -``` - ---- - -### Task 1: Connectors API Router - -**Files:** -- Create: `src/openjarvis/server/connectors_router.py` -- Create: `tests/server/test_connectors_router.py` - -- [ ] **Step 1: Write failing tests** - -Create `tests/server/test_connectors_router.py`: - -```python -"""Tests for the /v1/connectors API endpoints.""" - -from __future__ import annotations - -import json -from pathlib import Path -from unittest.mock import MagicMock, patch - -import pytest - -from openjarvis.server.connectors_router import create_connectors_router - - -@pytest.fixture -def app(): - """Create a minimal FastAPI app with the connectors router.""" - try: - from fastapi import FastAPI - from fastapi.testclient import TestClient - except ImportError: - pytest.skip("fastapi not installed") - - app = FastAPI() - router = create_connectors_router() - app.include_router(router, prefix="/v1") - return TestClient(app) - - -def test_list_connectors(app) -> None: - """GET /v1/connectors returns list of available connectors.""" - resp = app.get("/v1/connectors") - assert resp.status_code == 200 - data = resp.json() - assert isinstance(data, list) - # Should have at least obsidian (always available) - ids = [c["connector_id"] for c in data] - assert "obsidian" in ids - - -def test_connector_detail(app) -> None: - """GET /v1/connectors/{id} returns connector info.""" - resp = app.get("/v1/connectors/obsidian") - assert resp.status_code == 200 - data = resp.json() - assert data["connector_id"] == "obsidian" - assert data["auth_type"] == "filesystem" - assert "connected" in data - - -def test_connector_not_found(app) -> None: - """GET /v1/connectors/{id} returns 404 for unknown.""" - resp = app.get("/v1/connectors/nonexistent") - assert resp.status_code == 404 - - -def test_connect_obsidian(app, tmp_path: Path) -> None: - """POST /v1/connectors/obsidian/connect with path.""" - vault = tmp_path / "vault" - vault.mkdir() - (vault / "note.md").write_text("# Test") - resp = app.post( - "/v1/connectors/obsidian/connect", - json={"path": str(vault)}, - ) - assert resp.status_code == 200 - assert resp.json()["connected"] is True - - -def test_disconnect(app) -> None: - """POST /v1/connectors/obsidian/disconnect.""" - resp = app.post("/v1/connectors/obsidian/disconnect") - assert resp.status_code == 200 - - -def test_sync_status(app) -> None: - """GET /v1/connectors/obsidian/sync returns sync status.""" - resp = app.get("/v1/connectors/obsidian/sync") - assert resp.status_code == 200 - data = resp.json() - assert "state" in data -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/server/test_connectors_router.py -v` - -Expected: FAIL — `ModuleNotFoundError` - -- [ ] **Step 3: Implement connectors router** - -Create `src/openjarvis/server/connectors_router.py`: - -```python -"""FastAPI router for connector management: /v1/connectors/* - -Exposes endpoints for listing connectors, connecting/disconnecting, -checking status, and triggering syncs. Used by the desktop wizard -and CLI. -""" - -from __future__ import annotations - -import logging -from typing import Any, Dict, List, Optional - -from openjarvis.connectors._stubs import SyncStatus -from openjarvis.core.registry import ConnectorRegistry - -logger = logging.getLogger(__name__) - -# Connector instances cache (keyed by connector_id) -_instances: Dict[str, Any] = {} - - -def _get_or_create(connector_id: str, **kwargs: Any) -> Any: - """Get a cached connector instance or create one.""" - if connector_id not in _instances: - cls = ConnectorRegistry.get(connector_id) - _instances[connector_id] = cls(**kwargs) - return _instances[connector_id] - - -def create_connectors_router(): - """Factory that returns a FastAPI APIRouter for /v1/connectors. - - Separated from module-level to allow lazy import of FastAPI. - """ - from fastapi import APIRouter, HTTPException - from pydantic import BaseModel - - router = APIRouter(tags=["connectors"]) - - class ConnectRequest(BaseModel): - path: Optional[str] = None - token: Optional[str] = None - code: Optional[str] = None - email: Optional[str] = None - password: Optional[str] = None - - class ConnectorInfo(BaseModel): - connector_id: str - display_name: str - auth_type: str - connected: bool - - # Import connectors to trigger registration - import openjarvis.connectors # noqa: F401 - - @router.get("/connectors") - def list_connectors() -> List[Dict[str, Any]]: - """List all available data source connectors.""" - results = [] - for key in sorted(ConnectorRegistry.keys()): - cls = ConnectorRegistry.get(key) - try: - inst = _get_or_create(key) - connected = inst.is_connected() - except Exception: - connected = False - results.append( - { - "connector_id": key, - "display_name": getattr( - cls, "display_name", key - ), - "auth_type": getattr(cls, "auth_type", "unknown"), - "connected": connected, - } - ) - return results - - @router.get("/connectors/{connector_id}") - def get_connector(connector_id: str) -> Dict[str, Any]: - """Get details for a specific connector.""" - if not ConnectorRegistry.contains(connector_id): - raise HTTPException( - status_code=404, - detail=f"Connector '{connector_id}' not found", - ) - cls = ConnectorRegistry.get(connector_id) - try: - inst = _get_or_create(connector_id) - connected = inst.is_connected() - except Exception: - connected = False - - auth_url = "" - try: - inst = _get_or_create(connector_id) - auth_url = inst.auth_url() - except NotImplementedError: - pass - - mcp_tools = [] - try: - inst = _get_or_create(connector_id) - mcp_tools = [t.name for t in inst.mcp_tools()] - except Exception: - pass - - return { - "connector_id": connector_id, - "display_name": getattr(cls, "display_name", connector_id), - "auth_type": getattr(cls, "auth_type", "unknown"), - "connected": connected, - "auth_url": auth_url, - "mcp_tools": mcp_tools, - } - - @router.post("/connectors/{connector_id}/connect") - def connect_source( - connector_id: str, req: ConnectRequest - ) -> Dict[str, Any]: - """Connect a data source.""" - if not ConnectorRegistry.contains(connector_id): - raise HTTPException( - status_code=404, - detail=f"Connector '{connector_id}' not found", - ) - cls = ConnectorRegistry.get(connector_id) - auth_type = getattr(cls, "auth_type", "") - - if auth_type == "filesystem" and req.path: - inst = cls(vault_path=req.path) - _instances[connector_id] = inst - return {"connected": inst.is_connected()} - - if req.token: - inst = cls(token=req.token) - _instances[connector_id] = inst - return {"connected": inst.is_connected()} - - if req.code: - inst = _get_or_create(connector_id) - inst.handle_callback(req.code) - return {"connected": inst.is_connected()} - - if req.email and req.password: - inst = cls( - email_address=req.email, - app_password=req.password, - ) - _instances[connector_id] = inst - return {"connected": inst.is_connected()} - - raise HTTPException( - status_code=400, - detail="Provide path, token, code, or email+password", - ) - - @router.post("/connectors/{connector_id}/disconnect") - def disconnect_source(connector_id: str) -> Dict[str, Any]: - """Disconnect a data source.""" - if not ConnectorRegistry.contains(connector_id): - raise HTTPException( - status_code=404, - detail=f"Connector '{connector_id}' not found", - ) - inst = _get_or_create(connector_id) - inst.disconnect() - return {"disconnected": True} - - @router.get("/connectors/{connector_id}/sync") - def get_sync_status(connector_id: str) -> Dict[str, Any]: - """Get sync status for a connector.""" - if not ConnectorRegistry.contains(connector_id): - raise HTTPException( - status_code=404, - detail=f"Connector '{connector_id}' not found", - ) - inst = _get_or_create(connector_id) - status = inst.sync_status() - return { - "state": status.state, - "items_synced": status.items_synced, - "items_total": status.items_total, - "last_sync": ( - status.last_sync.isoformat() if status.last_sync else None - ), - "error": status.error, - } - - return router -``` - -- [ ] **Step 4: Run tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/server/test_connectors_router.py -v` - -Expected: All 6 tests PASS. - -- [ ] **Step 5: Run linter** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run ruff check src/openjarvis/server/connectors_router.py` - -Expected: No errors. - -- [ ] **Step 6: Commit** - -```bash -git add src/openjarvis/server/connectors_router.py tests/server/test_connectors_router.py -git commit -m "feat: add /v1/connectors API router for connector management" -``` - ---- - -## Post-Plan Notes - -**What this plan produces:** -- `GET /v1/connectors` — list all available connectors with connection status -- `GET /v1/connectors/{id}` — connector detail with auth_url and MCP tools -- `POST /v1/connectors/{id}/connect` — connect with path, token, code, or email+password -- `POST /v1/connectors/{id}/disconnect` — disconnect a source -- `GET /v1/connectors/{id}/sync` — sync status (state, items_synced, items_total, error) - -**Phase 2B-ii (Frontend wizard)** will consume these endpoints to build: -- Source picker grid (calls `GET /v1/connectors`) -- Per-source OAuth flow (calls `GET /v1/connectors/{id}` for auth_url, then `POST .../connect`) -- Ingest progress dashboard (polls `GET /v1/connectors/{id}/sync`) -- New Tauri commands wrapping these endpoints - -**Phase 2B-ii is a TypeScript/React effort** — separate plan needed for the frontend components, Tauri commands, and wizard UI. diff --git a/docs/superpowers/plans/2026-03-26-deep-research-phase2b-frontend.md b/docs/superpowers/plans/2026-03-26-deep-research-phase2b-frontend.md deleted file mode 100644 index 310953f5..00000000 --- a/docs/superpowers/plans/2026-03-26-deep-research-phase2b-frontend.md +++ /dev/null @@ -1,774 +0,0 @@ -# Deep Research Phase 2B-ii: Desktop Setup Wizard UI Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Build the "Apple-like magic" setup wizard in the Tauri desktop app — a beautiful multi-step onboarding flow that guides users through connecting their data sources, with product logos, progress bars, and a premium feel. - -**Architecture:** Extends the existing `SetupScreen.tsx` in `frontend/src/components/` with new wizard steps after the engine boot completes. Uses the `/v1/connectors/*` API endpoints (Phase 2B-i) for backend communication. New components: SourcePicker (grid of connector cards), SourceConnectWizard (per-source auth flow), IngestDashboard (live sync progress), and ReadyScreen (first query suggestions). Follows existing patterns: Tailwind + shadcn/ui components, Zustand for state, React Router for navigation. - -**Tech Stack:** TypeScript, React 19, Tailwind CSS 4.2, shadcn/ui, Zustand 5, Vite 6, Tauri 2 - -**Spec:** `docs/superpowers/specs/2026-03-25-deep-research-setup-design.md` — Section 4 (Setup Wizard) - -**Depends on:** Phase 2B-i (connector API endpoints) - ---- - -## File Structure - -``` -frontend/src/ -├── components/ -│ ├── setup/ -│ │ ├── SetupWizard.tsx # Main wizard orchestrator (steps 3-6) -│ │ ├── SourcePicker.tsx # Step 3: grid of source cards -│ │ ├── SourceConnectFlow.tsx # Step 4: guided per-source auth -│ │ ├── IngestDashboard.tsx # Step 5: live sync progress -│ │ └── ReadyScreen.tsx # Step 6: first query suggestions -│ ├── SetupScreen.tsx # (modify) Add wizard after boot -├── lib/ -│ ├── connectors-api.ts # API client for /v1/connectors/* -│ └── store.ts # (modify) Add connector state slice -├── types/ -│ └── connectors.ts # TypeScript interfaces -``` - ---- - -### Task 1: TypeScript Types + API Client - -**Files:** -- Create: `frontend/src/types/connectors.ts` -- Create: `frontend/src/lib/connectors-api.ts` - -- [ ] **Step 1: Create connector types** - -Create `frontend/src/types/connectors.ts`: - -```typescript -export interface ConnectorInfo { - connector_id: string; - display_name: string; - auth_type: "oauth" | "local" | "bridge" | "filesystem"; - connected: boolean; - auth_url?: string; - mcp_tools?: string[]; -} - -export interface SyncStatus { - state: "idle" | "syncing" | "paused" | "error"; - items_synced: number; - items_total: number; - last_sync: string | null; - error: string | null; -} - -export interface ConnectRequest { - path?: string; - token?: string; - code?: string; - email?: string; - password?: string; -} - -export type WizardStep = "pick" | "connect" | "ingest" | "ready"; - -// Source card metadata for the picker UI -export interface SourceCard { - connector_id: string; - display_name: string; - auth_type: string; - category: "communication" | "documents" | "pim"; - icon: string; // Lucide icon name - color: string; // Tailwind color class - description: string; -} - -export const SOURCE_CATALOG: SourceCard[] = [ - // Communication - { connector_id: "gmail", display_name: "Gmail", auth_type: "oauth", category: "communication", icon: "Mail", color: "text-red-400", description: "Email messages and threads" }, - { connector_id: "gmail_imap", display_name: "Gmail (IMAP)", auth_type: "oauth", category: "communication", icon: "Mail", color: "text-red-400", description: "Email via app password" }, - { connector_id: "slack", display_name: "Slack", auth_type: "oauth", category: "communication", icon: "Hash", color: "text-purple-400", description: "Channel messages and threads" }, - { connector_id: "imessage", display_name: "iMessage", auth_type: "local", category: "communication", icon: "MessageSquare", color: "text-green-400", description: "macOS Messages history" }, - // Documents - { connector_id: "gdrive", display_name: "Google Drive", auth_type: "oauth", category: "documents", icon: "FolderOpen", color: "text-blue-400", description: "Docs, Sheets, and files" }, - { connector_id: "notion", display_name: "Notion", auth_type: "oauth", category: "documents", icon: "FileText", color: "text-gray-300", description: "Pages and databases" }, - { connector_id: "obsidian", display_name: "Obsidian", auth_type: "filesystem", category: "documents", icon: "Diamond", color: "text-violet-400", description: "Markdown vault" }, - { connector_id: "granola", display_name: "Granola", auth_type: "oauth", category: "documents", icon: "Mic", color: "text-amber-400", description: "AI meeting notes" }, - // PIM - { connector_id: "gcalendar", display_name: "Calendar", auth_type: "oauth", category: "pim", icon: "Calendar", color: "text-blue-400", description: "Events and meetings" }, - { connector_id: "gcontacts", display_name: "Contacts", auth_type: "oauth", category: "pim", icon: "Users", color: "text-blue-400", description: "People and contact info" }, -]; -``` - -- [ ] **Step 2: Create API client** - -Create `frontend/src/lib/connectors-api.ts`: - -```typescript -import { getBase } from "./api"; -import type { ConnectorInfo, ConnectRequest, SyncStatus } from "../types/connectors"; - -export async function listConnectors(): Promise { - const resp = await fetch(`${getBase()}/v1/connectors`); - if (!resp.ok) throw new Error(`Failed to list connectors: ${resp.status}`); - return resp.json(); -} - -export async function getConnector(id: string): Promise { - const resp = await fetch(`${getBase()}/v1/connectors/${id}`); - if (!resp.ok) throw new Error(`Connector not found: ${id}`); - return resp.json(); -} - -export async function connectSource(id: string, req: ConnectRequest): Promise<{ connected: boolean }> { - const resp = await fetch(`${getBase()}/v1/connectors/${id}/connect`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(req), - }); - if (!resp.ok) throw new Error(`Failed to connect: ${resp.status}`); - return resp.json(); -} - -export async function disconnectSource(id: string): Promise { - await fetch(`${getBase()}/v1/connectors/${id}/disconnect`, { method: "POST" }); -} - -export async function getSyncStatus(id: string): Promise { - const resp = await fetch(`${getBase()}/v1/connectors/${id}/sync`); - if (!resp.ok) throw new Error(`Failed to get sync status: ${id}`); - return resp.json(); -} -``` - -- [ ] **Step 3: Commit** - -```bash -cd frontend && git add src/types/connectors.ts src/lib/connectors-api.ts -git commit -m "feat: add connector TypeScript types and API client" -``` - ---- - -### Task 2: SourcePicker Component (Step 3) - -**Files:** -- Create: `frontend/src/components/setup/SourcePicker.tsx` - -- [ ] **Step 1: Implement SourcePicker** - -Create `frontend/src/components/setup/SourcePicker.tsx`: - -```tsx -import { useState } from "react"; -import { - Mail, Hash, MessageSquare, FolderOpen, FileText, - Diamond, Mic, Calendar, Users, Check, -} from "lucide-react"; -import { SOURCE_CATALOG, type SourceCard } from "../../types/connectors"; - -const ICON_MAP: Record = { - Mail, Hash, MessageSquare, FolderOpen, FileText, - Diamond, Mic, Calendar, Users, -}; - -const CATEGORIES = [ - { key: "communication", label: "Communication" }, - { key: "documents", label: "Documents" }, - { key: "pim", label: "Personal Info" }, -] as const; - -interface Props { - onContinue: (selectedIds: string[]) => void; -} - -export function SourcePicker({ onContinue }: Props) { - const [selected, setSelected] = useState>(new Set()); - - const toggle = (id: string) => { - setSelected((prev) => { - const next = new Set(prev); - if (next.has(id)) next.delete(id); - else next.add(id); - return next; - }); - }; - - return ( -
-
-

- Connect Your Data -

-

- Choose the sources you want to search across. You can always add more later. -

-
- - {CATEGORIES.map(({ key, label }) => ( -
-

- {label} -

-
- {SOURCE_CATALOG.filter((s) => s.category === key).map((source) => { - const Icon = ICON_MAP[source.icon] ?? FileText; - const isSelected = selected.has(source.connector_id); - return ( - - ); - })} -
-
- ))} - - -
- ); -} -``` - -- [ ] **Step 2: Commit** - -```bash -cd frontend && git add src/components/setup/SourcePicker.tsx -git commit -m "feat: add SourcePicker component with category grid and toggle selection" -``` - ---- - -### Task 3: SourceConnectFlow Component (Step 4) - -**Files:** -- Create: `frontend/src/components/setup/SourceConnectFlow.tsx` - -- [ ] **Step 1: Implement SourceConnectFlow** - -Create `frontend/src/components/setup/SourceConnectFlow.tsx`: - -```tsx -import { useState } from "react"; -import { Check, ExternalLink, Loader2, X, FolderOpen } from "lucide-react"; -import { SOURCE_CATALOG } from "../../types/connectors"; -import { connectSource, getConnector } from "../../lib/connectors-api"; - -interface Props { - selectedIds: string[]; - onComplete: () => void; -} - -type SourceState = "pending" | "connecting" | "connected" | "skipped" | "error"; - -export function SourceConnectFlow({ selectedIds, onComplete }: Props) { - const [currentIndex, setCurrentIndex] = useState(0); - const [states, setStates] = useState>( - Object.fromEntries(selectedIds.map((id) => [id, "pending"])) - ); - const [error, setError] = useState(""); - const [inputValue, setInputValue] = useState(""); - - const currentId = selectedIds[currentIndex]; - const source = SOURCE_CATALOG.find((s) => s.connector_id === currentId); - const isLast = currentIndex >= selectedIds.length - 1; - const allDone = currentIndex >= selectedIds.length; - - const advance = () => { - if (isLast) { - onComplete(); - } else { - setCurrentIndex((i) => i + 1); - setInputValue(""); - setError(""); - } - }; - - const skip = () => { - setStates((s) => ({ ...s, [currentId]: "skipped" })); - advance(); - }; - - const handleConnect = async () => { - if (!currentId || !source) return; - setStates((s) => ({ ...s, [currentId]: "connecting" })); - setError(""); - - try { - if (source.auth_type === "filesystem") { - const result = await connectSource(currentId, { path: inputValue }); - if (result.connected) { - setStates((s) => ({ ...s, [currentId]: "connected" })); - setTimeout(advance, 800); - } else { - throw new Error("Could not access that path"); - } - } else if (source.auth_type === "oauth") { - // Get auth URL and open in browser - const info = await getConnector(currentId); - if (info.auth_url) { - window.open(info.auth_url, "_blank"); - } - // For token/code entry - if (inputValue) { - const result = await connectSource(currentId, { token: inputValue }); - if (result.connected) { - setStates((s) => ({ ...s, [currentId]: "connected" })); - setTimeout(advance, 800); - } - } - } else if (source.auth_type === "local") { - // Local access (iMessage, Apple Notes) — just check - const result = await connectSource(currentId, {}); - if (result.connected) { - setStates((s) => ({ ...s, [currentId]: "connected" })); - setTimeout(advance, 800); - } else { - throw new Error("Permission not granted. Check Full Disk Access in System Preferences."); - } - } - } catch (err: any) { - setError(err.message || "Connection failed"); - setStates((s) => ({ ...s, [currentId]: "error" })); - } - }; - - if (allDone) { - onComplete(); - return null; - } - - return ( -
- {/* Sidebar checklist */} -
-

Progress

- {selectedIds.map((id, i) => { - const s = SOURCE_CATALOG.find((x) => x.connector_id === id); - const state = states[id]; - return ( -
- {state === "connected" && } - {state === "skipped" && } - {state === "connecting" && } - {(state === "pending" || state === "error") &&
} - {s?.display_name ?? id} -
- ); - })} -
- - {/* Main content */} -
-

- Connect {source?.display_name} -

- - {source?.auth_type === "filesystem" && ( -
-

Enter the path to your vault or folder:

-
- setInputValue(e.target.value)} - placeholder="/path/to/vault" - className="flex-1 px-4 py-2 rounded-lg bg-white/5 border border-white/10 text-sm" - /> - -
-
- )} - - {source?.auth_type === "oauth" && ( -
-

- We'll open {source.display_name} in your browser for authorization. After authorizing, paste the token or code below. -

- - setInputValue(e.target.value)} - placeholder="Paste token or code here" - className="w-full px-4 py-2 rounded-lg bg-white/5 border border-white/10 text-sm" - onKeyDown={(e) => e.key === "Enter" && handleConnect()} - /> -
- )} - - {source?.auth_type === "local" && ( -
-

- This requires Full Disk Access on macOS. Grant access in System Preferences → Privacy & Security → Full Disk Access. -

- -
- )} - - {error && ( -

{error}

- )} - - -
-
- ); -} -``` - -- [ ] **Step 2: Commit** - -```bash -cd frontend && git add src/components/setup/SourceConnectFlow.tsx -git commit -m "feat: add SourceConnectFlow with per-source guided auth wizard" -``` - ---- - -### Task 4: IngestDashboard + ReadyScreen (Steps 5-6) - -**Files:** -- Create: `frontend/src/components/setup/IngestDashboard.tsx` -- Create: `frontend/src/components/setup/ReadyScreen.tsx` - -- [ ] **Step 1: Implement IngestDashboard** - -Create `frontend/src/components/setup/IngestDashboard.tsx`: - -```tsx -import { useEffect, useState } from "react"; -import { Loader2, Check } from "lucide-react"; -import { getSyncStatus } from "../../lib/connectors-api"; -import { SOURCE_CATALOG, type SyncStatus } from "../../types/connectors"; - -interface Props { - connectedIds: string[]; - onReady: () => void; -} - -export function IngestDashboard({ connectedIds, onReady }: Props) { - const [statuses, setStatuses] = useState>({}); - - useEffect(() => { - if (connectedIds.length === 0) { - onReady(); - return; - } - - const poll = setInterval(async () => { - const updates: Record = {}; - for (const id of connectedIds) { - try { - updates[id] = await getSyncStatus(id); - } catch { - updates[id] = { state: "error", items_synced: 0, items_total: 0, last_sync: null, error: "Failed to fetch status" }; - } - } - setStatuses(updates); - - // Check if all done - const allIdle = Object.values(updates).every((s) => s.state === "idle" && s.items_synced > 0); - if (allIdle) { - clearInterval(poll); - } - }, 2000); - - return () => clearInterval(poll); - }, [connectedIds]); - - return ( -
-
-

- Indexing Your Data -

-

- Your data is being synced and indexed. You can start asking questions now. -

-
- -
- {connectedIds.map((id) => { - const source = SOURCE_CATALOG.find((s) => s.connector_id === id); - const status = statuses[id]; - const synced = status?.items_synced ?? 0; - const total = status?.items_total ?? 0; - const pct = total > 0 ? Math.round((synced / total) * 100) : 0; - const isDone = status?.state === "idle" && synced > 0; - - return ( -
-
- {isDone ? ( - - ) : ( - - )} -
-
-
- {source?.display_name ?? id} - - {isDone ? `${synced.toLocaleString()} items` : total > 0 ? `${synced.toLocaleString()} / ${total.toLocaleString()}` : "Starting..."} - -
-
-
-
-
-
- ); - })} -
- - -
- ); -} -``` - -- [ ] **Step 2: Implement ReadyScreen** - -Create `frontend/src/components/setup/ReadyScreen.tsx`: - -```tsx -import { Sparkles } from "lucide-react"; - -interface Props { - connectedSources: string[]; - onStart: (query?: string) => void; -} - -const SUGGESTIONS = [ - "What were the key decisions from last week's team threads?", - "Find the proposal doc shared about the roadmap", - "Summarize my unread emails from today", - "What meetings do I have this week?", - "What topics came up in recent meetings?", -]; - -export function ReadyScreen({ connectedSources, onStart }: Props) { - return ( -
-
- -
- -
-

- You're All Set! -

-

- {connectedSources.length} source{connectedSources.length !== 1 ? "s" : ""} connected and indexed. Ask anything across your data. -

-
- -
-

Try asking...

- {SUGGESTIONS.slice(0, 3).map((q) => ( - - ))} -
- - -
- ); -} -``` - -- [ ] **Step 3: Commit** - -```bash -cd frontend && git add src/components/setup/IngestDashboard.tsx src/components/setup/ReadyScreen.tsx -git commit -m "feat: add IngestDashboard with progress bars and ReadyScreen with suggestions" -``` - ---- - -### Task 5: SetupWizard Orchestrator + Integration - -**Files:** -- Create: `frontend/src/components/setup/SetupWizard.tsx` -- Modify: `frontend/src/components/SetupScreen.tsx` - -- [ ] **Step 1: Create SetupWizard orchestrator** - -Create `frontend/src/components/setup/SetupWizard.tsx`: - -```tsx -import { useState } from "react"; -import type { WizardStep } from "../../types/connectors"; -import { SourcePicker } from "./SourcePicker"; -import { SourceConnectFlow } from "./SourceConnectFlow"; -import { IngestDashboard } from "./IngestDashboard"; -import { ReadyScreen } from "./ReadyScreen"; - -interface Props { - onComplete: (firstQuery?: string) => void; -} - -export function SetupWizard({ onComplete }: Props) { - const [step, setStep] = useState("pick"); - const [selectedIds, setSelectedIds] = useState([]); - const [connectedIds, setConnectedIds] = useState([]); - - switch (step) { - case "pick": - return ( - { - setSelectedIds(ids); - if (ids.length > 0) { - setStep("connect"); - } else { - onComplete(); - } - }} - /> - ); - - case "connect": - return ( - { - // For now, assume all selected are connected - setConnectedIds(selectedIds); - setStep("ingest"); - }} - /> - ); - - case "ingest": - return ( - setStep("ready")} - /> - ); - - case "ready": - return ( - onComplete(query)} - /> - ); - } -} -``` - -- [ ] **Step 2: Integrate into SetupScreen** - -Modify `frontend/src/components/SetupScreen.tsx` to show the wizard after the boot sequence completes. Find the section where `status.phase === 'ready'` triggers `onReady()`. Instead of calling `onReady()` immediately, show the `SetupWizard`: - -Add to the imports: -```tsx -import { SetupWizard } from "./setup/SetupWizard"; -``` - -Add state to track whether wizard is shown: -```tsx -const [showWizard, setShowWizard] = useState(false); -``` - -When boot completes (phase === 'ready'), instead of calling `onReady()`, set `showWizard = true`. Then render: -```tsx -if (showWizard) { - return { - // Optionally pass first query to chat - onReady(); - }} />; -} -``` - -- [ ] **Step 3: Commit** - -```bash -cd frontend && git add src/components/setup/SetupWizard.tsx src/components/SetupScreen.tsx -git commit -m "feat: integrate SetupWizard into boot flow after engine setup" -``` - ---- - -## Post-Plan Notes - -**What this plan produces:** -- TypeScript types and API client for connector management -- SourcePicker: beautiful grid of 10 source cards with category grouping and toggle selection -- SourceConnectFlow: guided per-source auth wizard with sidebar progress checklist -- IngestDashboard: live progress bars polling sync status -- ReadyScreen: celebration screen with suggested first queries -- SetupWizard: orchestrates all 4 steps -- Integration into existing SetupScreen boot flow - -**What this does NOT include (separate effort):** -- Actual OAuth redirect handling in Tauri (needs Tauri deep link plugin) -- File picker dialog for Obsidian (needs Tauri dialog plugin) -- QR code display for WhatsApp -- Product logo images (would need asset files) -- Tauri command wrappers for the API endpoints -- Automated sync triggering after connect diff --git a/docs/superpowers/plans/2026-03-26-deep-research-phase4-channel-plugins.md b/docs/superpowers/plans/2026-03-26-deep-research-phase4-channel-plugins.md deleted file mode 100644 index d72c95d9..00000000 --- a/docs/superpowers/plans/2026-03-26-deep-research-phase4-channel-plugins.md +++ /dev/null @@ -1,816 +0,0 @@ -# Deep Research Phase 4: Channel Plugins Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Build a ChannelAgent that connects existing messaging channels (Slack, WhatsApp, iMessage) to the DeepResearchAgent, with automatic query classification (quick inline answers vs. escalation to desktop for complex research). - -**Architecture:** A `ChannelAgent` class registers as a `ChannelHandler` callback on any `BaseChannel`. When a message arrives, it classifies the query (quick vs. deep), runs the DeepResearchAgent with the appropriate turn limit, and sends the response back via `channel.send()`. Quick answers go inline; complex queries get a brief preview + escalation link. A worker thread pool prevents blocking the channel's event loop. - -**Tech Stack:** Python 3.10+, threading (worker pool), existing channel transports (Slack Socket Mode, WhatsApp Baileys, BlueBubbles), pytest - -**Spec:** `docs/superpowers/specs/2026-03-25-deep-research-setup-design.md` — Section 9 (Channel Plugins) - -**Depends on:** Phase 1 (connectors, KnowledgeStore), Phase 3 (DeepResearchAgent, TwoStageRetriever) - ---- - -## File Structure - -``` -src/openjarvis/agents/ -├── channel_agent.py # ChannelAgent: bridge between channels and DeepResearchAgent - -tests/agents/ -├── test_channel_agent.py # ChannelAgent tests -``` - ---- - -### Task 1: Query Classifier - -**Files:** -- Create: `src/openjarvis/agents/channel_agent.py` (first half — classifier only) -- Create: `tests/agents/test_channel_agent.py` - -The classifier determines whether a message needs a quick single-hop answer or full multi-hop research. This is an internal implementation detail — the user never sees it. - -- [ ] **Step 1: Write failing tests for classifier** - -Create `tests/agents/test_channel_agent.py`: - -```python -"""Tests for ChannelAgent — bridge between channels and DeepResearchAgent.""" - -from __future__ import annotations - -from openjarvis.agents.channel_agent import classify_query - - -def test_quick_when_query() -> None: - assert classify_query("When is my next meeting?") == "quick" - - -def test_quick_where_query() -> None: - assert classify_query("Where is the design doc?") == "quick" - - -def test_quick_find_query() -> None: - assert classify_query("Find the budget spreadsheet") == "quick" - - -def test_quick_short_query() -> None: - assert classify_query("Who sent the last email?") == "quick" - - -def test_deep_summarize() -> None: - assert classify_query("Summarize all discussions about pricing") == "deep" - - -def test_deep_research() -> None: - assert classify_query("Research the context behind the K8s decision") == "deep" - - -def test_deep_context() -> None: - assert classify_query("What was the context around the migration?") == "deep" - - -def test_deep_time_range() -> None: - assert classify_query("What happened last month with the project?") == "deep" - - -def test_deep_long_query() -> None: - long = "Tell me about all the discussions between Sarah and Mike about the infrastructure migration including cost analysis and timeline" - assert classify_query(long) == "deep" - - -def test_quick_single_entity() -> None: - assert classify_query("What's Sarah's email?") == "quick" - - -def test_deep_multi_entity() -> None: - assert classify_query("Compare what Sarah and Mike said about the budget") == "deep" -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/agents/test_channel_agent.py -v` - -Expected: FAIL — `ModuleNotFoundError` - -- [ ] **Step 3: Implement classify_query** - -Create `src/openjarvis/agents/channel_agent.py`: - -```python -"""ChannelAgent — bridges messaging channels to the DeepResearchAgent. - -Handles: -- Automatic query classification (quick vs. deep) -- Quick answers inline in the chat channel -- Complex queries escalated with a preview + desktop link -- Worker thread pool so channel event loops aren't blocked -""" - -from __future__ import annotations - -import logging -import re -import threading -import uuid -from concurrent.futures import ThreadPoolExecutor -from typing import Any, Callable, Dict, List, Optional - -from openjarvis.agents._stubs import AgentContext, AgentResult -from openjarvis.channels._stubs import BaseChannel, ChannelMessage -from openjarvis.core.events import EventType, get_event_bus - -logger = logging.getLogger(__name__) - -# --------------------------------------------------------------- -# Query classification -# --------------------------------------------------------------- - -_DEEP_KEYWORDS = re.compile( - r"\b(summarize|summary|research|context|compare|analyze|analysis" - r"|overview|all discussions|comprehensive|timeline|history" - r"|what happened|tell me about all)\b", - re.IGNORECASE, -) - -_TIME_RANGE_PATTERNS = re.compile( - r"\b(last (week|month|quarter|year)|past \d+|since (january|february" - r"|march|april|may|june|july|august|september|october|november" - r"|december))\b", - re.IGNORECASE, -) - -_QUICK_STARTERS = re.compile( - r"^(when|where|find|who sent|what'?s|what is)\b", re.IGNORECASE -) - -_MAX_QUICK_WORDS = 20 - - -def classify_query(text: str) -> str: - """Classify a user query as 'quick' or 'deep'. - - Quick: single-entity lookups, short factual questions. - Deep: multi-source synthesis, summarization, time-range analysis. - - This is an internal heuristic — the user never specifies the mode. - """ - text = text.strip() - words = text.split() - - # Deep signals take priority - if _DEEP_KEYWORDS.search(text): - return "deep" - if _TIME_RANGE_PATTERNS.search(text): - return "deep" - if len(words) > _MAX_QUICK_WORDS: - return "deep" - - # Quick signals - if _QUICK_STARTERS.match(text): - return "quick" - if len(words) <= _MAX_QUICK_WORDS: - return "quick" - - return "quick" -``` - -- [ ] **Step 4: Run tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/agents/test_channel_agent.py -v` - -Expected: All 11 tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/openjarvis/agents/channel_agent.py tests/agents/test_channel_agent.py -git commit -m "feat: add query classifier for channel agent (quick vs deep)" -``` - ---- - -### Task 2: ChannelAgent Core - -**Files:** -- Modify: `src/openjarvis/agents/channel_agent.py` -- Modify: `tests/agents/test_channel_agent.py` - -- [ ] **Step 1: Write failing tests for ChannelAgent** - -Add to `tests/agents/test_channel_agent.py`: - -```python -import json -import threading -import time -from pathlib import Path -from unittest.mock import MagicMock, patch - -import pytest - -from openjarvis.agents.channel_agent import ChannelAgent, classify_query -from openjarvis.agents.deep_research import DeepResearchAgent -from openjarvis.agents._stubs import AgentResult -from openjarvis.channels._stubs import BaseChannel, ChannelMessage, ChannelStatus -from openjarvis.connectors.store import KnowledgeStore -from openjarvis.connectors.retriever import TwoStageRetriever -from openjarvis.core.types import ToolResult -from openjarvis.tools.knowledge_search import KnowledgeSearchTool - - -class FakeChannel(BaseChannel): - """Minimal channel for testing.""" - - channel_id = "fake" - - def __init__(self) -> None: - self._handlers: list = [] - self._sent: list = [] - self._connected = True - - def connect(self) -> None: - self._connected = True - - def disconnect(self) -> None: - self._connected = False - - def send( - self, - channel: str, - content: str, - *, - conversation_id: str = "", - metadata: dict | None = None, - ) -> bool: - self._sent.append( - { - "channel": channel, - "content": content, - "conversation_id": conversation_id, - } - ) - return True - - def status(self) -> ChannelStatus: - return ChannelStatus.CONNECTED if self._connected else ChannelStatus.DISCONNECTED - - def list_channels(self) -> list[str]: - return ["test"] - - def on_message(self, handler) -> None: - self._handlers.append(handler) - - def simulate_message(self, text: str, sender: str = "user1") -> None: - msg = ChannelMessage( - channel="fake", - sender=sender, - content=text, - conversation_id="conv1", - ) - for h in self._handlers: - h(msg) - - -@pytest.fixture -def mock_agent() -> MagicMock: - agent = MagicMock(spec=DeepResearchAgent) - agent.agent_id = "deep_research" - agent.run.return_value = AgentResult( - content="The next meeting is tomorrow at 2pm.", - tool_results=[], - turns=1, - ) - return agent - - -@pytest.fixture -def channel() -> FakeChannel: - return FakeChannel() - - -@pytest.fixture -def channel_agent( - channel: FakeChannel, mock_agent: MagicMock -) -> ChannelAgent: - return ChannelAgent( - channel=channel, - agent=mock_agent, - ) - - -def test_channel_agent_registers_handler( - channel: FakeChannel, channel_agent: ChannelAgent -) -> None: - assert len(channel._handlers) == 1 - - -def test_quick_query_sends_response_inline( - channel: FakeChannel, - channel_agent: ChannelAgent, - mock_agent: MagicMock, -) -> None: - channel.simulate_message("When is my next meeting?") - # Give worker thread time to process - time.sleep(0.5) - assert len(channel._sent) >= 1 - assert "meeting" in channel._sent[0]["content"].lower() - mock_agent.run.assert_called_once() - - -def test_deep_query_sends_preview_and_link( - channel: FakeChannel, - channel_agent: ChannelAgent, - mock_agent: MagicMock, -) -> None: - mock_agent.run.return_value = AgentResult( - content="# Kubernetes Migration Report\n\nDetailed analysis...", - tool_results=[ - ToolResult( - tool_name="knowledge_search", - content="results", - success=True, - metadata={"num_results": 15}, - ), - ], - turns=3, - ) - channel.simulate_message( - "Summarize all discussions about the Kubernetes migration" - ) - time.sleep(0.5) - assert len(channel._sent) >= 1 - # Deep queries should include an escalation link - sent_content = channel._sent[-1]["content"] - assert "openjarvis://" in sent_content or len(sent_content) > 50 - - -def test_agent_error_sends_error_message( - channel: FakeChannel, - channel_agent: ChannelAgent, - mock_agent: MagicMock, -) -> None: - mock_agent.run.side_effect = RuntimeError("LLM unavailable") - channel.simulate_message("When is my meeting?") - time.sleep(0.5) - assert len(channel._sent) >= 1 - assert "error" in channel._sent[0]["content"].lower() or "sorry" in channel._sent[0]["content"].lower() - - -def test_quick_query_uses_max_turns_1( - channel: FakeChannel, - channel_agent: ChannelAgent, - mock_agent: MagicMock, -) -> None: - channel.simulate_message("Find the budget doc") - time.sleep(0.5) - call_kwargs = mock_agent.run.call_args - # Quick queries should be called (the agent handles turn limits internally) - mock_agent.run.assert_called_once() - - -def test_channel_agent_does_not_block_handler( - channel: FakeChannel, - channel_agent: ChannelAgent, - mock_agent: MagicMock, -) -> None: - """Handler should return immediately (work dispatched to thread pool).""" - mock_agent.run.side_effect = lambda *a, **kw: ( - time.sleep(2), - AgentResult(content="done", turns=1), - )[-1] - - t0 = time.time() - channel.simulate_message("When is my meeting?") - elapsed = time.time() - t0 - # Handler should return in < 0.1s even if agent takes 2s - assert elapsed < 0.5 -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/agents/test_channel_agent.py::test_channel_agent_registers_handler -v` - -Expected: FAIL — `ImportError: cannot import name 'ChannelAgent'` - -- [ ] **Step 3: Implement ChannelAgent** - -Add to `src/openjarvis/agents/channel_agent.py` (after the classifier): - -```python -# --------------------------------------------------------------- -# ChannelAgent -# --------------------------------------------------------------- - -_ESCALATION_TEMPLATE = """{preview} - ---- -Full report ready — open in OpenJarvis: -openjarvis://research/{session_id}""" - -_ERROR_TEMPLATE = ( - "Sorry, I ran into an error processing your request. " - "Please try again or open OpenJarvis for the full experience." -) - -_MAX_INLINE_LENGTH = 500 - - -class ChannelAgent: - """Bridges a messaging channel to the DeepResearchAgent. - - Registers as a handler on the given channel. When a message - arrives, classifies it (quick vs deep), runs the agent in a - worker thread, and sends the response back via the channel. - - Quick queries get the full response inline. - Deep queries get a brief preview + escalation link. - - Parameters - ---------- - channel: - The messaging channel to listen on. - agent: - The DeepResearchAgent (or any BaseAgent) to handle queries. - max_workers: - Thread pool size for concurrent query processing. - """ - - def __init__( - self, - channel: BaseChannel, - agent: Any, - *, - max_workers: int = 2, - ) -> None: - self._channel = channel - self._agent = agent - self._pool = ThreadPoolExecutor( - max_workers=max_workers, - thread_name_prefix="channel_agent", - ) - - # Register handler - channel.on_message(self._handle_message) - - def _handle_message(self, msg: ChannelMessage) -> Optional[str]: - """Handler callback — dispatches work to thread pool.""" - self._pool.submit(self._process_message, msg) - return None # Don't block the channel's event loop - - def _process_message(self, msg: ChannelMessage) -> None: - """Process a message in a worker thread.""" - query = msg.content.strip() - if not query: - return - - classification = classify_query(query) - session_id = uuid.uuid4().hex[:12] - - bus = get_event_bus() - bus.publish( - EventType.AGENT_TURN_START, - { - "agent": "channel_agent", - "channel": msg.channel, - "sender": msg.sender, - "classification": classification, - }, - ) - - try: - result = self._agent.run(query) - except Exception as exc: - logger.error( - "ChannelAgent error for %s: %s", - msg.sender, - exc, - ) - self._channel.send( - msg.sender, - _ERROR_TEMPLATE, - conversation_id=msg.conversation_id, - ) - return - - content = result.content.strip() - - if classification == "deep" or len(content) > _MAX_INLINE_LENGTH: - # Truncate for preview - preview = content[:300] - if len(content) > 300: - preview = preview.rsplit(" ", 1)[0] + "..." - response = _ESCALATION_TEMPLATE.format( - preview=preview, session_id=session_id - ) - else: - response = content - - self._channel.send( - msg.sender, - response, - conversation_id=msg.conversation_id, - ) - - bus.publish( - EventType.AGENT_TURN_END, - { - "agent": "channel_agent", - "channel": msg.channel, - "turns": result.turns, - "classification": classification, - "session_id": session_id, - }, - ) - - def shutdown(self) -> None: - """Shut down the worker thread pool.""" - self._pool.shutdown(wait=False) -``` - -- [ ] **Step 4: Run tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/agents/test_channel_agent.py -v` - -Expected: All 17 tests PASS (11 classifier + 6 ChannelAgent). - -- [ ] **Step 5: Commit** - -```bash -git add src/openjarvis/agents/channel_agent.py tests/agents/test_channel_agent.py -git commit -m "feat: add ChannelAgent with query classification and escalation" -``` - ---- - -### Task 3: Integration Test — ChannelAgent + DeepResearchAgent + Real Data - -**Files:** -- Create: `tests/agents/test_channel_agent_integration.py` - -- [ ] **Step 1: Write integration test** - -Create `tests/agents/test_channel_agent_integration.py`: - -```python -"""Integration test — ChannelAgent with real KnowledgeStore and mock engine.""" - -from __future__ import annotations - -import json -import time -from pathlib import Path -from unittest.mock import MagicMock - -import pytest - -from openjarvis.agents.channel_agent import ChannelAgent -from openjarvis.agents.deep_research import DeepResearchAgent -from openjarvis.channels._stubs import BaseChannel, ChannelMessage, ChannelStatus -from openjarvis.connectors._stubs import Document -from openjarvis.connectors.pipeline import IngestionPipeline -from openjarvis.connectors.retriever import TwoStageRetriever -from openjarvis.connectors.store import KnowledgeStore -from openjarvis.tools.knowledge_search import KnowledgeSearchTool - - -class FakeChannel(BaseChannel): - channel_id = "fake" - - def __init__(self) -> None: - self._handlers: list = [] - self._sent: list = [] - - def connect(self) -> None: - pass - - def disconnect(self) -> None: - pass - - def send( - self, - channel: str, - content: str, - *, - conversation_id: str = "", - metadata: dict | None = None, - ) -> bool: - self._sent.append({"content": content, "conv": conversation_id}) - return True - - def status(self) -> ChannelStatus: - return ChannelStatus.CONNECTED - - def list_channels(self) -> list[str]: - return ["test"] - - def on_message(self, handler) -> None: - self._handlers.append(handler) - - def simulate(self, text: str) -> None: - msg = ChannelMessage( - channel="fake", - sender="user", - content=text, - conversation_id="conv1", - ) - for h in self._handlers: - h(msg) - - -@pytest.fixture -def populated_store(tmp_path: Path) -> KnowledgeStore: - store = KnowledgeStore( - db_path=str(tmp_path / "channel_int.db") - ) - pipeline = IngestionPipeline(store=store, max_tokens=256) - pipeline.ingest( - [ - Document( - doc_id="gcalendar:evt1", - source="gcalendar", - doc_type="event", - content="Sprint Planning\nWhen: Tomorrow 2pm\nAttendees: Sarah, Mike", - title="Sprint Planning", - ), - Document( - doc_id="slack:msg1", - source="slack", - doc_type="message", - content="We need to finalize the API redesign before Friday", - title="#engineering", - author="sarah", - ), - Document( - doc_id="gmail:msg1", - source="gmail", - doc_type="email", - content="Budget report attached. Q3 spending up 15%.", - title="Re: Q3 Budget", - author="mike", - ), - ] - ) - return store - - -def test_quick_query_through_full_stack( - populated_store: KnowledgeStore, -) -> None: - """Quick query → agent searches → inline response.""" - mock_engine = MagicMock() - mock_engine.engine_id = "mock" - mock_engine.generate.return_value = { - "content": "Your next meeting is Sprint Planning tomorrow at 2pm with Sarah and Mike.", - "usage": {"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150}, - "model": "test", - "finish_reason": "stop", - } - - retriever = TwoStageRetriever(store=populated_store) - ks_tool = KnowledgeSearchTool( - store=populated_store, retriever=retriever - ) - agent = DeepResearchAgent( - engine=mock_engine, - model="test", - tools=[ks_tool], - max_turns=2, - ) - - channel = FakeChannel() - ca = ChannelAgent(channel=channel, agent=agent) - - channel.simulate("When is my next meeting?") - time.sleep(1.0) - - assert len(channel._sent) >= 1 - response = channel._sent[0]["content"] - assert "Sprint Planning" in response or "meeting" in response.lower() - # Quick query should NOT have escalation link - assert "openjarvis://" not in response - - ca.shutdown() - - -def test_deep_query_gets_escalation_link( - populated_store: KnowledgeStore, -) -> None: - """Deep query → agent searches → preview + escalation link.""" - search_call = { - "content": "", - "usage": {"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150}, - "model": "test", - "finish_reason": "tool_calls", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "knowledge_search", - "arguments": json.dumps({"query": "budget API"}), - }, - } - ], - } - final_answer = { - "content": ( - "## Research Report\n\n" - + "Based on analysis of Slack messages and emails, " - + "the team discussed API redesign and Q3 budget. " - * 10 - + "\n\n**Sources:**\n1. [slack] #engineering\n2. [gmail] Q3 Budget" - ), - "usage": {"prompt_tokens": 500, "completion_tokens": 300, "total_tokens": 800}, - "model": "test", - "finish_reason": "stop", - } - - mock_engine = MagicMock() - mock_engine.engine_id = "mock" - mock_engine.generate.side_effect = [search_call, final_answer] - - retriever = TwoStageRetriever(store=populated_store) - ks_tool = KnowledgeSearchTool( - store=populated_store, retriever=retriever - ) - agent = DeepResearchAgent( - engine=mock_engine, - model="test", - tools=[ks_tool], - max_turns=3, - ) - - channel = FakeChannel() - ca = ChannelAgent(channel=channel, agent=agent) - - channel.simulate( - "Summarize all discussions about budget and API redesign" - ) - time.sleep(1.0) - - assert len(channel._sent) >= 1 - response = channel._sent[-1]["content"] - # Deep query should have escalation link - assert "openjarvis://" in response - - ca.shutdown() -``` - -- [ ] **Step 2: Run integration test** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/agents/test_channel_agent_integration.py -v` - -Expected: All 2 tests PASS. - -- [ ] **Step 3: Run full test suite** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/agents/test_channel_agent.py tests/agents/test_channel_agent_integration.py tests/agents/test_deep_research.py tests/connectors/ tests/tools/test_knowledge_search.py -v --tb=short` - -Expected: All tests PASS. - -- [ ] **Step 4: Run linter** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run ruff check src/openjarvis/agents/channel_agent.py tests/agents/test_channel_agent.py tests/agents/test_channel_agent_integration.py` - -Expected: No errors. - -- [ ] **Step 5: Commit** - -```bash -git add tests/agents/test_channel_agent_integration.py -git commit -m "feat: add integration test for ChannelAgent with DeepResearchAgent" -``` - ---- - -## Post-Plan Notes - -**What this plan produces:** -- `classify_query()` — automatic quick/deep classification (no user input needed) -- `ChannelAgent` — bridges any `BaseChannel` to the `DeepResearchAgent` with: - - Non-blocking handler (thread pool) - - Quick inline answers for simple lookups - - Preview + `openjarvis://research/{session_id}` escalation link for complex research - - Error handling with user-friendly messages - - Event bus telemetry -- Integration test proving the full stack: channel message → classification → agent → tool call → retrieval → response - -**What this does NOT do (future work):** -- Platform-specific formatting (Slack Block Kit, WhatsApp markdown) — can be added per-channel later -- BlueBubbles incoming message support — needs upstream work on the bridge -- Tauri `openjarvis://` URL handler registration — that's Phase 2B (desktop) -- Persistent conversation context across messages — would need session store integration - -**Connecting to real channels:** After this plan, wiring to real Slack/WhatsApp is just: -```python -channel = SlackChannel(bot_token="xoxb-...", app_token="xapp-...") -channel.connect() -agent = DeepResearchAgent(engine, model, tools=[ks_tool]) -ca = ChannelAgent(channel=channel, agent=agent) -# Now the Slack bot responds to DMs with Deep Research answers -``` diff --git a/docs/superpowers/plans/2026-03-26-deep-research-phase5-polish.md b/docs/superpowers/plans/2026-03-26-deep-research-phase5-polish.md deleted file mode 100644 index 36f9483d..00000000 --- a/docs/superpowers/plans/2026-03-26-deep-research-phase5-polish.md +++ /dev/null @@ -1,596 +0,0 @@ -# Deep Research Phase 5: Incremental Sync + Attachment Store Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add incremental delta sync (only fetch new/modified items after initial bulk), content-addressed attachment storage with SHA-256 dedup, a sync trigger API endpoint, and text extraction from PDF/Office attachments. - -**Architecture:** The SyncEngine is extended to pass `since=last_sync_time` to connectors on subsequent syncs. A new `AttachmentStore` manages content-addressed blob storage at `~/.openjarvis/blobs/` with SHA-256 dedup and a metadata table. The IngestionPipeline is extended to extract text from attachments and index it alongside parent documents. A `POST /v1/connectors/{id}/sync` endpoint triggers syncs via the API. - -**Tech Stack:** Python 3.10+, sqlite3, hashlib (SHA-256), pdfplumber (optional, PDF text extraction), pytest - -**Spec:** `docs/superpowers/specs/2026-03-25-deep-research-setup-design.md` — Section 6 (Ingestion Pipeline), Phase 5 - -**Depends on:** Phase 1 (SyncEngine, IngestionPipeline, KnowledgeStore), Phase 2B-i (API router) - ---- - -## File Structure - -``` -src/openjarvis/connectors/ -├── attachment_store.py # Content-addressed blob store with SHA-256 dedup -├── sync_engine.py # (modify) Wire incremental sync via since param -├── pipeline.py # (modify) Process attachments during ingest -├── gmail.py # (modify) Wire since param for incremental - -src/openjarvis/server/ -├── connectors_router.py # (modify) Add POST /connectors/{id}/sync endpoint - -tests/connectors/ -├── test_attachment_store.py # AttachmentStore tests -├── test_incremental_sync.py # Incremental sync tests -``` - ---- - -### Task 1: Incremental Sync (Wire `since` Through SyncEngine) - -**Files:** -- Modify: `src/openjarvis/connectors/sync_engine.py` -- Create: `tests/connectors/test_incremental_sync.py` - -- [ ] **Step 1: Write failing tests** - -Create `tests/connectors/test_incremental_sync.py`: - -```python -"""Tests for incremental sync — only fetches items after last sync time.""" - -from __future__ import annotations - -from datetime import datetime, timezone -from pathlib import Path -from typing import Iterator, Optional - -import pytest - -from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus -from openjarvis.connectors.pipeline import IngestionPipeline -from openjarvis.connectors.store import KnowledgeStore -from openjarvis.connectors.sync_engine import SyncEngine - - -class TimestampConnector(BaseConnector): - """Connector that records what `since` value it receives.""" - - connector_id = "ts_test" - display_name = "Timestamp Test" - auth_type = "filesystem" - - def __init__(self, docs: list[Document] | None = None) -> None: - self._docs = docs or [] - self._received_since: Optional[datetime] = None - - def is_connected(self) -> bool: - return True - - def disconnect(self) -> None: - pass - - def sync( - self, - *, - since: Optional[datetime] = None, - cursor: Optional[str] = None, - ) -> Iterator[Document]: - self._received_since = since - for doc in self._docs: - if since and doc.timestamp < since: - continue - yield doc - - def sync_status(self) -> SyncStatus: - return SyncStatus(state="idle") - - -def _make_doc( - doc_id: str, ts: datetime, content: str = "test" -) -> Document: - return Document( - doc_id=doc_id, - source="ts_test", - doc_type="message", - content=content, - timestamp=ts, - ) - - -@pytest.fixture -def setup(tmp_path: Path): - store = KnowledgeStore( - db_path=str(tmp_path / "inc.db") - ) - pipeline = IngestionPipeline(store=store) - engine = SyncEngine( - pipeline=pipeline, - state_db=str(tmp_path / "state.db"), - ) - return store, pipeline, engine - - -def test_first_sync_passes_no_since(setup) -> None: - store, pipeline, engine = setup - old = _make_doc( - "ts:1", - datetime(2024, 1, 1, tzinfo=timezone.utc), - ) - conn = TimestampConnector(docs=[old]) - engine.sync(conn) - assert conn._received_since is None - - -def test_second_sync_passes_since(setup) -> None: - store, pipeline, engine = setup - old = _make_doc( - "ts:1", - datetime(2024, 1, 1, tzinfo=timezone.utc), - ) - conn1 = TimestampConnector(docs=[old]) - engine.sync(conn1) - - # Second sync should receive since= from last sync - new = _make_doc( - "ts:2", - datetime(2024, 6, 1, tzinfo=timezone.utc), - ) - conn2 = TimestampConnector(docs=[old, new]) - conn2.connector_id = "ts_test" - engine.sync(conn2) - assert conn2._received_since is not None - - -def test_incremental_only_adds_new_items(setup) -> None: - store, pipeline, engine = setup - old = _make_doc( - "ts:1", - datetime(2024, 1, 1, tzinfo=timezone.utc), - content="Old item", - ) - conn1 = TimestampConnector(docs=[old]) - engine.sync(conn1) - assert store.count() >= 1 - - count_after_first = store.count() - - # Second sync with same + new doc - new = _make_doc( - "ts:2", - datetime(2024, 6, 1, tzinfo=timezone.utc), - content="New item", - ) - conn2 = TimestampConnector(docs=[old, new]) - conn2.connector_id = "ts_test" - engine.sync(conn2) - - # Should have more chunks now - assert store.count() > count_after_first -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_incremental_sync.py -v` - -Expected: `test_second_sync_passes_since` FAILS (since is never passed) - -- [ ] **Step 3: Modify SyncEngine to pass `since`** - -In `src/openjarvis/connectors/sync_engine.py`, modify the `sync()` method. After loading the checkpoint, parse `last_sync` into a datetime and pass it as `since=` to the connector: - -Find the line that calls `connector.sync(cursor=prior_cursor)` and change it to also pass `since`: - -```python -# Parse last_sync from checkpoint into datetime -since: Optional[datetime] = None -if checkpoint and checkpoint.get("last_sync"): - try: - since = datetime.fromisoformat(checkpoint["last_sync"]) - except (ValueError, TypeError): - pass - -docs = connector.sync(since=since, cursor=prior_cursor) -``` - -Add `from datetime import datetime` to imports if not already present. - -- [ ] **Step 4: Run tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_incremental_sync.py -v` - -Expected: All 3 tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/openjarvis/connectors/sync_engine.py tests/connectors/test_incremental_sync.py -git commit -m "feat: wire incremental sync via since param in SyncEngine" -``` - ---- - -### Task 2: Attachment Store (Content-Addressed Blobs) - -**Files:** -- Create: `src/openjarvis/connectors/attachment_store.py` -- Create: `tests/connectors/test_attachment_store.py` - -- [ ] **Step 1: Write failing tests** - -Create `tests/connectors/test_attachment_store.py`: - -```python -"""Tests for AttachmentStore — content-addressed blob storage.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from openjarvis.connectors.attachment_store import AttachmentStore - - -@pytest.fixture -def store(tmp_path: Path) -> AttachmentStore: - return AttachmentStore(base_dir=str(tmp_path / "blobs")) - - -def test_store_returns_sha256(store: AttachmentStore) -> None: - sha = store.store( - content=b"Hello, world!", - filename="hello.txt", - mime_type="text/plain", - source_doc_id="gmail:msg1", - ) - assert len(sha) == 64 # SHA-256 hex digest - - -def test_store_creates_blob_file( - store: AttachmentStore, tmp_path: Path -) -> None: - sha = store.store( - content=b"PDF content here", - filename="report.pdf", - mime_type="application/pdf", - source_doc_id="gdrive:doc1", - ) - blob_path = Path(store._base_dir) / sha[:2] / sha - assert blob_path.exists() - assert blob_path.read_bytes() == b"PDF content here" - - -def test_dedup_same_content(store: AttachmentStore) -> None: - content = b"Identical file content" - sha1 = store.store( - content=content, - filename="file1.txt", - mime_type="text/plain", - source_doc_id="gmail:1", - ) - sha2 = store.store( - content=content, - filename="file2.txt", - mime_type="text/plain", - source_doc_id="slack:1", - ) - assert sha1 == sha2 # Same content → same hash - - -def test_dedup_tracks_multiple_sources( - store: AttachmentStore, -) -> None: - content = b"Shared file" - store.store( - content=content, - filename="shared.pdf", - mime_type="application/pdf", - source_doc_id="gmail:1", - ) - store.store( - content=content, - filename="shared.pdf", - mime_type="application/pdf", - source_doc_id="slack:1", - ) - meta = store.get_metadata( - store.store( - content=content, - filename="shared.pdf", - mime_type="application/pdf", - source_doc_id="gdrive:1", - ) - ) - assert len(meta["source_doc_ids"]) >= 3 - - -def test_get_metadata(store: AttachmentStore) -> None: - sha = store.store( - content=b"test", - filename="test.txt", - mime_type="text/plain", - source_doc_id="gmail:1", - ) - meta = store.get_metadata(sha) - assert meta["filename"] == "test.txt" - assert meta["mime_type"] == "text/plain" - assert meta["size_bytes"] == 4 - - -def test_get_content(store: AttachmentStore) -> None: - sha = store.store( - content=b"retrieve me", - filename="data.bin", - mime_type="application/octet-stream", - source_doc_id="test:1", - ) - assert store.get_content(sha) == b"retrieve me" - - -def test_get_content_nonexistent( - store: AttachmentStore, -) -> None: - assert store.get_content("nonexistent") is None -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_attachment_store.py -v` - -Expected: FAIL — `ModuleNotFoundError` - -- [ ] **Step 3: Implement AttachmentStore** - -Create `src/openjarvis/connectors/attachment_store.py`: - -```python -"""Content-addressed attachment store with SHA-256 dedup. - -Blobs stored at: {base_dir}/{sha256[:2]}/{sha256} -Metadata tracked in SQLite alongside the blobs. -""" - -from __future__ import annotations - -import hashlib -import json -import logging -import sqlite3 -from pathlib import Path -from typing import Any, Dict, List, Optional - -from openjarvis.core.config import DEFAULT_CONFIG_DIR - -logger = logging.getLogger(__name__) - -_DEFAULT_BASE_DIR = str(DEFAULT_CONFIG_DIR / "blobs") - -_META_SCHEMA = """\ -CREATE TABLE IF NOT EXISTS attachments ( - sha256 TEXT PRIMARY KEY, - filename TEXT NOT NULL, - mime_type TEXT NOT NULL DEFAULT '', - size_bytes INTEGER NOT NULL DEFAULT 0, - source_doc_ids TEXT NOT NULL DEFAULT '[]', - created_at TEXT NOT NULL -); -""" - - -class AttachmentStore: - """Content-addressed blob store for email attachments, shared files, etc. - - Each file is stored once by SHA-256 hash. Multiple source documents - can reference the same blob (dedup across Gmail, Slack, Drive, etc.). - """ - - def __init__(self, base_dir: str = "") -> None: - self._base_dir = base_dir or _DEFAULT_BASE_DIR - Path(self._base_dir).mkdir(parents=True, exist_ok=True) - db_path = str(Path(self._base_dir) / "attachments.db") - self._conn = sqlite3.connect(db_path) - self._conn.row_factory = sqlite3.Row - self._conn.executescript(_META_SCHEMA) - - def store( - self, - content: bytes, - *, - filename: str, - mime_type: str = "", - source_doc_id: str = "", - ) -> str: - """Store a blob and return its SHA-256 hash. - - If the same content already exists, just adds the source_doc_id - to the existing metadata (dedup). - """ - sha = hashlib.sha256(content).hexdigest() - - # Write blob file (idempotent) - blob_dir = Path(self._base_dir) / sha[:2] - blob_dir.mkdir(parents=True, exist_ok=True) - blob_path = blob_dir / sha - if not blob_path.exists(): - blob_path.write_bytes(content) - - # Upsert metadata - row = self._conn.execute( - "SELECT source_doc_ids FROM attachments WHERE sha256 = ?", - (sha,), - ).fetchone() - - if row is None: - from datetime import datetime, timezone - - self._conn.execute( - """INSERT INTO attachments - (sha256, filename, mime_type, size_bytes, - source_doc_ids, created_at) - VALUES (?, ?, ?, ?, ?, ?)""", - ( - sha, - filename, - mime_type, - len(content), - json.dumps([source_doc_id] if source_doc_id else []), - datetime.now(tz=timezone.utc).isoformat(), - ), - ) - else: - # Add source_doc_id if not already tracked - ids = json.loads(row["source_doc_ids"]) - if source_doc_id and source_doc_id not in ids: - ids.append(source_doc_id) - self._conn.execute( - "UPDATE attachments SET source_doc_ids = ? WHERE sha256 = ?", - (json.dumps(ids), sha), - ) - - self._conn.commit() - return sha - - def get_metadata(self, sha: str) -> Optional[Dict[str, Any]]: - """Get metadata for an attachment by SHA-256.""" - row = self._conn.execute( - "SELECT * FROM attachments WHERE sha256 = ?", (sha,) - ).fetchone() - if row is None: - return None - return { - "sha256": row["sha256"], - "filename": row["filename"], - "mime_type": row["mime_type"], - "size_bytes": row["size_bytes"], - "source_doc_ids": json.loads(row["source_doc_ids"]), - "created_at": row["created_at"], - } - - def get_content(self, sha: str) -> Optional[bytes]: - """Read blob content by SHA-256. Returns None if not found.""" - blob_path = Path(self._base_dir) / sha[:2] / sha - if not blob_path.exists(): - return None - return blob_path.read_bytes() - - def close(self) -> None: - self._conn.close() -``` - -- [ ] **Step 4: Run tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run pytest tests/connectors/test_attachment_store.py -v` - -Expected: All 7 tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/openjarvis/connectors/attachment_store.py tests/connectors/test_attachment_store.py -git commit -m "feat: add content-addressed AttachmentStore with SHA-256 dedup" -``` - ---- - -### Task 3: Sync Trigger API Endpoint - -**Files:** -- Modify: `src/openjarvis/server/connectors_router.py` -- Modify: `tests/server/test_connectors_router.py` - -- [ ] **Step 1: Add test for POST /sync** - -Add to `tests/server/test_connectors_router.py`: - -```python -def test_trigger_sync(app, tmp_path: Path) -> None: - """POST /v1/connectors/obsidian/sync triggers a sync.""" - vault = tmp_path / "vault" - vault.mkdir() - (vault / "note.md").write_text("# Test note\n\nContent here.") - # First connect - app.post( - "/v1/connectors/obsidian/connect", - json={"path": str(vault)}, - ) - # Then trigger sync - resp = app.post("/v1/connectors/obsidian/sync") - assert resp.status_code == 200 - data = resp.json() - assert data["chunks_indexed"] >= 1 -``` - -- [ ] **Step 2: Implement POST /sync endpoint** - -Add to the router in `src/openjarvis/server/connectors_router.py`: - -```python -@router.post("/connectors/{connector_id}/sync") -def trigger_sync(connector_id: str) -> Dict[str, Any]: - """Trigger an incremental sync for a connector.""" - if not ConnectorRegistry.contains(connector_id): - raise HTTPException( - status_code=404, - detail=f"Connector '{connector_id}' not found", - ) - inst = _get_or_create(connector_id) - if not inst.is_connected(): - raise HTTPException( - status_code=400, - detail=f"Connector '{connector_id}' is not connected", - ) - - from openjarvis.connectors.pipeline import IngestionPipeline - from openjarvis.connectors.store import KnowledgeStore - from openjarvis.connectors.sync_engine import SyncEngine - - store = KnowledgeStore() - pipeline = IngestionPipeline(store=store) - engine = SyncEngine(pipeline=pipeline) - chunks = engine.sync(inst) - - return { - "connector_id": connector_id, - "chunks_indexed": chunks, - "status": "complete", - } -``` - -- [ ] **Step 3: Run tests** - -Run: `cd /lambda/nfs/lambda-stanford/jonsf/scratch_v2/OpenJarvis && uv run --extra server pytest tests/server/test_connectors_router.py -v` - -Expected: All tests PASS (existing + new). - -- [ ] **Step 4: Commit** - -```bash -git add src/openjarvis/server/connectors_router.py tests/server/test_connectors_router.py -git commit -m "feat: add POST /v1/connectors/{id}/sync to trigger incremental sync" -``` - ---- - -## Post-Plan Notes - -**What this plan produces:** -- Incremental sync via `since=last_sync_time` passed to connectors on subsequent syncs -- Content-addressed `AttachmentStore` with SHA-256 dedup and multi-source tracking -- `POST /v1/connectors/{id}/sync` API endpoint to trigger syncs -- Tests for all of the above - -**What's NOT in this plan (future):** -- PDF text extraction from attachments (needs pdfplumber wired into pipeline) -- Office doc extraction (python-docx/openpyxl) -- Wiring attachment extraction into the IngestionPipeline -- Populating `doc.attachments` in Gmail/Drive connectors - -These are natural follow-ups once the store exists. The current plan establishes the foundation (dedup blob store + incremental sync + sync trigger API). diff --git a/docs/superpowers/plans/2026-03-26-deep-research-vertical-slice.md b/docs/superpowers/plans/2026-03-26-deep-research-vertical-slice.md deleted file mode 100644 index 7261f4e6..00000000 --- a/docs/superpowers/plans/2026-03-26-deep-research-vertical-slice.md +++ /dev/null @@ -1,757 +0,0 @@ -# Deep Research Vertical Slice Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Get the full Deep Research experience working end-to-end on a MacBook with real data — connect local sources, ingest, retrieve, and run the DeepResearchAgent with Qwen3.5 4B via Ollama producing cited research reports. - -**Architecture:** Fix the Apple Notes protobuf extraction test, build a `jarvis deep-research-setup` CLI command that auto-detects local sources (Apple Notes, iMessage, Obsidian), ingests them into a shared KnowledgeStore, and launches an interactive chat session with the DeepResearchAgent. Then wire the missing API router so the desktop wizard UI works too. - -**Tech Stack:** Python 3.10+, Click (CLI), SQLite/FTS5, Ollama (Qwen3.5 4B), ColBERT (optional reranking), React/Tauri (wizard smoke test) - ---- - -### Task 1: Fix Apple Notes test - -**Files:** -- Modify: `src/openjarvis/connectors/apple_notes.py` (lines 73-103, `_extract_text_from_zdata`) -- Modify: `tests/connectors/test_apple_notes.py` (lines 42-58, test data + lines 127-140, assertion) - -The `_extract_text_from_zdata` function currently strips protobuf control bytes but not HTML tags. The test creates fake notes with HTML content. Real Apple Notes uses protobuf. Fix: strip HTML tags too (handles both formats gracefully), and update the test data to use protobuf-like content that exercises the actual code path. - -- [ ] **Step 1: Read current files** - -Read both files to confirm the exact current state (there are uncommitted changes from the earlier session): - -```bash -git diff src/openjarvis/connectors/apple_notes.py -git diff tests/connectors/test_apple_notes.py -``` - -- [ ] **Step 2: Update `_extract_text_from_zdata` to also strip HTML tags** - -In `src/openjarvis/connectors/apple_notes.py`, the function should strip HTML tags after stripping protobuf bytes. This handles both real protobuf data and HTML test data. Edit the function: - -```python -def _extract_text_from_zdata(zdata: bytes) -> str: - """Decompress gzip bytes and extract plain text from the protobuf payload. - - Parameters - ---------- - zdata: - Raw bytes from the ``ZDATA`` column — gzip-compressed protobuf - (``com.apple.notes.ICNote``). - - Returns - ------- - str - Plain text with protobuf control bytes stripped. Returns an empty - string if decompression fails. - """ - try: - raw = gzip.decompress(zdata) - except Exception: # noqa: BLE001 - return "" - - text = raw.decode("utf-8", errors="replace") - # Strip HTML tags (older Notes versions or test data may contain HTML) - text = re.sub(r"<[^>]+>", "", text) - # Strip non-printable control bytes and U+FFFD replacement chars that - # come from the protobuf wire format. - cleaned = re.sub(r"[\x00-\x09\x0b\x0c\x0e-\x1f\x7f-\x9f\ufffd]+", " ", text) - # Collapse whitespace runs - cleaned = re.sub(r" {2,}", " ", cleaned) - cleaned = re.sub(r"\n{3,}", "\n\n", cleaned) - cleaned = cleaned.strip() - # Strip short leading protobuf varint artifacts (e.g. "3 3 " or "b b ") - cleaned = re.sub(r"^(?:[a-z0-9] ){1,4}", "", cleaned) - return cleaned.strip() -``` - -- [ ] **Step 3: Run the Apple Notes tests** - -```bash -uv run pytest tests/connectors/test_apple_notes.py -v --tb=short -``` - -Expected: All 9 tests PASS (including `test_sync_decompresses_content`). - -- [ ] **Step 4: Run the iMessage tests too (regression check)** - -```bash -uv run pytest tests/connectors/test_imessage.py -v --tb=short -``` - -Expected: All tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/openjarvis/connectors/apple_notes.py tests/connectors/test_apple_notes.py -git commit -m "fix: Apple Notes protobuf extraction handles HTML too, fix ZTITLE1 column" -``` - ---- - -### Task 2: Build `jarvis deep-research-setup` CLI command - -**Files:** -- Create: `src/openjarvis/cli/deep_research_setup_cmd.py` -- Modify: `src/openjarvis/cli/__init__.py` (add command registration) -- Test: `tests/cli/test_deep_research_setup.py` - -This is the core of the vertical slice — a single command that auto-detects local sources, ingests data, and drops you into a research chat. - -- [ ] **Step 1: Write the test file** - -Create `tests/cli/test_deep_research_setup.py`: - -```python -"""Tests for the deep-research-setup CLI command.""" - -from __future__ import annotations - -import sqlite3 -import tempfile -from pathlib import Path -from unittest.mock import patch - -import pytest -from click.testing import CliRunner - -from openjarvis.cli import cli - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def _create_fake_notes_db(db_path: Path) -> None: - """Create a minimal Apple Notes SQLite database.""" - import gzip - - conn = sqlite3.connect(str(db_path)) - conn.executescript(""" - CREATE TABLE ZICCLOUDSYNCINGOBJECT ( - Z_PK INTEGER PRIMARY KEY, - ZTITLE TEXT, - ZTITLE1 TEXT, - ZMODIFICATIONDATE REAL, - ZIDENTIFIER TEXT, - ZNOTE INTEGER - ); - CREATE TABLE ZICNOTEDATA ( - Z_PK INTEGER PRIMARY KEY, - ZDATA BLOB, - ZNOTE INTEGER - ); - """) - content = gzip.compress(b"Test note about meetings") - conn.execute( - "INSERT INTO ZICCLOUDSYNCINGOBJECT VALUES (1, NULL, 'Test Note', 694310400.0, 'n1', 1)" - ) - conn.execute("INSERT INTO ZICNOTEDATA VALUES (1, ?, 1)", (content,)) - conn.commit() - conn.close() - - -def _create_fake_imessage_db(db_path: Path) -> None: - """Create a minimal iMessage SQLite database.""" - conn = sqlite3.connect(str(db_path)) - conn.executescript(""" - CREATE TABLE handle (ROWID INTEGER PRIMARY KEY, id TEXT); - CREATE TABLE chat (ROWID INTEGER PRIMARY KEY, chat_identifier TEXT, display_name TEXT); - CREATE TABLE chat_message_join (chat_id INTEGER, message_id INTEGER); - CREATE TABLE message ( - ROWID INTEGER PRIMARY KEY, text TEXT, handle_id INTEGER, - date INTEGER, is_from_me INTEGER - ); - """) - conn.execute("INSERT INTO handle VALUES (1, '+15551234567')") - conn.execute("INSERT INTO chat VALUES (1, '+15551234567', 'Test Chat')") - conn.execute("INSERT INTO chat_message_join VALUES (1, 1)") - conn.execute("INSERT INTO message VALUES (1, 'Hello from test', 1, 694310400000000000, 0)") - conn.commit() - conn.close() - - -# --------------------------------------------------------------------------- -# Tests -# --------------------------------------------------------------------------- - - -def test_detect_local_sources(tmp_path: Path) -> None: - """Auto-detection finds Apple Notes and iMessage when DBs exist.""" - from openjarvis.cli.deep_research_setup_cmd import detect_local_sources - - notes_db = tmp_path / "NoteStore.sqlite" - imessage_db = tmp_path / "chat.db" - _create_fake_notes_db(notes_db) - _create_fake_imessage_db(imessage_db) - - sources = detect_local_sources( - notes_db_path=notes_db, - imessage_db_path=imessage_db, - obsidian_vault_path=None, - ) - ids = [s["connector_id"] for s in sources] - assert "apple_notes" in ids - assert "imessage" in ids - - -def test_detect_skips_missing_sources(tmp_path: Path) -> None: - """Auto-detection skips sources whose files don't exist.""" - from openjarvis.cli.deep_research_setup_cmd import detect_local_sources - - sources = detect_local_sources( - notes_db_path=tmp_path / "nonexistent.sqlite", - imessage_db_path=tmp_path / "nonexistent.db", - obsidian_vault_path=None, - ) - assert len(sources) == 0 - - -def test_detect_includes_obsidian_when_vault_exists(tmp_path: Path) -> None: - """Auto-detection includes Obsidian when vault path exists.""" - from openjarvis.cli.deep_research_setup_cmd import detect_local_sources - - vault = tmp_path / "vault" - vault.mkdir() - (vault / "note.md").write_text("# Hello") - - sources = detect_local_sources( - notes_db_path=tmp_path / "nonexistent.sqlite", - imessage_db_path=tmp_path / "nonexistent.db", - obsidian_vault_path=vault, - ) - ids = [s["connector_id"] for s in sources] - assert "obsidian" in ids - - -def test_ingest_sources(tmp_path: Path) -> None: - """ingest_sources connects and ingests documents into KnowledgeStore.""" - from openjarvis.cli.deep_research_setup_cmd import detect_local_sources, ingest_sources - from openjarvis.connectors.store import KnowledgeStore - - notes_db = tmp_path / "NoteStore.sqlite" - _create_fake_notes_db(notes_db) - - sources = detect_local_sources( - notes_db_path=notes_db, - imessage_db_path=tmp_path / "nonexistent.db", - obsidian_vault_path=None, - ) - - db_path = tmp_path / "knowledge.db" - store = KnowledgeStore(str(db_path)) - total = ingest_sources(sources, store) - - assert total > 0 - assert store.count() > 0 - store.close() -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/cli/test_deep_research_setup.py -v --tb=short -``` - -Expected: FAIL — `ModuleNotFoundError: No module named 'openjarvis.cli.deep_research_setup_cmd'` - -- [ ] **Step 3: Create the CLI command module** - -Create `src/openjarvis/cli/deep_research_setup_cmd.py`: - -```python -"""``jarvis deep-research-setup`` — auto-detect local sources, ingest, and chat. - -Walks the user through connecting local data sources (Apple Notes, iMessage, -Obsidian), ingesting them into a shared KnowledgeStore, and launching an -interactive Deep Research chat session with Qwen3.5 via Ollama. -""" - -from __future__ import annotations - -import sys -from pathlib import Path -from typing import Any, Dict, List, Optional - -import click -from rich.console import Console -from rich.table import Table - -from openjarvis.connectors.pipeline import IngestionPipeline -from openjarvis.connectors.store import KnowledgeStore -from openjarvis.connectors.sync_engine import SyncEngine -from openjarvis.core.config import DEFAULT_CONFIG_DIR - -# --------------------------------------------------------------------------- -# Constants -# --------------------------------------------------------------------------- - -_DEFAULT_NOTES_DB = ( - Path.home() - / "Library" - / "Group Containers" - / "group.com.apple.notes" - / "NoteStore.sqlite" -) - -_DEFAULT_IMESSAGE_DB = Path.home() / "Library" / "Messages" / "chat.db" - -_OLLAMA_MODEL = "qwen3.5:4b" - -# --------------------------------------------------------------------------- -# Detection -# --------------------------------------------------------------------------- - - -def detect_local_sources( - *, - notes_db_path: Optional[Path] = None, - imessage_db_path: Optional[Path] = None, - obsidian_vault_path: Optional[Path] = None, -) -> List[Dict[str, Any]]: - """Return a list of available local sources with their config. - - Each entry is a dict with keys: ``connector_id``, ``display_name``, - ``config`` (kwargs for the connector constructor). - """ - sources: List[Dict[str, Any]] = [] - - notes_path = notes_db_path or _DEFAULT_NOTES_DB - if notes_path.exists(): - sources.append({ - "connector_id": "apple_notes", - "display_name": "Apple Notes", - "config": {"db_path": str(notes_path)}, - }) - - imessage_path = imessage_db_path or _DEFAULT_IMESSAGE_DB - if imessage_path.exists(): - sources.append({ - "connector_id": "imessage", - "display_name": "iMessage", - "config": {"db_path": str(imessage_path)}, - }) - - if obsidian_vault_path and obsidian_vault_path.is_dir(): - sources.append({ - "connector_id": "obsidian", - "display_name": "Obsidian / Markdown", - "config": {"vault_path": str(obsidian_vault_path)}, - }) - - return sources - - -# --------------------------------------------------------------------------- -# Ingestion -# --------------------------------------------------------------------------- - - -def _instantiate_connector(connector_id: str, config: Dict[str, Any]) -> Any: - """Lazily import and instantiate a connector by ID.""" - if connector_id == "apple_notes": - from openjarvis.connectors.apple_notes import AppleNotesConnector - return AppleNotesConnector(db_path=config.get("db_path", "")) - elif connector_id == "imessage": - from openjarvis.connectors.imessage import IMessageConnector - return IMessageConnector(db_path=config.get("db_path", "")) - elif connector_id == "obsidian": - from openjarvis.connectors.obsidian import ObsidianConnector - return ObsidianConnector(vault_path=config.get("vault_path", "")) - else: - msg = f"Unknown connector: {connector_id}" - raise ValueError(msg) - - -def ingest_sources( - sources: List[Dict[str, Any]], - store: KnowledgeStore, -) -> int: - """Connect and ingest all sources into the KnowledgeStore. - - Returns total chunks indexed across all sources. - """ - pipeline = IngestionPipeline(store) - engine = SyncEngine(pipeline) - total = 0 - for src in sources: - connector = _instantiate_connector(src["connector_id"], src["config"]) - chunks = engine.sync(connector) - total += chunks - return total - - -# --------------------------------------------------------------------------- -# Chat launch -# --------------------------------------------------------------------------- - - -def _launch_chat(store: KnowledgeStore, console: Console) -> None: - """Start an interactive Deep Research chat session.""" - from openjarvis.agents.deep_research import DeepResearchAgent - from openjarvis.connectors.retriever import TwoStageRetriever - from openjarvis.engine.ollama import OllamaEngine - from openjarvis.tools.knowledge_search import KnowledgeSearchTool - - console.print("\n[bold]Setting up Deep Research agent...[/bold]") - - # Engine - engine = OllamaEngine() - if not engine.health(): - console.print( - "[red]Ollama is not running.[/red] Start it with: " - "[bold]ollama serve[/bold]" - ) - return - - models = engine.list_models() - if _OLLAMA_MODEL not in models and f"{_OLLAMA_MODEL}:latest" not in models: - # Check without tag too - base_name = _OLLAMA_MODEL.split(":")[0] - matching = [m for m in models if m.startswith(base_name)] - if not matching: - console.print( - f"[yellow]Model {_OLLAMA_MODEL} not found.[/yellow] " - f"Pull it with: [bold]ollama pull {_OLLAMA_MODEL}[/bold]" - ) - return - - # Retriever + tool - retriever = TwoStageRetriever(store) - search_tool = KnowledgeSearchTool(retriever=retriever) - - # Agent - agent = DeepResearchAgent( - engine=engine, - model=_OLLAMA_MODEL, - tools=[search_tool], - interactive=True, - ) - - console.print( - f"[green]Ready![/green] Using [bold]{_OLLAMA_MODEL}[/bold] via Ollama.\n" - "Type your research question. Type [bold]/quit[/bold] to exit.\n" - ) - - # REPL - while True: - try: - query = console.input("[bold blue]research>[/bold blue] ").strip() - except (EOFError, KeyboardInterrupt): - break - - if not query: - continue - if query.lower() in ("/quit", "/exit", "quit", "exit"): - break - - try: - result = agent.run(query) - console.print(f"\n{result.content}\n") - if result.metadata and result.metadata.get("sources"): - console.print("[dim]Sources:[/dim]") - for s in result.metadata["sources"]: - console.print(f" [dim]- {s}[/dim]") - console.print() - except Exception as exc: # noqa: BLE001 - console.print(f"[red]Error: {exc}[/red]\n") - - -# --------------------------------------------------------------------------- -# CLI command -# --------------------------------------------------------------------------- - - -@click.command("deep-research-setup") -@click.option( - "--obsidian-vault", - type=click.Path(exists=True, file_okay=False), - default=None, - help="Path to an Obsidian vault directory.", -) -@click.option("--skip-chat", is_flag=True, help="Ingest only, don't launch chat.") -def deep_research_setup(obsidian_vault: Optional[str], skip_chat: bool) -> None: - """Auto-detect local data sources, ingest, and launch Deep Research chat.""" - console = Console() - console.print("\n[bold]Deep Research Setup[/bold]\n") - - # 1. Detect - vault_path = Path(obsidian_vault) if obsidian_vault else None - sources = detect_local_sources(obsidian_vault_path=vault_path) - - if not sources: - console.print( - "[yellow]No local data sources detected.[/yellow]\n" - "On macOS, ensure Full Disk Access is granted in " - "System Settings > Privacy & Security." - ) - sys.exit(1) - - # 2. Confirm - table = Table(title="Detected Sources") - table.add_column("Source", style="bold") - table.add_column("Status", style="green") - for src in sources: - table.add_row(src["display_name"], "ready") - console.print(table) - console.print() - - if not click.confirm("Ingest these sources?", default=True): - sys.exit(0) - - # 3. Ingest - db_path = DEFAULT_CONFIG_DIR / "knowledge.db" - db_path.parent.mkdir(parents=True, exist_ok=True) - store = KnowledgeStore(str(db_path)) - - console.print("\n[bold]Ingesting...[/bold]") - for src in sources: - connector = _instantiate_connector(src["connector_id"], src["config"]) - pipeline = IngestionPipeline(store) - engine = SyncEngine(pipeline) - chunks = engine.sync(connector) - console.print(f" {src['display_name']}: [green]{chunks} chunks[/green]") - - console.print(f"\n[bold green]Done![/bold green] {store.count()} total chunks in {db_path}\n") - - # 4. Chat - if skip_chat: - return - - _launch_chat(store, console) -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -uv run pytest tests/cli/test_deep_research_setup.py -v --tb=short -``` - -Expected: All 4 tests PASS. - -- [ ] **Step 5: Register the command in CLI __init__.py** - -In `src/openjarvis/cli/__init__.py`, add the import and registration alongside the existing commands. - -Add the import near the other CLI imports: - -```python -from openjarvis.cli.deep_research_setup_cmd import deep_research_setup -``` - -Add the registration near the other `cli.add_command()` calls: - -```python -cli.add_command(deep_research_setup, "deep-research-setup") -``` - -- [ ] **Step 6: Verify the command is discoverable** - -```bash -uv run jarvis --help | grep deep-research -``` - -Expected: `deep-research-setup` appears in the command list. - -- [ ] **Step 7: Commit** - -```bash -git add src/openjarvis/cli/deep_research_setup_cmd.py tests/cli/test_deep_research_setup.py src/openjarvis/cli/__init__.py -git commit -m "feat: add jarvis deep-research-setup CLI command" -``` - ---- - -### Task 3: Wire connectors API router into FastAPI app - -**Files:** -- Modify: `src/openjarvis/server/app.py` - -The `connectors_router.py` already has all 6 endpoints implemented. It was never registered in the FastAPI app. This is a one-line fix. - -- [ ] **Step 1: Read app.py to find where routers are included** - -```bash -grep -n "include_router" src/openjarvis/server/app.py -``` - -Identify the section where `app.include_router(router)` and `app.include_router(dashboard_router)` are called. - -- [ ] **Step 2: Add the connectors router import and registration** - -Add the import at the top of `app.py` with the other router imports: - -```python -from openjarvis.server.connectors_router import create_connectors_router -``` - -Add the router registration in the same block as the other `app.include_router()` calls: - -```python -app.include_router(create_connectors_router()) -``` - -- [ ] **Step 3: Verify the endpoints are now exposed** - -```bash -uv run python3 -c " -from openjarvis.server.app import create_app -from openjarvis.engine.ollama import OllamaEngine - -engine = OllamaEngine() -app = create_app(engine, 'test') -routes = [r.path for r in app.routes] -connector_routes = [r for r in routes if 'connector' in r] -print(f'Connector routes: {connector_routes}') -assert any('connector' in r for r in routes), 'No connector routes found!' -print('PASS: Connectors router is registered') -" -``` - -Expected: Lists `/connectors`, `/connectors/{connector_id}`, etc. and prints PASS. - -- [ ] **Step 4: Commit** - -```bash -git add src/openjarvis/server/app.py -git commit -m "fix: register connectors API router in FastAPI app" -``` - ---- - -### Task 4: End-to-end test with real data - -**Files:** None (manual testing) - -This is the live validation. We pull the Ollama model, run the setup command, and test the Deep Research agent with real personal data. - -- [ ] **Step 1: Ensure Ollama is running and pull the model** - -```bash -ollama list -ollama pull qwen3.5:4b -``` - -Expected: Model downloaded. If Ollama isn't running, start it with `ollama serve &`. - -- [ ] **Step 2: Run the deep research setup** - -```bash -uv run jarvis deep-research-setup -``` - -Expected output: -``` -Deep Research Setup - - Detected Sources -┌──────────────┬────────┐ -│ Source │ Status │ -├──────────────┼────────┤ -│ Apple Notes │ ready │ -│ iMessage │ ready │ -└──────────────┴────────┘ - -Ingest these sources? [Y/n]: Y - -Ingesting... - Apple Notes: ~100 chunks - iMessage: ~52000 chunks - -Done! ~52100 total chunks in ~/.openjarvis/knowledge.db - -Setting up Deep Research agent... -Ready! Using qwen3.5:4b via Ollama. -Type your research question. Type /quit to exit. - -research> -``` - -If iMessage ingestion is too slow (52K messages), you can Ctrl+C and re-run with a smaller dataset — we can add a `--limit` flag later. - -- [ ] **Step 3: Test retrieval quality with 3 queries** - -At the `research>` prompt, try: - -1. A person's name who appears in both Apple Notes and iMessage -2. A topic from your notes (e.g. "Georgia Tech" which appeared in Apple Notes) -3. A time-bounded query like "what messages did I send last week" - -Evaluate: Does the agent use `knowledge_search`? Does it cite sources? Does it make multiple hops? - -- [ ] **Step 4: Document results** - -Note: query, response quality, number of tool calls, latency, and any errors. This informs whether we need a larger model or retrieval tuning. - ---- - -### Task 5: Smoke test wizard UI - -**Files:** -- None (manual testing) - -- [ ] **Step 1: Start the FastAPI server** - -```bash -uv run jarvis serve --host 127.0.0.1 --port 8000 -``` - -- [ ] **Step 2: Open the frontend dev server** - -In a second terminal: - -```bash -cd frontend && npm run dev -``` - -Expected: Vite dev server starts at `http://localhost:5173`. - -- [ ] **Step 3: Walk through the wizard** - -Open `http://localhost:5173` in a browser. The setup wizard should: -1. Show the source picker with Apple Notes, iMessage, etc. -2. Allow connecting local sources (click → instant green checkmark for local auth types) -3. Show the ingest dashboard with progress polling from `/v1/connectors/{id}/sync` -4. Land on the "Ready" screen - -Note any errors — the frontend was built on a remote server and never build-tested until now. - -- [ ] **Step 4: Commit any fixes needed** - -If frontend or API fixes are required, commit them: - -```bash -git add -A -git commit -m "fix: wizard UI smoke test fixes" -``` - ---- - -### Task 6: Run full test suite (regression check) - -**Files:** None - -- [ ] **Step 1: Run all connector + agent tests** - -```bash -uv run pytest tests/connectors/ tests/agents/test_deep_research.py tests/agents/test_deep_research_integration.py tests/agents/test_channel_agent.py tests/agents/test_channel_agent_integration.py tests/cli/test_deep_research_setup.py -v --tb=short -``` - -Expected: All tests PASS (220+ existing + 4 new from Task 2). - -- [ ] **Step 2: Run linter** - -```bash -uv run ruff check src/openjarvis/cli/deep_research_setup_cmd.py src/openjarvis/server/app.py src/openjarvis/connectors/apple_notes.py -``` - -Expected: No errors. - -- [ ] **Step 3: Commit and push** - -```bash -git push origin feat/deep-research-setup -``` diff --git a/docs/superpowers/plans/2026-03-26-outlook-and-token-sources.md b/docs/superpowers/plans/2026-03-26-outlook-and-token-sources.md deleted file mode 100644 index c630afb1..00000000 --- a/docs/superpowers/plans/2026-03-26-outlook-and-token-sources.md +++ /dev/null @@ -1,724 +0,0 @@ -# Outlook Connector + Token Source Integration Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Generalize Gmail IMAP to support Outlook, and extend `jarvis deep-research-setup` to detect, connect, and ingest token-based sources (Slack, Notion, Granola, Gmail IMAP, Outlook) alongside local sources. - -**Architecture:** Add `imap_host` parameter to `GmailIMAPConnector`, create a thin `OutlookConnector` subclass, then extend the CLI command with `detect_token_sources()` and an interactive connect prompt that saves credentials and ingests all sources in one flow. - -**Tech Stack:** Python 3.10+, imaplib, Click, Rich, SQLite/FTS5, pytest - ---- - -### Task 1: Generalize GmailIMAPConnector with `imap_host` parameter - -**Files:** -- Modify: `src/openjarvis/connectors/gmail_imap.py` -- Modify: `tests/connectors/test_gmail_imap.py` - -- [ ] **Step 1: Read the current test file to understand existing test patterns** - -```bash -cat tests/connectors/test_gmail_imap.py -``` - -- [ ] **Step 2: Add `imap_host` parameter to `GmailIMAPConnector.__init__`** - -In `src/openjarvis/connectors/gmail_imap.py`, add a class attribute and constructor parameter: - -```python -# Add class attribute after line 84 (after display_name): -_default_imap_host = "imap.gmail.com" - -# Change __init__ signature to add imap_host: -def __init__( - self, - email_address: str = "", - app_password: str = "", - credentials_path: str = "", - *, - imap_host: str = "", - max_messages: int = 500, -) -> None: - self._email = email_address - self._password = app_password - self._credentials_path = credentials_path or _DEFAULT_CREDENTIALS_PATH - self._imap_host = imap_host or self._default_imap_host - self._max_messages = max_messages - self._items_synced = 0 - self._items_total = 0 -``` - -- [ ] **Step 3: Update `sync()` to use `self._imap_host`** - -In `src/openjarvis/connectors/gmail_imap.py`, line 147, change: - -```python -# Before: -imap = imaplib.IMAP4_SSL("imap.gmail.com") - -# After: -imap = imaplib.IMAP4_SSL(self._imap_host) -``` - -- [ ] **Step 4: Run existing tests to verify nothing breaks** - -```bash -uv run pytest tests/connectors/test_gmail_imap.py -v --tb=short -``` - -Expected: All existing tests PASS (they mock IMAP so the host doesn't matter). - -- [ ] **Step 5: Commit** - -```bash -git add src/openjarvis/connectors/gmail_imap.py -git commit -m "feat: add imap_host parameter to GmailIMAPConnector" -``` - ---- - -### Task 2: Create Outlook connector - -**Files:** -- Create: `src/openjarvis/connectors/outlook.py` -- Create: `tests/connectors/test_outlook.py` - -- [ ] **Step 1: Write the test file** - -Create `tests/connectors/test_outlook.py`: - -```python -"""Tests for OutlookConnector — thin subclass of GmailIMAPConnector.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from openjarvis.core.registry import ConnectorRegistry - - -def test_outlook_registered() -> None: - """OutlookConnector is discoverable via ConnectorRegistry.""" - from openjarvis.connectors.outlook import OutlookConnector - - ConnectorRegistry.register_value("outlook", OutlookConnector) - assert ConnectorRegistry.contains("outlook") - cls = ConnectorRegistry.get("outlook") - assert cls.connector_id == "outlook" - assert cls.display_name == "Outlook / Microsoft 365" - - -def test_outlook_uses_correct_imap_host() -> None: - """OutlookConnector defaults to outlook.office365.com.""" - from openjarvis.connectors.outlook import OutlookConnector - - conn = OutlookConnector() - assert conn._imap_host == "outlook.office365.com" - - -def test_outlook_auth_url() -> None: - """auth_url() points to Microsoft security page.""" - from openjarvis.connectors.outlook import OutlookConnector - - conn = OutlookConnector() - assert "microsoft.com" in conn.auth_url() - - -def test_outlook_handle_callback(tmp_path: Path) -> None: - """handle_callback saves email:password credentials.""" - from openjarvis.connectors.outlook import OutlookConnector - from openjarvis.connectors.oauth import load_tokens - - creds_path = str(tmp_path / "outlook.json") - conn = OutlookConnector(credentials_path=creds_path) - conn.handle_callback("user@outlook.com:mypassword123") - - tokens = load_tokens(creds_path) - assert tokens is not None - assert tokens["email"] == "user@outlook.com" - assert tokens["password"] == "mypassword123" - - -def test_outlook_is_connected(tmp_path: Path) -> None: - """is_connected returns True when credentials file has email+password.""" - from openjarvis.connectors.outlook import OutlookConnector - - creds_path = str(tmp_path / "outlook.json") - conn = OutlookConnector(credentials_path=creds_path) - assert conn.is_connected() is False - - conn.handle_callback("user@outlook.com:pass") - assert conn.is_connected() is True - - -def test_outlook_sync_source_is_outlook(tmp_path: Path) -> None: - """Documents yielded by sync() have source='outlook' and doc_id prefix 'outlook:'.""" - from unittest.mock import MagicMock, patch - from openjarvis.connectors.outlook import OutlookConnector - - creds_path = str(tmp_path / "outlook.json") - conn = OutlookConnector(credentials_path=creds_path) - conn.handle_callback("user@outlook.com:pass") - - # Mock IMAP to return one email - mock_imap = MagicMock() - mock_imap.login.return_value = ("OK", []) - mock_imap.select.return_value = ("OK", []) - mock_imap.search.return_value = ("OK", [b"1"]) - - raw_email = ( - b"From: sender@test.com\r\n" - b"To: user@outlook.com\r\n" - b"Subject: Test Email\r\n" - b"Date: Mon, 01 Jan 2024 00:00:00 +0000\r\n" - b"Message-ID: \r\n" - b"\r\n" - b"Hello from Outlook test" - ) - mock_imap.fetch.return_value = ("OK", [(b"1", raw_email)]) - mock_imap.logout.return_value = ("OK", []) - - with patch("openjarvis.connectors.gmail_imap.imaplib") as mock_imaplib: - mock_imaplib.IMAP4_SSL.return_value = mock_imap - mock_imaplib.IMAP4 = type(mock_imap) - docs = list(conn.sync()) - - assert len(docs) == 1 - assert docs[0].source == "outlook" - assert docs[0].doc_id.startswith("outlook:") - assert docs[0].title == "Test Email" -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/connectors/test_outlook.py -v --tb=short -``` - -Expected: FAIL — `ModuleNotFoundError: No module named 'openjarvis.connectors.outlook'` - -- [ ] **Step 3: Create the Outlook connector** - -Create `src/openjarvis/connectors/outlook.py`: - -```python -"""Outlook / Microsoft 365 connector — reads email via IMAP with app password. - -Thin subclass of GmailIMAPConnector that defaults to the Outlook IMAP host -and relabels documents with source='outlook'. - -Setup: enable 2FA on your Microsoft account, then generate an app password -at https://account.microsoft.com/security -""" - -from __future__ import annotations - -from typing import Iterator, Optional - -from datetime import datetime - -from openjarvis.connectors._stubs import Document -from openjarvis.connectors.gmail_imap import GmailIMAPConnector -from openjarvis.core.config import DEFAULT_CONFIG_DIR -from openjarvis.core.registry import ConnectorRegistry - -_DEFAULT_CREDENTIALS_PATH = str( - DEFAULT_CONFIG_DIR / "connectors" / "outlook.json" -) - - -@ConnectorRegistry.register("outlook") -class OutlookConnector(GmailIMAPConnector): - """Outlook connector using IMAP + app password. - - Inherits all IMAP logic from :class:`GmailIMAPConnector` and overrides - the IMAP host, credential path, auth URL, and document source labels. - """ - - connector_id = "outlook" - display_name = "Outlook / Microsoft 365" - _default_imap_host = "outlook.office365.com" - - def __init__( - self, - email_address: str = "", - app_password: str = "", - credentials_path: str = "", - *, - max_messages: int = 500, - ) -> None: - super().__init__( - email_address, - app_password, - credentials_path or _DEFAULT_CREDENTIALS_PATH, - max_messages=max_messages, - ) - - def auth_url(self) -> str: - """Return the Microsoft account security page for app passwords.""" - return "https://account.microsoft.com/security" - - def sync( - self, - *, - since: Optional[datetime] = None, - cursor: Optional[str] = None, - ) -> Iterator[Document]: - """Sync emails and relabel with source='outlook'.""" - for doc in super().sync(since=since, cursor=cursor): - doc.source = "outlook" - doc.doc_id = doc.doc_id.replace("gmail:", "outlook:", 1) - yield doc -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -uv run pytest tests/connectors/test_outlook.py -v --tb=short -``` - -Expected: All 6 tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add src/openjarvis/connectors/outlook.py tests/connectors/test_outlook.py -git commit -m "feat: add Outlook connector as IMAP subclass" -``` - ---- - -### Task 3: Extend deep-research-setup with token source detection and interactive connect - -**Files:** -- Modify: `src/openjarvis/cli/deep_research_setup_cmd.py` -- Modify: `tests/cli/test_deep_research_setup.py` - -- [ ] **Step 1: Write tests for token source detection** - -Append to `tests/cli/test_deep_research_setup.py`: - -```python -def test_detect_token_sources_finds_connected(tmp_path: Path) -> None: - """detect_token_sources finds sources with valid credential files.""" - from openjarvis.cli.deep_research_setup_cmd import detect_token_sources - - connectors_dir = tmp_path / "connectors" - connectors_dir.mkdir() - (connectors_dir / "slack.json").write_text('{"token": "xoxb-test"}') - (connectors_dir / "notion.json").write_text('{"token": "ntn_test"}') - - sources = detect_token_sources(connectors_dir=connectors_dir) - ids = [s["connector_id"] for s in sources] - assert "slack" in ids - assert "notion" in ids - - -def test_detect_token_sources_skips_empty(tmp_path: Path) -> None: - """detect_token_sources skips files with empty or invalid JSON.""" - from openjarvis.cli.deep_research_setup_cmd import detect_token_sources - - connectors_dir = tmp_path / "connectors" - connectors_dir.mkdir() - (connectors_dir / "slack.json").write_text("{}") - (connectors_dir / "notion.json").write_text("invalid json") - - sources = detect_token_sources(connectors_dir=connectors_dir) - assert len(sources) == 0 - - -def test_detect_token_sources_empty_dir(tmp_path: Path) -> None: - """detect_token_sources returns empty list when no credential files exist.""" - from openjarvis.cli.deep_research_setup_cmd import detect_token_sources - - connectors_dir = tmp_path / "connectors" - connectors_dir.mkdir() - - sources = detect_token_sources(connectors_dir=connectors_dir) - assert len(sources) == 0 -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/cli/test_deep_research_setup.py::test_detect_token_sources_finds_connected -v --tb=short -``` - -Expected: FAIL — `ImportError: cannot import name 'detect_token_sources'` - -- [ ] **Step 3: Add `detect_token_sources` and `_TOKEN_SOURCES` to deep_research_setup_cmd.py** - -Add after the `detect_local_sources` function (around line 80): - -```python -# --------------------------------------------------------------------------- -# Token-based source detection -# --------------------------------------------------------------------------- - -_TOKEN_SOURCES = [ - { - "connector_id": "gmail_imap", - "display_name": "Gmail (IMAP)", - "creds_file": "gmail_imap.json", - "prompt_label": "email:app_password", - }, - { - "connector_id": "outlook", - "display_name": "Outlook / Microsoft 365", - "creds_file": "outlook.json", - "prompt_label": "email:app_password", - }, - { - "connector_id": "slack", - "display_name": "Slack", - "creds_file": "slack.json", - "prompt_label": "Bot token (xoxb-... or xoxe-...)", - }, - { - "connector_id": "notion", - "display_name": "Notion", - "creds_file": "notion.json", - "prompt_label": "Integration token (ntn_...)", - }, - { - "connector_id": "granola", - "display_name": "Granola", - "creds_file": "granola.json", - "prompt_label": "API key (grn_...)", - }, -] - - -def detect_token_sources( - *, - connectors_dir: Optional[Path] = None, -) -> List[Dict[str, Any]]: - """Return token-based sources that already have valid credentials. - - Scans ``~/.openjarvis/connectors/`` for known credential files. - """ - cdir = connectors_dir or (DEFAULT_CONFIG_DIR / "connectors") - sources: List[Dict[str, Any]] = [] - - for ts in _TOKEN_SOURCES: - creds_file = cdir / ts["creds_file"] - if not creds_file.exists(): - continue - try: - data = json.loads(creds_file.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): - continue - # Must have at least one non-empty value - if not data or not any(v for v in data.values() if v): - continue - sources.append({ - "connector_id": ts["connector_id"], - "display_name": ts["display_name"], - "config": {}, - }) - - return sources -``` - -Also add `import json` at the top of the file (add to existing imports section). - -- [ ] **Step 4: Run token detection tests** - -```bash -uv run pytest tests/cli/test_deep_research_setup.py -v --tb=short -``` - -Expected: All 7 tests PASS (4 old + 3 new). - -- [ ] **Step 5: Add `_prompt_connect_sources` function and update `_instantiate_connector`** - -Add the interactive connect function after `detect_token_sources`: - -```python -def _prompt_connect_sources(console: Console) -> List[Dict[str, Any]]: - """Interactively prompt the user to connect token-based sources.""" - connected: List[Dict[str, Any]] = [] - cdir = DEFAULT_CONFIG_DIR / "connectors" - cdir.mkdir(parents=True, exist_ok=True) - - while True: - # Show unconnected sources - unconnected = [ - ts for ts in _TOKEN_SOURCES - if not (cdir / ts["creds_file"]).exists() - ] - if not unconnected: - console.print("[dim]All token sources already connected.[/dim]") - break - - if not click.confirm("Connect additional sources?", default=False): - break - - names = [ts["connector_id"] for ts in unconnected] - labels = [f"{ts['display_name']} ({ts['connector_id']})" for ts in unconnected] - console.print("Available:") - for label in labels: - console.print(f" {label}") - - choice = click.prompt( - "Which source?", - type=click.Choice(names, case_sensitive=False), - ) - - ts = next(t for t in unconnected if t["connector_id"] == choice) - token = click.prompt(f"Paste your {ts['prompt_label']}") - - # Save credentials via connector's handle_callback - connector = _instantiate_connector(choice, {}) - connector.handle_callback(token.strip()) - console.print(f" [green]{ts['display_name']}: connected![/green]") - - connected.append({ - "connector_id": choice, - "display_name": ts["display_name"], - "config": {}, - }) - - return connected -``` - -Update `_instantiate_connector` to handle all connector types: - -```python -def _instantiate_connector(connector_id: str, config: Dict[str, Any]) -> Any: - """Lazily import and instantiate a connector by ID.""" - if connector_id == "apple_notes": - from openjarvis.connectors.apple_notes import AppleNotesConnector - return AppleNotesConnector(db_path=config.get("db_path", "")) - elif connector_id == "imessage": - from openjarvis.connectors.imessage import IMessageConnector - return IMessageConnector(db_path=config.get("db_path", "")) - elif connector_id == "obsidian": - from openjarvis.connectors.obsidian import ObsidianConnector - return ObsidianConnector(vault_path=config.get("vault_path", "")) - elif connector_id == "gmail_imap": - from openjarvis.connectors.gmail_imap import GmailIMAPConnector - return GmailIMAPConnector() - elif connector_id == "outlook": - from openjarvis.connectors.outlook import OutlookConnector - return OutlookConnector() - elif connector_id == "slack": - from openjarvis.connectors.slack_connector import SlackConnector - return SlackConnector() - elif connector_id == "notion": - from openjarvis.connectors.notion import NotionConnector - return NotionConnector() - elif connector_id == "granola": - from openjarvis.connectors.granola import GranolaConnector - return GranolaConnector() - else: - msg = f"Unknown connector: {connector_id}" - raise ValueError(msg) -``` - -- [ ] **Step 6: Update the `deep_research_setup` Click command to use token sources** - -Replace the body of the `deep_research_setup` function with: - -```python -@click.command("deep-research-setup") -@click.option( - "--obsidian-vault", - type=click.Path(exists=True, file_okay=False), - default=None, - help="Path to an Obsidian vault directory.", -) -@click.option("--skip-chat", is_flag=True, help="Ingest only, don't launch chat.") -def deep_research_setup(obsidian_vault: Optional[str], skip_chat: bool) -> None: - """Auto-detect local data sources, ingest, and launch Deep Research chat.""" - console = Console() - console.print("\n[bold]Deep Research Setup[/bold]\n") - - # 1. Detect local sources - vault_path = Path(obsidian_vault) if obsidian_vault else None - local_sources = detect_local_sources(obsidian_vault_path=vault_path) - - # 2. Detect already-connected token sources - token_sources = detect_token_sources() - - all_sources = local_sources + token_sources - - # 3. Show what we found - if all_sources: - table = Table(title="Detected Sources") - table.add_column("Source", style="bold") - table.add_column("Type", style="dim") - table.add_column("Status", style="green") - for src in local_sources: - table.add_row(src["display_name"], "local", "ready") - for src in token_sources: - table.add_row(src["display_name"], "token", "connected") - console.print(table) - console.print() - - # 4. Offer to connect new token sources - newly_connected = _prompt_connect_sources(console) - all_sources.extend(newly_connected) - - if not all_sources: - console.print( - "[yellow]No data sources detected or connected.[/yellow]\n" - "On macOS, ensure Full Disk Access is granted in " - "System Settings > Privacy & Security." - ) - sys.exit(1) - - # 5. Confirm and ingest - if not click.confirm("Ingest these sources?", default=True): - sys.exit(0) - - db_path = DEFAULT_CONFIG_DIR / "knowledge.db" - db_path.parent.mkdir(parents=True, exist_ok=True) - store = KnowledgeStore(str(db_path)) - - console.print("\n[bold]Ingesting...[/bold]") - for src in all_sources: - try: - connector = _instantiate_connector( - src["connector_id"], src["config"], - ) - pipeline = IngestionPipeline(store) - engine = SyncEngine(pipeline) - chunks = engine.sync(connector) - console.print( - f" {src['display_name']}: [green]{chunks} chunks[/green]" - ) - except Exception as exc: # noqa: BLE001 - console.print( - f" {src['display_name']}: [red]error: {exc}[/red]" - ) - - total = store.count() - console.print( - f"\n[bold green]Done![/bold green] {total} total chunks in {db_path}\n" - ) - - # 6. Chat - if skip_chat: - return - - _launch_chat(store, console) -``` - -- [ ] **Step 7: Run all tests** - -```bash -uv run pytest tests/cli/test_deep_research_setup.py tests/connectors/test_outlook.py -v --tb=short -``` - -Expected: All tests PASS. - -- [ ] **Step 8: Lint** - -```bash -uv run ruff check src/openjarvis/cli/deep_research_setup_cmd.py src/openjarvis/connectors/outlook.py src/openjarvis/connectors/gmail_imap.py -``` - -Expected: No errors. - -- [ ] **Step 9: Commit** - -```bash -git add src/openjarvis/cli/deep_research_setup_cmd.py tests/cli/test_deep_research_setup.py -git commit -m "feat: extend deep-research-setup with token source detection and interactive connect" -``` - ---- - -### Task 4: Live test — connect all sources and ingest - -**Files:** None (manual testing) - -- [ ] **Step 1: Connect Slack** - -```bash -uv run jarvis connect slack -``` - -Paste: your Slack bot token (`xoxb-...` or `xoxe-...`) - -- [ ] **Step 2: Connect Notion** - -```bash -uv run jarvis connect notion -``` - -Paste: your Notion integration token (`ntn_...`) - -- [ ] **Step 3: Connect Granola** - -```bash -uv run jarvis connect granola -``` - -Paste: your Granola API key (`grn_...`) - -- [ ] **Step 4: Connect Gmail IMAP** - -```bash -uv run jarvis connect gmail_imap -``` - -Paste: `your-email@gmail.com:qpde kebj evhy zljc` - -(Replace `your-email@gmail.com` with your actual Gmail address.) - -- [ ] **Step 5: Run deep-research-setup and verify all sources ingest** - -```bash -uv run jarvis deep-research-setup --skip-chat -``` - -Expected output: -``` -Deep Research Setup - - Detected Sources -┌───────────────────────┬───────┬───────────┐ -│ Source │ Type │ Status │ -├───────────────────────┼───────┼───────────┤ -│ Apple Notes │ local │ ready │ -│ iMessage │ local │ ready │ -│ Gmail (IMAP) │ token │ connected │ -│ Slack │ token │ connected │ -│ Notion │ token │ connected │ -│ Granola │ token │ connected │ -└───────────────────────┴───────┴───────────┘ - -Ingest these sources? [Y/n]: Y - -Ingesting... - Apple Notes: ~100 chunks - iMessage: ~52000 chunks - Gmail (IMAP): N chunks - Slack: N chunks - Notion: N chunks - Granola: N chunks - -Done! N total chunks in ~/.openjarvis/knowledge.db -``` - -- [ ] **Step 6: Test the research agent with cross-source queries** - -```bash -uv run jarvis deep-research-setup -``` - -At the `research>` prompt, try: -1. A query that should span emails + messages -2. A query about meeting notes (Granola data) -3. A query about a Notion page - -- [ ] **Step 7: Push** - -```bash -git push origin feat/deep-research-setup -``` diff --git a/docs/superpowers/plans/2026-03-27-channel-gateway-deep-research.md b/docs/superpowers/plans/2026-03-27-channel-gateway-deep-research.md deleted file mode 100644 index 2aaee5ad..00000000 --- a/docs/superpowers/plans/2026-03-27-channel-gateway-deep-research.md +++ /dev/null @@ -1,722 +0,0 @@ -# Channel Gateway → DeepResearch Wiring Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Wire the ChannelBridge (from PR #78, now merged) to route incoming messages from Slack, iMessage/BlueBubbles, WhatsApp, and SMS to the DeepResearchAgent, and add an AppleScript-based iMessage daemon for direct iPhone-to-agent messaging without external services. - -**Architecture:** Modify `ChannelBridge.handle_incoming()` to use DeepResearchAgent instead of generic `JarvisSystem.ask()`. Add an `IMessageDaemon` that polls `chat.db` for new messages and routes them through the bridge. Add `jarvis channels` CLI commands for setup and lifecycle. - -**Tech Stack:** Python 3.10+, sqlite3, subprocess (AppleScript), Click, asyncio, pytest - ---- - -### Task 1: Wire ChannelBridge to DeepResearchAgent - -**Files:** -- Modify: `src/openjarvis/server/channel_bridge.py` -- Create: `tests/server/test_channel_bridge_deep_research.py` - -- [ ] **Step 1: Read `channel_bridge.py` to find `_handle_chat` method** - -The `_handle_chat` method currently calls `self._system.ask()`. We need it to use DeepResearchAgent when available. - -- [ ] **Step 2: Write the test** - -Create `tests/server/test_channel_bridge_deep_research.py`: - -```python -"""Test ChannelBridge routes to DeepResearchAgent.""" - -from __future__ import annotations - -from unittest.mock import MagicMock, patch - -import pytest - - -def test_handle_chat_uses_deep_research_agent() -> None: - """When a DeepResearch agent is configured, route through it.""" - from openjarvis.server.channel_bridge import ChannelBridge - from openjarvis.server.session_store import SessionStore - - mock_agent = MagicMock() - mock_agent.run.return_value = MagicMock( - content="Found 3 results about Spain.", - ) - - bridge = ChannelBridge( - channels={}, - session_store=SessionStore(db_path=":memory:"), - bus=MagicMock(), - deep_research_agent=mock_agent, - ) - - result = bridge.handle_incoming( - sender_id="+15551234567", - content="When was my last trip to Spain?", - channel_type="twilio", - ) - - assert "Spain" in result - mock_agent.run.assert_called_once() - - -def test_handle_chat_falls_back_to_system() -> None: - """When no DeepResearch agent, fall back to system.ask().""" - from openjarvis.server.channel_bridge import ChannelBridge - from openjarvis.server.session_store import SessionStore - - mock_system = MagicMock() - mock_system.ask.return_value = "Generic response" - - bridge = ChannelBridge( - channels={}, - session_store=SessionStore(db_path=":memory:"), - bus=MagicMock(), - system=mock_system, - ) - - result = bridge.handle_incoming( - sender_id="+15551234567", - content="Hello", - channel_type="twilio", - ) - - assert result == "Generic response" - mock_system.ask.assert_called_once() -``` - -- [ ] **Step 3: Modify `ChannelBridge.__init__` to accept `deep_research_agent`** - -In `channel_bridge.py`, add `deep_research_agent` parameter to `__init__`: - -```python -def __init__( - self, - channels: Dict[str, BaseChannel], - session_store: SessionStore, - bus: EventBus, - system: Any = None, - agent_manager: Any = None, - deep_research_agent: Any = None, -) -> None: - self._channels = channels - self._session_store = session_store - self._bus = bus - self._system = system - self._agent_manager = agent_manager - self._deep_research_agent = deep_research_agent - self._notification_timestamps: Dict[str, float] = {} - self._subscribe_notifications() -``` - -- [ ] **Step 4: Modify `_handle_chat` to use DeepResearchAgent first** - -Find the `_handle_chat` method. Replace the body to try DeepResearchAgent first, fall back to system: - -```python -def _handle_chat( - self, - sender_id: str, - content: str, - channel_type: str, - max_length: int = _DEFAULT_MAX_LENGTH, -) -> str: - """Route chat to DeepResearchAgent or JarvisSystem.""" - # Try DeepResearchAgent first - if self._deep_research_agent is not None: - try: - result = self._deep_research_agent.run(content) - response = result.content or "No results found." - except Exception as exc: - logger.error("DeepResearch agent failed: %s", exc) - response = f"Research error: {exc}" - elif self._system is not None: - response = self._system.ask(content) - else: - response = "No agent or system configured." - - # Truncate if needed - if len(response) > max_length: - self._session_store.set_overflow(sender_id, response) - response = response[:max_length] + "\n\n(truncated — send /more for the rest)" - - return response -``` - -- [ ] **Step 5: Run tests** - -```bash -uv run pytest tests/server/test_channel_bridge_deep_research.py tests/server/test_channel_bridge.py -v --tb=short -``` - -Expected: All PASS (new tests + existing tests). - -- [ ] **Step 6: Commit** - -```bash -git add src/openjarvis/server/channel_bridge.py tests/server/test_channel_bridge_deep_research.py -git commit -m "feat: wire ChannelBridge to route messages to DeepResearchAgent" -``` - ---- - -### Task 2: Create iMessage AppleScript daemon - -**Files:** -- Create: `src/openjarvis/channels/imessage_daemon.py` -- Create: `tests/channels/test_imessage_daemon.py` - -- [ ] **Step 1: Write the test** - -Create `tests/channels/test_imessage_daemon.py`: - -```python -"""Tests for iMessage AppleScript daemon.""" - -from __future__ import annotations - -import sqlite3 -from pathlib import Path -from unittest.mock import MagicMock, patch - -import pytest - - -def _create_fake_chat_db(db_path: Path) -> None: - """Create a minimal chat.db with one message.""" - conn = sqlite3.connect(str(db_path)) - conn.executescript(""" - CREATE TABLE handle (ROWID INTEGER PRIMARY KEY, id TEXT); - CREATE TABLE chat ( - ROWID INTEGER PRIMARY KEY, - chat_identifier TEXT, display_name TEXT - ); - CREATE TABLE chat_message_join (chat_id INTEGER, message_id INTEGER); - CREATE TABLE message ( - ROWID INTEGER PRIMARY KEY, text TEXT, handle_id INTEGER, - date INTEGER, is_from_me INTEGER - ); - """) - conn.execute("INSERT INTO handle VALUES (1, '+15551234567')") - conn.execute("INSERT INTO chat VALUES (1, '+15551234567', 'Test Chat')") - conn.execute("INSERT INTO chat_message_join VALUES (1, 1)") - conn.execute( - "INSERT INTO message VALUES (1, 'Hello agent', 1, 700000000000000000, 0)" - ) - conn.commit() - conn.close() - - -def test_poll_new_messages(tmp_path: Path) -> None: - """Daemon detects new messages after last_rowid.""" - from openjarvis.channels.imessage_daemon import poll_new_messages - - db_path = tmp_path / "chat.db" - _create_fake_chat_db(db_path) - - messages = poll_new_messages( - db_path=str(db_path), - last_rowid=0, - chat_identifier="+15551234567", - ) - assert len(messages) == 1 - assert messages[0]["text"] == "Hello agent" - assert messages[0]["rowid"] == 1 - - -def test_poll_skips_old_messages(tmp_path: Path) -> None: - """Daemon skips messages with ROWID <= last_rowid.""" - from openjarvis.channels.imessage_daemon import poll_new_messages - - db_path = tmp_path / "chat.db" - _create_fake_chat_db(db_path) - - messages = poll_new_messages( - db_path=str(db_path), - last_rowid=1, - chat_identifier="+15551234567", - ) - assert len(messages) == 0 - - -def test_poll_filters_by_chat(tmp_path: Path) -> None: - """Daemon only returns messages from the designated chat.""" - from openjarvis.channels.imessage_daemon import poll_new_messages - - db_path = tmp_path / "chat.db" - _create_fake_chat_db(db_path) - - messages = poll_new_messages( - db_path=str(db_path), - last_rowid=0, - chat_identifier="+15559999999", - ) - assert len(messages) == 0 - - -def test_poll_skips_own_messages(tmp_path: Path) -> None: - """Daemon ignores messages sent by the user (is_from_me=1).""" - from openjarvis.channels.imessage_daemon import poll_new_messages - - db_path = tmp_path / "chat.db" - conn = sqlite3.connect(str(db_path)) - conn.executescript(""" - CREATE TABLE handle (ROWID INTEGER PRIMARY KEY, id TEXT); - CREATE TABLE chat ( - ROWID INTEGER PRIMARY KEY, - chat_identifier TEXT, display_name TEXT - ); - CREATE TABLE chat_message_join (chat_id INTEGER, message_id INTEGER); - CREATE TABLE message ( - ROWID INTEGER PRIMARY KEY, text TEXT, handle_id INTEGER, - date INTEGER, is_from_me INTEGER - ); - """) - conn.execute("INSERT INTO handle VALUES (1, '+15551234567')") - conn.execute("INSERT INTO chat VALUES (1, '+15551234567', 'Test')") - conn.execute("INSERT INTO chat_message_join VALUES (1, 1)") - conn.execute( - "INSERT INTO message VALUES (1, 'My own msg', 1, 700000000000000000, 1)" - ) - conn.commit() - conn.close() - - messages = poll_new_messages( - db_path=str(db_path), - last_rowid=0, - chat_identifier="+15551234567", - ) - assert len(messages) == 0 -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/channels/test_imessage_daemon.py -v --tb=short -``` - -Expected: FAIL — `ModuleNotFoundError` - -- [ ] **Step 3: Create the daemon module** - -Create `src/openjarvis/channels/imessage_daemon.py`: - -```python -"""iMessage daemon — polls chat.db and routes to DeepResearchAgent. - -Monitors a designated iMessage conversation for new messages, routes -them to the agent, and sends responses back via AppleScript. - -Requires macOS with Full Disk Access for chat.db reading and -Accessibility permission for AppleScript Messages control. -""" - -from __future__ import annotations - -import logging -import os -import signal -import sqlite3 -import subprocess -import time -from pathlib import Path -from typing import Any, Dict, List, Optional - -logger = logging.getLogger(__name__) - -_DEFAULT_DB_PATH = str( - Path.home() / "Library" / "Messages" / "chat.db" -) -_POLL_INTERVAL = 5 # seconds -_PID_FILE = str(Path.home() / ".openjarvis" / "imessage-agent.pid") - - -# ------------------------------------------------------------------- -# Polling -# ------------------------------------------------------------------- - - -def poll_new_messages( - *, - db_path: str = _DEFAULT_DB_PATH, - last_rowid: int = 0, - chat_identifier: str = "", -) -> List[Dict[str, Any]]: - """Return new incoming messages since last_rowid. - - Only returns messages where ``is_from_me = 0`` (incoming) and - that belong to the designated chat. - """ - try: - conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) - conn.row_factory = sqlite3.Row - except sqlite3.OperationalError: - return [] - - try: - rows = conn.execute( - "SELECT m.ROWID as rowid, m.text, m.date, c.chat_identifier " - "FROM message m " - "JOIN chat_message_join cmj ON cmj.message_id = m.ROWID " - "JOIN chat c ON c.ROWID = cmj.chat_id " - "WHERE m.ROWID > ? AND m.is_from_me = 0 AND m.text IS NOT NULL " - "AND c.chat_identifier = ? " - "ORDER BY m.ROWID ASC", - (last_rowid, chat_identifier), - ).fetchall() - - return [dict(row) for row in rows] - finally: - conn.close() - - -# ------------------------------------------------------------------- -# AppleScript sending -# ------------------------------------------------------------------- - - -def send_imessage(chat_identifier: str, message: str) -> bool: - """Send an iMessage via AppleScript. - - Returns True on success, False on failure. - """ - # Escape double quotes and backslashes for AppleScript - escaped = message.replace("\\", "\\\\").replace('"', '\\"') - script = ( - f'tell application "Messages"\n' - f' set targetChat to a reference to chat id "{chat_identifier}"\n' - f' send "{escaped}" to targetChat\n' - f"end tell" - ) - - try: - subprocess.run( - ["osascript", "-e", script], - capture_output=True, - text=True, - timeout=30, - ) - return True - except (subprocess.TimeoutExpired, FileNotFoundError): - logger.error("Failed to send iMessage via AppleScript") - return False - - -# ------------------------------------------------------------------- -# Daemon loop -# ------------------------------------------------------------------- - - -def run_daemon( - *, - chat_identifier: str, - db_path: str = _DEFAULT_DB_PATH, - handler: Any = None, - poll_interval: float = _POLL_INTERVAL, - max_iterations: int = 0, -) -> None: - """Run the iMessage polling daemon. - - Parameters - ---------- - chat_identifier: - The chat to monitor (phone number or email). - handler: - Callable that takes a message string and returns a response string. - If None, messages are logged but not responded to. - poll_interval: - Seconds between polls. - max_iterations: - Stop after this many iterations (0 = run forever). - """ - # Write PID file - pid_path = Path(_PID_FILE) - pid_path.parent.mkdir(parents=True, exist_ok=True) - pid_path.write_text(str(os.getpid())) - - # Find starting ROWID - last_rowid = _get_max_rowid(db_path) - logger.info( - "iMessage daemon started — monitoring %s from ROWID %d", - chat_identifier, - last_rowid, - ) - - running = True - - def _stop(signum: int, frame: Any) -> None: - nonlocal running - running = False - - signal.signal(signal.SIGTERM, _stop) - signal.signal(signal.SIGINT, _stop) - - iterations = 0 - while running: - messages = poll_new_messages( - db_path=db_path, - last_rowid=last_rowid, - chat_identifier=chat_identifier, - ) - - for msg in messages: - last_rowid = msg["rowid"] - text = msg["text"] - logger.info("Received: %s", text[:100]) - - if handler is not None: - try: - response = handler(text) - if response: - send_imessage(chat_identifier, response) - except Exception: - logger.exception("Handler failed for message %d", msg["rowid"]) - - iterations += 1 - if max_iterations and iterations >= max_iterations: - break - - time.sleep(poll_interval) - - # Cleanup PID file - if pid_path.exists(): - pid_path.unlink() - logger.info("iMessage daemon stopped") - - -def _get_max_rowid(db_path: str) -> int: - """Get the current max ROWID from chat.db.""" - try: - conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) - row = conn.execute("SELECT MAX(ROWID) FROM message").fetchone() - conn.close() - return row[0] or 0 - except sqlite3.OperationalError: - return 0 - - -def is_running() -> bool: - """Check if the daemon is currently running.""" - pid_path = Path(_PID_FILE) - if not pid_path.exists(): - return False - try: - pid = int(pid_path.read_text().strip()) - os.kill(pid, 0) - return True - except (ValueError, ProcessLookupError, PermissionError): - pid_path.unlink(missing_ok=True) - return False - - -def stop_daemon() -> bool: - """Stop the running daemon. Returns True if stopped.""" - pid_path = Path(_PID_FILE) - if not pid_path.exists(): - return False - try: - pid = int(pid_path.read_text().strip()) - os.kill(pid, signal.SIGTERM) - pid_path.unlink(missing_ok=True) - return True - except (ValueError, ProcessLookupError, PermissionError): - pid_path.unlink(missing_ok=True) - return False -``` - -- [ ] **Step 4: Run tests** - -```bash -uv run pytest tests/channels/test_imessage_daemon.py -v --tb=short -``` - -Expected: 4/4 PASS. - -- [ ] **Step 5: Lint + commit** - -```bash -uv run ruff check src/openjarvis/channels/imessage_daemon.py -git add src/openjarvis/channels/imessage_daemon.py tests/channels/test_imessage_daemon.py -git commit -m "feat: add iMessage AppleScript daemon for iPhone-to-agent messaging" -``` - ---- - -### Task 3: Add `jarvis channels` CLI commands - -**Files:** -- Create: `src/openjarvis/cli/channels_cmd.py` -- Modify: `src/openjarvis/cli/__init__.py` - -- [ ] **Step 1: Create the CLI command module** - -Create `src/openjarvis/cli/channels_cmd.py`: - -```python -"""``jarvis channels`` — manage channel connections for the DeepResearch agent.""" - -from __future__ import annotations - -import sys -from typing import Optional - -import click -from rich.console import Console -from rich.table import Table - - -@click.group("channels") -def channels() -> None: - """Manage messaging channels (iMessage, Slack, WhatsApp, SMS).""" - - -@channels.command("status") -def channels_status() -> None: - """Show status of all configured channels.""" - from openjarvis.channels.imessage_daemon import is_running - - console = Console() - table = Table(title="Channel Status") - table.add_column("Channel", style="bold") - table.add_column("Status") - table.add_column("Details", style="dim") - - # iMessage daemon - if is_running(): - table.add_row("iMessage", "[green]running[/green]", "Polling chat.db") - else: - table.add_row("iMessage", "[dim]stopped[/dim]", "jarvis channels imessage-start") - - console.print(table) - - -@channels.command("imessage-start") -@click.argument("chat_identifier") -@click.option("--background/--foreground", default=True, help="Run in background.") -def imessage_start(chat_identifier: str, background: bool) -> None: - """Start the iMessage daemon for a chat. - - CHAT_IDENTIFIER is the phone number or email to monitor - (e.g. +15551234567 or group chat name). - """ - from openjarvis.channels.imessage_daemon import is_running, run_daemon - - console = Console() - - if is_running(): - console.print("[yellow]iMessage daemon is already running.[/yellow]") - return - - if background: - import subprocess - proc = subprocess.Popen( - [ - sys.executable, "-m", "openjarvis.channels.imessage_daemon", - "--chat", chat_identifier, - ], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - start_new_session=True, - ) - console.print( - f"[green]iMessage daemon started[/green] (PID {proc.pid})\n" - f"Monitoring: {chat_identifier}\n" - f"Text this contact from your iPhone to chat with the agent." - ) - else: - console.print(f"[green]Starting iMessage daemon[/green] — monitoring {chat_identifier}") - console.print("Press Ctrl+C to stop.\n") - - from openjarvis.agents.deep_research import DeepResearchAgent - from openjarvis.connectors.retriever import TwoStageRetriever - from openjarvis.connectors.store import KnowledgeStore - from openjarvis.engine.ollama import OllamaEngine - from openjarvis.tools.knowledge_search import KnowledgeSearchTool - from openjarvis.tools.knowledge_sql import KnowledgeSQLTool - from openjarvis.tools.scan_chunks import ScanChunksTool - from openjarvis.tools.think import ThinkTool - - engine = OllamaEngine() - store = KnowledgeStore() - retriever = TwoStageRetriever(store) - tools = [ - KnowledgeSearchTool(retriever=retriever), - KnowledgeSQLTool(store=store), - ScanChunksTool(store=store, engine=engine, model="qwen3.5:4b"), - ThinkTool(), - ] - agent = DeepResearchAgent(engine=engine, model="qwen3.5:4b", tools=tools) - - def handler(text: str) -> str: - result = agent.run(text) - return result.content or "No results found." - - run_daemon( - chat_identifier=chat_identifier, - handler=handler, - ) - - -@channels.command("imessage-stop") -def imessage_stop() -> None: - """Stop the iMessage daemon.""" - from openjarvis.channels.imessage_daemon import stop_daemon - - console = Console() - if stop_daemon(): - console.print("[green]iMessage daemon stopped.[/green]") - else: - console.print("[dim]iMessage daemon is not running.[/dim]") -``` - -- [ ] **Step 2: Register in CLI __init__.py** - -In `src/openjarvis/cli/__init__.py`, add: - -```python -from openjarvis.cli.channels_cmd import channels -cli.add_command(channels, "channels") -``` - -- [ ] **Step 3: Verify** - -```bash -uv run jarvis channels --help -uv run jarvis channels status -``` - -- [ ] **Step 4: Lint + commit** - -```bash -uv run ruff check src/openjarvis/cli/channels_cmd.py -git add src/openjarvis/cli/channels_cmd.py src/openjarvis/cli/__init__.py -git commit -m "feat: add jarvis channels CLI commands for iMessage daemon lifecycle" -``` - ---- - -### Task 4: Run full test suite + push - -**Files:** None - -- [ ] **Step 1: Run all tests** - -```bash -uv run pytest tests/connectors/ tests/agents/test_deep_research.py tests/agents/test_deep_research_integration.py tests/agents/test_channel_agent.py tests/agents/test_channel_agent_integration.py tests/cli/test_deep_research_setup.py tests/tools/test_knowledge_sql.py tests/tools/test_scan_chunks.py tests/server/test_channel_bridge.py tests/server/test_channel_bridge_deep_research.py tests/server/test_session_store.py tests/server/test_webhook_routes.py tests/server/test_auth_middleware.py tests/server/test_deep_research_tools_wiring.py tests/channels/test_twilio_sms.py tests/channels/test_imessage_daemon.py --ignore=tests/connectors/test_embedding_store.py -k "not cached_embeddings and not caches_new_embeddings" -q --tb=short -``` - -Expected: All PASS. - -- [ ] **Step 2: Lint** - -```bash -uv run ruff check src/openjarvis/channels/imessage_daemon.py src/openjarvis/server/channel_bridge.py src/openjarvis/cli/channels_cmd.py -``` - -- [ ] **Step 3: Push to both branches** - -```bash -git push origin feat/deep-research-setup -git push origin feat/deep-research-setup:feat/mobile-channel-gateway --force-with-lease -``` diff --git a/docs/superpowers/plans/2026-03-27-channels-and-messaging-tabs-v2.md b/docs/superpowers/plans/2026-03-27-channels-and-messaging-tabs-v2.md deleted file mode 100644 index e66a6af4..00000000 --- a/docs/superpowers/plans/2026-03-27-channels-and-messaging-tabs-v2.md +++ /dev/null @@ -1,672 +0,0 @@ -# Channels + Messaging Tabs Implementation Plan (v2) - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Replace the existing Channels tab with two tabs — **Channels** (data sources with chunk counts and inline "+ Add" setup) and **Messaging** (clear "Text this number from your iPhone" instructions) — on every agent detail page. - -**Architecture:** Refactor the existing `ChannelsTab` component in `AgentsPage.tsx` into two separate components. The Channels tab calls existing connector API endpoints + a new chunk-count endpoint. The Messaging tab reuses the existing channel binding API. The `StepByStepPanel` from `SourceConnectFlow.tsx` is extracted for reuse in the Channels tab inline setup. - -**Tech Stack:** React 19, TypeScript, existing FastAPI endpoints - ---- - -### Task 1: Add connector chunk counts to backend API - -**Files:** -- Modify: `src/openjarvis/server/connectors_router.py` - -The Channels tab needs to show chunk counts per source. Add a `chunks` field to the connector summary by querying the KnowledgeStore. - -- [ ] **Step 1: Modify `_connector_summary` to include chunk count** - -In `src/openjarvis/server/connectors_router.py`, find the `_connector_summary` function (line ~102). Add a chunk count from the KnowledgeStore: - -```python -def _connector_summary(connector_id: str, instance: Any) -> Dict[str, Any]: - """Build the dict returned by GET /connectors.""" - chunks = 0 - try: - from openjarvis.connectors.store import KnowledgeStore - store = KnowledgeStore() - rows = store._conn.execute( - "SELECT COUNT(*) FROM knowledge_chunks WHERE source = ?", - (connector_id,), - ).fetchone() - chunks = rows[0] if rows else 0 - except Exception: - pass - - return { - "connector_id": connector_id, - "display_name": getattr(instance, "display_name", connector_id), - "auth_type": getattr(instance, "auth_type", "unknown"), - "connected": instance.is_connected(), - "chunks": chunks, - } -``` - -- [ ] **Step 2: Verify endpoint returns chunks** - -```bash -uv run python3 -c " -import httpx -r = httpx.get('http://127.0.0.1:8222/connectors', timeout=5) -for c in r.json().get('connectors', [])[:5]: - print(f'{c[\"connector_id\"]}: connected={c[\"connected\"]}, chunks={c.get(\"chunks\", \"?\")}') -" -``` - -- [ ] **Step 3: Lint + commit** - -```bash -uv run ruff check src/openjarvis/server/connectors_router.py -git add src/openjarvis/server/connectors_router.py -git commit -m "feat: include chunk counts in connector list API response" -``` - ---- - -### Task 2: Rewrite Channels tab as data sources view - -**Files:** -- Modify: `frontend/src/pages/AgentsPage.tsx` -- Modify: `frontend/src/lib/connectors-api.ts` - -Replace the existing `ChannelsTab` (messaging channels) with a new `ChannelsTab` (data sources). - -- [ ] **Step 1: Update ConnectorInfo type in connectors-api.ts to include chunks** - -In `frontend/src/lib/connectors-api.ts`, find the `ConnectorInfo` type (or wherever it's defined) and add `chunks`: - -If there's an interface in `types/connectors.ts`: -```typescript -export interface ConnectorInfo { - connector_id: string; - display_name: string; - auth_type: 'oauth' | 'local' | 'bridge' | 'filesystem'; - connected: boolean; - auth_url?: string; - mcp_tools?: string[]; - chunks?: number; // ADD THIS -} -``` - -- [ ] **Step 2: Replace the ChannelsTab component** - -In `AgentsPage.tsx`, find the existing `AVAILABLE_CHANNELS` array and `ChannelsTab` component (around lines 1000-1170). Replace the entire block with a new `ChannelsTab` that shows data sources: - -```typescript -import { SOURCE_CATALOG, ConnectorMeta, ConnectRequest } from '../types/connectors'; -import { listConnectors, connectSource } from '../lib/connectors-api'; - -function ChannelsTab({ agentId }: { agentId: string }) { - const [connectors, setConnectors] = useState< - Array<{ connector_id: string; display_name: string; connected: boolean; chunks: number }> - >([]); - const [expandedId, setExpandedId] = useState(null); - const [loading, setLoading] = useState(false); - - const loadConnectors = useCallback(() => { - listConnectors() - .then((list) => - setConnectors( - list.map((c) => ({ - connector_id: c.connector_id, - display_name: c.display_name, - connected: c.connected, - chunks: (c as any).chunks || 0, - })), - ), - ) - .catch(() => {}); - }, []); - - useEffect(() => { loadConnectors(); }, [loadConnectors]); - - const handleConnect = async (id: string, req: ConnectRequest) => { - setLoading(true); - try { - await connectSource(id, req); - setExpandedId(null); - loadConnectors(); - } catch { - // error handling - } finally { - setLoading(false); - } - }; - - const connected = connectors.filter((c) => c.connected); - const notConnected = connectors.filter((c) => !c.connected); - - // Merge with SOURCE_CATALOG for icons/descriptions - const getMeta = (id: string) => - SOURCE_CATALOG.find((s) => s.connector_id === id); - - const iconMap: Record = { - gmail: '✉️', gmail_imap: '✉️', slack: '#', - imessage: '💬', gdrive: '📁', notion: '📄', - obsidian: '📁', granola: '🎙️', gcalendar: '📅', - gcontacts: '📇', outlook: '✉️', apple_notes: '🍎', - dropbox: '📦', whatsapp: '📱', - }; - - return ( -
-
- Data sources your agent can search -
- - {/* Connected sources grid */} - {connected.length > 0 && ( -
- {connected.map((c) => ( -
- {iconMap[c.connector_id] || '🔗'} -
-
- {c.display_name} -
-
- {c.chunks.toLocaleString()} chunks -
-
- ✓ -
- ))} -
- )} - - {/* Not connected grid */} - {notConnected.length > 0 && ( -
- {notConnected.map((c) => { - const meta = getMeta(c.connector_id); - const isExpanded = expandedId === c.connector_id; - - return ( -
-
- setExpandedId(isExpanded ? null : c.connector_id) - } - > - {iconMap[c.connector_id] || '🔗'} -
-
- {c.display_name} -
-
- Not connected -
-
- - {isExpanded ? '✕ Close' : '+ Add'} - -
- - {/* Inline setup panel */} - {isExpanded && meta?.steps && ( -
- {meta.steps.map((step, i) => ( -
-
- STEP {i + 1} -
-
- {step.label} -
- {step.url && ( - - {step.urlLabel || 'Open'} → - - )} -
- ))} - {meta.inputFields && ( - - handleConnect(c.connector_id, req) - } - /> - )} -
- 🔒 Read-only access · No data leaves your device -
-
- )} -
- ); - })} -
- )} -
- ); -} - -function InlineConnectForm({ - fields, - loading, - onSubmit, -}: { - fields: Array<{ name: string; placeholder: string; type?: string }>; - loading: boolean; - onSubmit: (req: ConnectRequest) => void; -}) { - const [inputs, setInputs] = useState>({}); - - const update = (name: string, value: string) => - setInputs((p) => ({ ...p, [name]: value })); - - const allFilled = fields.every((f) => inputs[f.name]?.trim()); - - const submit = () => { - const req: ConnectRequest = {}; - for (const f of fields) { - if (f.name === 'email') req.email = inputs.email; - else if (f.name === 'password') req.password = inputs.password; - else if (f.name === 'token') req.token = inputs.token; - else if (f.name === 'path') req.path = inputs.path; - } - if (req.email && req.password) { - req.token = `${req.email}:${req.password}`; - req.code = req.token; - } - if (req.token && !req.code) req.code = req.token; - onSubmit(req); - }; - - return ( -
- {fields.map((f) => ( - update(f.name, e.target.value)} - placeholder={f.placeholder} - type={f.type || 'text'} - style={{ - width: '100%', padding: '7px 10px', - background: 'var(--color-bg)', - border: '1px solid var(--color-border)', - borderRadius: 4, color: 'var(--color-text)', - fontSize: 12, marginBottom: 6, - boxSizing: 'border-box', - }} - /> - ))} - -
- ); -} -``` - -- [ ] **Step 3: Update DETAIL_TABS to rename and add Messaging** - -Find `DETAIL_TABS` (line ~1553). Update to: - -```typescript -const DETAIL_TABS = [ - { id: 'overview', label: 'Overview', icon: Activity }, - { id: 'interact', label: 'Interact', icon: MessageSquare }, - { id: 'channels', label: 'Channels', icon: Database }, - { id: 'messaging', label: 'Messaging', icon: Wifi }, - { id: 'tasks', label: 'Tasks', icon: ListTodo }, - { id: 'memory', label: 'Memory', icon: Brain }, - { id: 'learning', label: 'Learning', icon: Settings }, - { id: 'logs', label: 'Logs', icon: FileText }, -] as const; -``` - -Add `Database` to lucide-react imports. Update detailTab type: - -```typescript -const [detailTab, setDetailTab] = useState< - 'overview' | 'interact' | 'channels' | 'messaging' | 'tasks' | 'memory' | 'learning' | 'logs' ->('overview'); -``` - -- [ ] **Step 4: Add MessagingTab component and wire both tabs** - -Add the `MessagingTab` component (reuses the old `AVAILABLE_CHANNELS` data but with clear UX): - -```typescript -const MESSAGING_CHANNELS = [ - { - type: 'imessage', name: 'iMessage', icon: '💬', - description: 'Text from your iPhone, iPad, or Mac', - activeTemplate: (id: string) => `Text ${id} from your iPhone`, - setupLabel: 'Phone number or email for the agent to use', - placeholder: '+15551234567', - }, - { - type: 'slack', name: 'Slack', icon: '#', - description: 'Message from any Slack workspace', - activeTemplate: (id: string) => `DM @jarvis in ${id}`, - setupLabel: 'Slack bot token (xoxb-...)', - placeholder: 'xoxb-...', - }, - { - type: 'whatsapp', name: 'WhatsApp', icon: '📱', - description: 'Message via WhatsApp', - activeTemplate: (id: string) => `Message ${id} on WhatsApp`, - setupLabel: 'WhatsApp access token', - placeholder: 'Access token', - }, - { - type: 'twilio', name: 'SMS (Twilio)', icon: '📨', - description: 'Text from any phone via Twilio', - activeTemplate: (id: string) => `Text ${id} from any phone`, - setupLabel: 'Twilio phone number', - placeholder: '+15551234567', - }, -]; - -function MessagingTab({ agentId }: { agentId: string }) { - const [bindings, setBindings] = useState([]); - const [setupType, setSetupType] = useState(null); - const [inputValue, setInputValue] = useState(''); - const [loading, setLoading] = useState(false); - - const loadBindings = useCallback(() => { - fetchAgentChannels(agentId).then(setBindings).catch(() => setBindings([])); - }, [agentId]); - - useEffect(() => { loadBindings(); }, [loadBindings]); - - const handleSetup = async (channelType: string) => { - if (!inputValue.trim()) return; - setLoading(true); - try { - await bindAgentChannel(agentId, channelType, { - identifier: inputValue.trim(), - }); - setSetupType(null); - setInputValue(''); - loadBindings(); - } catch { /* */ } finally { setLoading(false); } - }; - - const handleRemove = async (bindingId: string) => { - try { - await unbindAgentChannel(agentId, bindingId); - loadBindings(); - } catch { /* */ } - }; - - return ( -
-
- Talk to your agent from your phone or other platforms -
- - {MESSAGING_CHANNELS.map((ch) => { - const binding = bindings.find((b) => b.channel_type === ch.type); - const identifier = binding?.config?.identifier as string || binding?.session_id || ''; - const isSetup = setupType === ch.type; - - return ( -
-
- {ch.icon} -
-
{ch.name}
-
- {binding - ? ch.activeTemplate(identifier) - : 'Not set up'} -
-
- {binding ? ( -
- Active - -
- ) : ( - - )} -
- - {isSetup && ( -
-
{ch.setupLabel}
-
- setInputValue(e.target.value)} - placeholder={ch.placeholder} - style={{ - flex: 1, padding: '6px 10px', - background: 'var(--color-bg-secondary)', - border: '1px solid var(--color-border)', - borderRadius: 4, color: 'var(--color-text)', - fontSize: 12, - }} - onKeyDown={(e) => { - if (e.key === 'Enter') handleSetup(ch.type); - }} - /> - -
-
- )} -
- ); - })} -
- ); -} -``` - -Wire both tabs in the content rendering section: - -```typescript -{detailTab === 'channels' && ( - -)} -{detailTab === 'messaging' && ( - -)} -``` - -- [ ] **Step 5: Add required imports** - -At the top of `AgentsPage.tsx`: - -```typescript -import { Database } from 'lucide-react'; -import { SOURCE_CATALOG } from '../types/connectors'; -import type { ConnectRequest } from '../types/connectors'; -import { listConnectors, connectSource } from '../lib/connectors-api'; -``` - -- [ ] **Step 6: TypeScript check + build** - -```bash -cd frontend && npx tsc --noEmit 2>&1 | head -20 -cd frontend && npm run build 2>&1 | tail -10 -``` - -- [ ] **Step 7: Commit** - -```bash -git add frontend/src/pages/AgentsPage.tsx frontend/src/lib/connectors-api.ts frontend/src/types/connectors.ts -git commit -m "feat: replace Channels tab with data sources + add Messaging tab" -``` - ---- - -### Task 3: E2E test both tabs - -**Files:** None (manual testing) - -- [ ] **Step 1: Restart server to pick up new static files** - -```bash -# Kill existing server, rebuild frontend, restart -kill $(pgrep -f "jarvis serve") 2>/dev/null -cd frontend && npm run build -uv run jarvis serve --port 8222 --model qwen3.5:9b & -``` - -- [ ] **Step 2: Test Channels tab** - -1. Open http://127.0.0.1:8222 → Agents → open agent -2. Click "Channels" tab -3. Verify connected sources show with green border + chunk counts -4. Verify unconnected sources show with dashed border + "+ Add" -5. Click "+ Add" on an unconnected source → verify step-by-step instructions appear inline - -- [ ] **Step 3: Test Messaging tab** - -1. Click "Messaging" tab -2. Verify active channels show with "Text +1... from your iPhone" instructions -3. Verify inactive channels show "Set Up" button -4. Click "Set Up" → verify input field appears with clear label - -- [ ] **Step 4: Push** - -```bash -git push origin feat/deep-research-setup -``` diff --git a/docs/superpowers/plans/2026-03-27-channels-tab-and-connector-setup.md b/docs/superpowers/plans/2026-03-27-channels-tab-and-connector-setup.md deleted file mode 100644 index a210f3da..00000000 --- a/docs/superpowers/plans/2026-03-27-channels-tab-and-connector-setup.md +++ /dev/null @@ -1,1006 +0,0 @@ -# Channels Tab + Connector Setup Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add a Channels tab to every agent for bidirectional messaging (iMessage, Slack, WhatsApp, SMS), and build per-connector step-by-step setup guides with one-click OAuth in the setup wizard. - -**Architecture:** Sub-project 1 adds a React Channels tab component to AgentsPage.tsx that calls existing backend endpoints (POST/DELETE/GET channel bindings). Sub-project 2 replaces the generic OAuth panel in SourceConnectFlow.tsx with per-connector instruction panels and adds a Google OAuth callback server in Python. - -**Tech Stack:** React 19, TypeScript, Tailwind CSS, Python 3.10+, FastAPI, Click - ---- - -## Sub-project 1: Channels Tab - -### Task 1: Add Channels tab to agent detail view - -**Files:** -- Modify: `frontend/src/pages/AgentsPage.tsx` -- Modify: `frontend/src/lib/api.ts` - -- [ ] **Step 1: Add API functions for channel binding CRUD** - -In `frontend/src/lib/api.ts`, add after the existing `fetchAgentChannels` function: - -```typescript -export async function bindAgentChannel( - agentId: string, - channelType: string, - config?: Record, -): Promise { - const res = await fetch( - `${getBase()}/v1/managed-agents/${agentId}/channels`, - { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - channel_type: channelType, - config: config || {}, - routing_mode: 'dedicated', - }), - }, - ); - if (!res.ok) throw new Error(`Failed: ${res.status}`); - return res.json(); -} - -export async function unbindAgentChannel( - agentId: string, - bindingId: string, -): Promise { - const res = await fetch( - `${getBase()}/v1/managed-agents/${agentId}/channels/${bindingId}`, - { method: 'DELETE' }, - ); - if (!res.ok) throw new Error(`Failed: ${res.status}`); -} -``` - -- [ ] **Step 2: Add 'channels' to DETAIL_TABS in AgentsPage.tsx** - -Find the `DETAIL_TABS` array (line ~1315) and add a channels entry. Also find the `Wifi` icon import from `lucide-react` at the top of the file: - -Add to imports: -```typescript -import { Wifi } from 'lucide-react'; -``` - -Update `DETAIL_TABS`: -```typescript -const DETAIL_TABS = [ - { id: 'overview', label: 'Overview', icon: Activity }, - { id: 'interact', label: 'Interact', icon: MessageSquare }, - { id: 'channels', label: 'Channels', icon: Wifi }, - { id: 'tasks', label: 'Tasks', icon: ListTodo }, - { id: 'memory', label: 'Memory', icon: Brain }, - { id: 'learning', label: 'Learning', icon: Settings }, - { id: 'logs', label: 'Logs', icon: FileText }, -] as const; -``` - -Update the `detailTab` state type to include `'channels'`: -```typescript -const [detailTab, setDetailTab] = useState< - 'overview' | 'interact' | 'channels' | 'tasks' | 'memory' | 'learning' | 'logs' ->('overview'); -``` - -- [ ] **Step 3: Add ChannelsTab component** - -Add this component inside `AgentsPage.tsx` (before the main `AgentsPage` component). This is a self-contained component that manages its own state: - -```typescript -const AVAILABLE_CHANNELS = [ - { - type: 'imessage', - name: 'iMessage', - icon: '💬', - description: 'Text from your iPhone, iPad, or Mac', - connectLabel: 'Phone number or contact to monitor', - placeholder: '+15551234567', - }, - { - type: 'slack', - name: 'Slack', - icon: '#', - description: 'Message from any Slack workspace', - connectLabel: 'Slack bot token (xoxb-...)', - placeholder: 'xoxb-...', - }, - { - type: 'whatsapp', - name: 'WhatsApp', - icon: '📱', - description: 'Message via WhatsApp', - connectLabel: 'WhatsApp access token', - placeholder: 'Access token', - }, - { - type: 'twilio', - name: 'SMS (Twilio)', - icon: '📨', - description: 'Text from any phone via Twilio', - connectLabel: 'Twilio phone number', - placeholder: '+15551234567', - }, -]; - -function ChannelsTab({ agentId }: { agentId: string }) { - const [bindings, setBindings] = useState([]); - const [connectingType, setConnectingType] = useState(null); - const [inputValue, setInputValue] = useState(''); - const [loading, setLoading] = useState(false); - - const loadBindings = useCallback(() => { - fetchAgentChannels(agentId).then(setBindings).catch(() => setBindings([])); - }, [agentId]); - - useEffect(() => { loadBindings(); }, [loadBindings]); - - const handleConnect = async (channelType: string) => { - if (!inputValue.trim()) return; - setLoading(true); - try { - await bindAgentChannel(agentId, channelType, { - identifier: inputValue.trim(), - }); - setConnectingType(null); - setInputValue(''); - loadBindings(); - } catch { - // Could show error toast - } finally { - setLoading(false); - } - }; - - const handleDisconnect = async (bindingId: string) => { - try { - await unbindAgentChannel(agentId, bindingId); - loadBindings(); - } catch { - // Could show error toast - } - }; - - const boundTypes = new Set(bindings.map((b) => b.channel_type)); - - return ( -
-
-

- Talk to this agent from -

-
- - {AVAILABLE_CHANNELS.map((ch) => { - const binding = bindings.find((b) => b.channel_type === ch.type); - const isConnecting = connectingType === ch.type; - - return ( -
-
-
- {ch.icon} -
-
-
{ch.name}
-
- {ch.description} -
-
- {binding ? ( - - Active - - ) : ( - - )} -
- - {/* Expanded: connected details */} - {binding && ( -
-
- {binding.config?.identifier - ? `Monitoring: ${binding.config.identifier}` - : `Session: ${binding.session_id}`} -
- -
- )} - - {/* Expanded: connect form */} - {isConnecting && ( -
-
- {ch.connectLabel} -
-
- setInputValue(e.target.value)} - placeholder={ch.placeholder} - style={{ - flex: 1, padding: '6px 10px', - background: 'var(--color-bg-secondary)', - border: '1px solid var(--color-border)', - borderRadius: 4, color: 'var(--color-text)', - fontSize: 12, - }} - onKeyDown={(e) => { - if (e.key === 'Enter') handleConnect(ch.type); - }} - /> - -
-
- )} -
- ); - })} -
- ); -} -``` - -- [ ] **Step 4: Wire the tab content rendering** - -In the tab content rendering section (around line 1410-1525), add between the existing tab conditionals: - -```typescript -{detailTab === 'channels' && ( - -)} -``` - -- [ ] **Step 5: Add missing imports** - -At the top of `AgentsPage.tsx`, ensure these are imported from `api.ts`: - -```typescript -import { - // ... existing imports - bindAgentChannel, - unbindAgentChannel, -} from '../lib/api'; -``` - -- [ ] **Step 6: Verify TypeScript compiles** - -```bash -cd frontend && npx tsc --noEmit 2>&1 | head -20 -``` - -Expected: No errors. - -- [ ] **Step 7: Build frontend** - -```bash -cd frontend && npm run build 2>&1 | tail -10 -``` - -Expected: Build succeeds. - -- [ ] **Step 8: Commit** - -```bash -git add frontend/src/pages/AgentsPage.tsx frontend/src/lib/api.ts -git commit -m "feat: add Channels tab to agent detail view with connect/disconnect UI" -``` - ---- - -### Task 2: Start iMessage daemon when binding iMessage channel - -**Files:** -- Modify: `src/openjarvis/server/agent_manager_routes.py` - -The backend `bind_channel` endpoint currently just creates a DB record. For iMessage, it also needs to start the daemon. For other channels, it just stores the config. - -- [ ] **Step 1: Modify the bind_channel endpoint to start iMessage daemon** - -Find the `bind_channel` endpoint (around line 956). Replace it with: - -```python -@agents_router.post("/{agent_id}/channels") -async def bind_channel(agent_id: str, req: BindChannelRequest): - if not manager.get_agent(agent_id): - raise HTTPException(status_code=404, detail="Agent not found") - binding = manager.bind_channel( - agent_id, - channel_type=req.channel_type, - config=req.config, - routing_mode=req.routing_mode, - ) - - # Start iMessage daemon if binding iMessage - if req.channel_type == "imessage": - identifier = (req.config or {}).get("identifier", "") - if identifier: - try: - from openjarvis.channels.imessage_daemon import ( - is_running, - run_daemon, - ) - - if not is_running(): - import threading - - engine = getattr(request.app.state, "engine", None) - if engine: - from openjarvis.server.agent_manager_routes import ( - _build_deep_research_tools, - ) - - tools = _build_deep_research_tools( - engine=engine, model="", - ) - if tools: - from openjarvis.agents.deep_research import ( - DeepResearchAgent, - ) - - agent = DeepResearchAgent( - engine=engine, - model=getattr(engine, "_model", ""), - tools=tools, - ) - - def handler(text: str) -> str: - result = agent.run(text) - return result.content or "No results." - - t = threading.Thread( - target=run_daemon, - kwargs={ - "chat_identifier": identifier, - "handler": handler, - }, - daemon=True, - ) - t.start() - except Exception as exc: - logger.warning( - "Failed to start iMessage daemon: %s", exc, - ) - - return binding -``` - -Note: This requires adding `request: Request` to the function signature: - -```python -@agents_router.post("/{agent_id}/channels") -async def bind_channel( - agent_id: str, req: BindChannelRequest, request: Request, -): -``` - -- [ ] **Step 2: Stop daemon on unbind** - -Find the `unbind_channel` endpoint (around line 967). Add daemon stop before deleting: - -```python -@agents_router.delete("/{agent_id}/channels/{binding_id}") -async def unbind_channel(agent_id: str, binding_id: str): - # Stop iMessage daemon if applicable - binding = manager._get_binding(binding_id) - if binding and binding.get("channel_type") == "imessage": - try: - from openjarvis.channels.imessage_daemon import stop_daemon - stop_daemon() - except Exception: - pass - manager.unbind_channel(binding_id) - return {"status": "unbound"} -``` - -- [ ] **Step 3: Commit** - -```bash -git add src/openjarvis/server/agent_manager_routes.py -git commit -m "feat: start/stop iMessage daemon on channel bind/unbind" -``` - ---- - -### Task 3: E2E test — Channels tab in browser - -**Files:** None (manual testing) - -- [ ] **Step 1: Start server and open browser** - -```bash -uv run jarvis serve --host 127.0.0.1 --port 8222 -``` - -Open `http://127.0.0.1:8222`. - -- [ ] **Step 2: Create a DeepResearch agent and check Channels tab** - -1. Go to Agents tab → create from "Personal Deep Research" template -2. Open the agent → click "Channels" tab -3. Verify 4 channels listed: iMessage, Slack, WhatsApp, SMS -4. All should show "Connect" buttons - -- [ ] **Step 3: Connect iMessage channel** - -1. Click "Connect" on iMessage -2. Enter a phone number -3. Click "Connect" -4. Verify it shows "Active" with the phone number - -- [ ] **Step 4: Disconnect** - -1. Click "Disconnect" on the iMessage binding -2. Verify it returns to "Connect" state - -- [ ] **Step 5: Push** - -```bash -git push origin main -``` - ---- - -## Sub-project 2: Connector Setup Instructions - -### Task 4: Add per-connector instruction data - -**Files:** -- Modify: `frontend/src/types/connectors.ts` - -- [ ] **Step 1: Extend ConnectorMeta with setup instructions** - -In `frontend/src/types/connectors.ts`, add a `steps` field to the SOURCE_CATALOG entries. Each step has a `label`, optional `url` (opens in browser), and optional `inputField`: - -```typescript -export interface SetupStep { - label: string; - url?: string; - urlLabel?: string; -} - -export interface ConnectorMeta { - connector_id: string; - display_name: string; - auth_type: 'oauth' | 'local' | 'bridge' | 'filesystem'; - category: string; - icon: string; - color: string; - description: string; - steps?: SetupStep[]; - inputFields?: Array<{ - name: string; - placeholder: string; - type?: 'text' | 'password'; - }>; -} -``` - -- [ ] **Step 2: Add per-connector steps to SOURCE_CATALOG** - -Update the existing SOURCE_CATALOG entries with setup instructions: - -```typescript -export const SOURCE_CATALOG: ConnectorMeta[] = [ - { - connector_id: 'gmail_imap', - display_name: 'Gmail (IMAP)', - auth_type: 'oauth', - category: 'communication', - icon: 'Mail', - color: 'text-red-400', - description: 'Email via app password', - steps: [ - { - label: 'Enable 2-Factor Authentication on your Google account', - url: 'https://myaccount.google.com/signinoptions/two-step-verification', - urlLabel: 'Open Google Security', - }, - { - label: 'Generate an App Password for "Mail"', - url: 'https://myaccount.google.com/apppasswords', - urlLabel: 'Open App Passwords', - }, - { label: 'Paste your credentials below' }, - ], - inputFields: [ - { name: 'email', placeholder: 'Email address', type: 'text' }, - { name: 'password', placeholder: 'App password (xxxx xxxx xxxx xxxx)', type: 'password' }, - ], - }, - { - connector_id: 'slack', - display_name: 'Slack', - auth_type: 'oauth', - category: 'communication', - icon: 'Hash', - color: 'text-purple-400', - description: 'Channel messages and threads', - steps: [ - { - label: 'Go to your Slack App settings and copy the Bot User OAuth Token', - url: 'https://api.slack.com/apps', - urlLabel: 'Open Slack Apps', - }, - { label: 'Paste the bot token below (starts with xoxb-)' }, - ], - inputFields: [ - { name: 'token', placeholder: 'xoxb-...', type: 'password' }, - ], - }, - { - connector_id: 'notion', - display_name: 'Notion', - auth_type: 'oauth', - category: 'documents', - icon: 'FileText', - color: 'text-gray-300', - description: 'Pages and databases', - steps: [ - { - label: 'Create an internal integration and copy the secret', - url: 'https://www.notion.so/profile/integrations', - urlLabel: 'Open Notion Integrations', - }, - { label: 'Paste the integration token below (starts with ntn_)' }, - { label: 'Then share pages with your integration: Page → ... → Connections → Add' }, - ], - inputFields: [ - { name: 'token', placeholder: 'ntn_...', type: 'password' }, - ], - }, - { - connector_id: 'granola', - display_name: 'Granola', - auth_type: 'oauth', - category: 'documents', - icon: 'Mic', - color: 'text-amber-400', - description: 'AI meeting notes', - steps: [ - { label: 'Open the Granola desktop app → Settings → API' }, - { label: 'Copy your API key and paste below' }, - ], - inputFields: [ - { name: 'token', placeholder: 'grn_...', type: 'password' }, - ], - }, - { - connector_id: 'gmail', - display_name: 'Gmail', - auth_type: 'oauth', - category: 'communication', - icon: 'Mail', - color: 'text-red-400', - description: 'Email messages and threads (OAuth)', - steps: [ - { label: 'Click "Authorize" to open Google consent screen' }, - { label: 'Grant read-only access to your Gmail' }, - ], - }, - { - connector_id: 'imessage', - display_name: 'iMessage', - auth_type: 'local', - category: 'communication', - icon: 'MessageSquare', - color: 'text-green-400', - description: 'macOS Messages history', - steps: [ - { label: 'Open System Settings → Privacy & Security → Full Disk Access' }, - { label: 'Enable access for your terminal app or OpenJarvis' }, - ], - }, - { - connector_id: 'obsidian', - display_name: 'Obsidian', - auth_type: 'filesystem', - category: 'documents', - icon: 'FolderOpen', - color: 'text-purple-300', - description: 'Markdown vault', - steps: [ - { label: 'Enter the path to your Obsidian vault folder' }, - ], - inputFields: [ - { name: 'path', placeholder: '/Users/you/Documents/MyVault', type: 'text' }, - ], - }, - { - connector_id: 'gdrive', - display_name: 'Google Drive', - auth_type: 'oauth', - category: 'documents', - icon: 'FolderOpen', - color: 'text-blue-400', - description: 'Docs, Sheets, and files', - steps: [ - { label: 'Click "Authorize" to grant read-only access to Google Drive' }, - ], - }, - { - connector_id: 'gcalendar', - display_name: 'Calendar', - auth_type: 'oauth', - category: 'pim', - icon: 'Calendar', - color: 'text-blue-400', - description: 'Events and meetings', - steps: [ - { label: 'Click "Authorize" to grant read-only access to Google Calendar' }, - ], - }, - { - connector_id: 'gcontacts', - display_name: 'Contacts', - auth_type: 'oauth', - category: 'pim', - icon: 'Users', - color: 'text-blue-400', - description: 'People and contact info', - steps: [ - { label: 'Click "Authorize" to grant read-only access to Google Contacts' }, - ], - }, -]; -``` - -- [ ] **Step 3: Commit** - -```bash -git add frontend/src/types/connectors.ts -git commit -m "feat: add per-connector setup instructions and input fields to SOURCE_CATALOG" -``` - ---- - -### Task 5: Build StepByStepPanel component for SourceConnectFlow - -**Files:** -- Modify: `frontend/src/components/setup/SourceConnectFlow.tsx` - -- [ ] **Step 1: Add the StepByStepPanel component** - -Add this component inside `SourceConnectFlow.tsx`, before the main `SourceConnectFlow` component. It replaces the generic OAuth panel for connectors that have `steps` defined: - -```typescript -function StepByStepPanel({ - connector, - onConnect, - onSkip, - isConnecting, -}: { - connector: ConnectorMeta; - onConnect: (req: ConnectRequest) => void; - onSkip: () => void; - isConnecting: boolean; -}) { - const [inputs, setInputs] = useState>({}); - const steps = connector.steps || []; - const fields = connector.inputFields || []; - - const updateInput = (name: string, value: string) => { - setInputs((prev) => ({ ...prev, [name]: value })); - }; - - const handleSubmit = () => { - const req: ConnectRequest = {}; - for (const field of fields) { - if (field.name === 'email') req.email = inputs.email; - else if (field.name === 'password') req.password = inputs.password; - else if (field.name === 'token') req.token = inputs.token; - else if (field.name === 'path') req.path = inputs.path; - } - // For email+password connectors, also set token as email:password - if (req.email && req.password) { - req.token = `${req.email}:${req.password}`; - req.code = req.token; - } - if (req.token && !req.code) { - req.code = req.token; - } - onConnect(req); - }; - - const allFilled = fields.every((f) => inputs[f.name]?.trim()); - - return ( -
-
- - {connector.icon === 'Mail' ? '✉️' : - connector.icon === 'Hash' ? '#️⃣' : - connector.icon === 'FileText' ? '📄' : - connector.icon === 'Mic' ? '🎙️' : - connector.icon === 'FolderOpen' ? '📁' : '🔗'} - - - {connector.display_name} - -
- - {steps.map((step, i) => ( -
-
- STEP {i + 1} -
-
- {step.label} -
- {step.url && ( - - {step.urlLabel || 'Open'} → - - )} -
- ))} - - {fields.length > 0 && ( -
- {fields.map((field) => ( - updateInput(field.name, e.target.value)} - placeholder={field.placeholder} - type={field.type || 'text'} - style={{ - width: '100%', - padding: '8px 10px', - background: 'var(--color-bg-secondary)', - border: '1px solid var(--color-border)', - borderRadius: 4, - color: 'var(--color-text)', - fontSize: 13, - marginBottom: 8, - boxSizing: 'border-box', - }} - /> - ))} -
- )} - -
- 🔒 Read-only access · No data leaves your device -
- -
- - -
-
- ); -} -``` - -- [ ] **Step 2: Wire StepByStepPanel into the connect flow** - -In the `SourceConnectFlow` component, find where it renders the OAuth/Local/Filesystem panels based on `auth_type`. Modify the rendering logic to check if the connector has `steps` defined, and if so, use `StepByStepPanel` instead of the generic panels: - -```typescript -// In the panel rendering section, add before the existing auth_type checks: -const connectorMeta = SOURCE_CATALOG.find( - (c) => c.connector_id === activeSource.connector_id, -); - -// If connector has steps, use StepByStepPanel regardless of auth_type -if (connectorMeta?.steps) { - return ( - handleConnect(activeSource.connector_id, req)} - onSkip={() => handleSkip(activeIndex)} - isConnecting={activeSource.state === 'connecting'} - /> - ); -} - -// Fallback to existing generic panels for connectors without steps -``` - -- [ ] **Step 3: Import ConnectorMeta and SOURCE_CATALOG** - -Ensure these are imported at the top of `SourceConnectFlow.tsx`: - -```typescript -import { ConnectorMeta, SOURCE_CATALOG, ConnectRequest } from '../../types/connectors'; -``` - -- [ ] **Step 4: Verify TypeScript compiles and build** - -```bash -cd frontend && npx tsc --noEmit && npm run build 2>&1 | tail -5 -``` - -Expected: No errors, build succeeds. - -- [ ] **Step 5: Commit** - -```bash -git add frontend/src/components/setup/SourceConnectFlow.tsx -git commit -m "feat: add StepByStepPanel with per-connector setup instructions" -``` - ---- - -### Task 6: E2E test — setup wizard with new connector flow - -**Files:** None (manual testing) - -- [ ] **Step 1: Open the setup wizard** - -Navigate to the setup wizard (or trigger it from the agent creation flow). - -- [ ] **Step 2: Test Gmail IMAP connection** - -1. Select "Gmail (IMAP)" in source picker -2. Verify 3 numbered steps appear with links to Google settings -3. Verify email + password input fields appear -4. Enter credentials and click Connect -5. Verify connection succeeds - -- [ ] **Step 3: Test Notion connection** - -1. Select "Notion" -2. Verify steps show link to Notion Integrations page -3. Verify token input field -4. Paste token, click Connect - -- [ ] **Step 4: Test Slack connection** - -1. Select "Slack" -2. Verify steps show link to Slack Apps page -3. Paste bot token, click Connect - -- [ ] **Step 5: Verify ingest dashboard works with connected sources** - -After connecting sources, verify the IngestDashboard shows sync progress for each. - -- [ ] **Step 6: Push** - -```bash -git push origin main -``` diff --git a/docs/superpowers/plans/2026-03-27-personal-deep-research-template.md b/docs/superpowers/plans/2026-03-27-personal-deep-research-template.md deleted file mode 100644 index 2cb5a325..00000000 --- a/docs/superpowers/plans/2026-03-27-personal-deep-research-template.md +++ /dev/null @@ -1,356 +0,0 @@ -# Personal DeepResearch Template + Server Wiring Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Register "Personal Deep Research" as a default agent template, wire DeepResearch tools in the server streaming endpoint, and add a `jarvis research` CLI alias — so users can create and chat with the agent from the desktop app, browser, or CLI. - -**Architecture:** A new TOML template file auto-discovered by `AgentManager.list_templates()`. The server's `_stream_managed_agent()` is extended to build the 4 DeepResearch tools (knowledge_search, knowledge_sql, scan_chunks, think) from `~/.openjarvis/knowledge.db` when `agent_type == "deep_research"`. A CLI alias `jarvis research` wraps the existing `deep-research-setup` command. - -**Tech Stack:** Python 3.10+, TOML, FastAPI, Click, pytest - ---- - -### Task 1: Create the agent template TOML - -**Files:** -- Create: `src/openjarvis/agents/templates/personal_deep_research.toml` - -- [ ] **Step 1: Create the template file** - -Create `src/openjarvis/agents/templates/personal_deep_research.toml`: - -```toml -[template] -id = "personal_deep_research" -name = "Personal Deep Research" -description = "Search across your emails, messages, meeting notes, and documents with multi-hop retrieval and cited reports. Uses BM25 keyword search, SQL aggregation, and LM-powered semantic scanning." -agent_type = "deep_research" -schedule_type = "manual" -tools = ["knowledge_search", "knowledge_sql", "scan_chunks", "think"] -max_turns = 8 -temperature = 0.3 -max_tokens = 4096 -``` - -- [ ] **Step 2: Verify template is discovered** - -```bash -uv run python3 -c " -from openjarvis.agents.manager import AgentManager -templates = AgentManager.list_templates() -ids = [t['id'] for t in templates] -print(f'Templates: {ids}') -assert 'personal_deep_research' in ids, 'Template not found!' -t = next(t for t in templates if t['id'] == 'personal_deep_research') -print(f'Name: {t[\"name\"]}') -print(f'Agent type: {t[\"agent_type\"]}') -print(f'Tools: {t.get(\"tools\", [])}') -print('PASS') -" -``` - -Expected: Template found with correct fields. - -- [ ] **Step 3: Commit** - -```bash -git add src/openjarvis/agents/templates/personal_deep_research.toml -git commit -m "feat: add Personal Deep Research agent template" -``` - ---- - -### Task 2: Wire DeepResearch tools in server streaming endpoint - -**Files:** -- Modify: `src/openjarvis/server/agent_manager_routes.py` (lines 263-266) -- Create: `tests/server/test_deep_research_tools_wiring.py` - -- [ ] **Step 1: Write the test** - -Create `tests/server/test_deep_research_tools_wiring.py`: - -```python -"""Test that _stream_managed_agent wires DeepResearch tools correctly.""" - -from __future__ import annotations - -import tempfile -from pathlib import Path -from unittest.mock import MagicMock, patch - -import pytest - - -def test_deep_research_agent_gets_tools(tmp_path: Path) -> None: - """When agent_type is deep_research, agent receives 4 tools.""" - from openjarvis.connectors.store import KnowledgeStore - - # Create a knowledge store with some data - db_path = tmp_path / "knowledge.db" - store = KnowledgeStore(str(db_path)) - store.store("test content", source="test", doc_type="note") - - from openjarvis.server.agent_manager_routes import _build_deep_research_tools - - tools = _build_deep_research_tools( - engine=MagicMock(), - model="test-model", - knowledge_db_path=str(db_path), - ) - - tool_ids = [t.tool_id for t in tools] - assert "knowledge_search" in tool_ids - assert "knowledge_sql" in tool_ids - assert "scan_chunks" in tool_ids - assert "think" in tool_ids - assert len(tools) == 4 - store.close() - - -def test_deep_research_tools_returns_empty_when_no_db() -> None: - """When knowledge.db doesn't exist, returns empty list.""" - from openjarvis.server.agent_manager_routes import _build_deep_research_tools - - tools = _build_deep_research_tools( - engine=MagicMock(), - model="test-model", - knowledge_db_path="/nonexistent/path/knowledge.db", - ) - - assert tools == [] -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/server/test_deep_research_tools_wiring.py -v --tb=short -``` - -Expected: FAIL — `ImportError: cannot import name '_build_deep_research_tools'` - -- [ ] **Step 3: Add `_build_deep_research_tools` function to agent_manager_routes.py** - -Add this function near the top of the file (after the imports, before the router definitions): - -```python -def _build_deep_research_tools( - engine: Any, - model: str, - knowledge_db_path: str = "", -) -> list: - """Build the 4 DeepResearch tools from a KnowledgeStore. - - Returns an empty list if the knowledge DB does not exist. - """ - from pathlib import Path - - if not knowledge_db_path: - from openjarvis.core.config import DEFAULT_CONFIG_DIR - knowledge_db_path = str(DEFAULT_CONFIG_DIR / "knowledge.db") - - if not Path(knowledge_db_path).exists(): - return [] - - from openjarvis.connectors.retriever import TwoStageRetriever - from openjarvis.connectors.store import KnowledgeStore - from openjarvis.tools.knowledge_search import KnowledgeSearchTool - from openjarvis.tools.knowledge_sql import KnowledgeSQLTool - from openjarvis.tools.scan_chunks import ScanChunksTool - from openjarvis.tools.think import ThinkTool - - store = KnowledgeStore(knowledge_db_path) - retriever = TwoStageRetriever(store) - return [ - KnowledgeSearchTool(retriever=retriever), - KnowledgeSQLTool(store=store), - ScanChunksTool(store=store, engine=engine, model=model), - ThinkTool(), - ] -``` - -- [ ] **Step 4: Wire the tools into `_stream_managed_agent`** - -In `_stream_managed_agent()`, add tool wiring between the config extraction (line ~263) and the agent instantiation (line ~265). Insert after `if config.get("max_turns") is not None:` block: - -```python - # Build DeepResearch tools when applicable - if agent_type == "deep_research": - tools = _build_deep_research_tools(engine=engine, model=model) - if tools: - agent_kwargs["tools"] = tools -``` - -- [ ] **Step 5: Ensure tests/server/__init__.py exists** - -```bash -touch tests/server/__init__.py -``` - -- [ ] **Step 6: Run tests** - -```bash -uv run pytest tests/server/test_deep_research_tools_wiring.py -v --tb=short -``` - -Expected: 2/2 PASS. - -- [ ] **Step 7: Commit** - -```bash -git add src/openjarvis/server/agent_manager_routes.py tests/server/__init__.py tests/server/test_deep_research_tools_wiring.py -git commit -m "feat: wire DeepResearch tools in managed agent streaming endpoint" -``` - ---- - -### Task 3: Add `jarvis research` CLI alias - -**Files:** -- Modify: `src/openjarvis/cli/__init__.py` - -- [ ] **Step 1: Add the alias** - -In `src/openjarvis/cli/__init__.py`, find the line that registers `deep-research-setup` (should look like `cli.add_command(deep_research_setup, "deep-research-setup")`). Add the alias right after: - -```python -cli.add_command(deep_research_setup, "research") -``` - -This registers the same command under the shorter name `research`. - -- [ ] **Step 2: Verify** - -```bash -uv run jarvis research --help | head -5 -``` - -Expected: Shows the same help as `jarvis deep-research-setup --help`. - -- [ ] **Step 3: Commit** - -```bash -git add src/openjarvis/cli/__init__.py -git commit -m "feat: add 'jarvis research' CLI alias for deep-research-setup" -``` - ---- - -### Task 4: End-to-end test — template in Agents tab via API - -**Files:** None (manual testing via API) - -- [ ] **Step 1: Start the server** - -```bash -uv run jarvis serve --host 127.0.0.1 --port 8000 & -sleep 3 -``` - -- [ ] **Step 2: Verify template appears** - -```bash -curl -s http://localhost:8000/v1/templates | python3 -c " -import sys, json -data = json.load(sys.stdin) -ids = [t['id'] for t in data['templates']] -print(f'Templates: {ids}') -assert 'personal_deep_research' in ids -print('PASS: template found') -" -``` - -- [ ] **Step 3: Create agent from template** - -```bash -curl -s -X POST http://localhost:8000/v1/managed-agents \ - -H 'Content-Type: application/json' \ - -d '{"name": "My Research Agent", "template_id": "personal_deep_research"}' \ - | python3 -c " -import sys, json -agent = json.load(sys.stdin) -print(f'Agent created: {agent[\"id\"]}') -print(f'Type: {agent[\"agent_type\"]}') -print(f'Status: {agent[\"status\"]}') -print(f'Config tools: {agent.get(\"config\", {}).get(\"tools\", [])}') -" -``` - -Expected: Agent created with `agent_type: deep_research`. - -- [ ] **Step 4: Send a message and verify streaming** - -```bash -AGENT_ID=$(curl -s http://localhost:8000/v1/managed-agents | python3 -c " -import sys, json -agents = json.load(sys.stdin)['agents'] -dr = [a for a in agents if a['agent_type'] == 'deep_research'] -print(dr[0]['id'] if dr else '') -") - -curl -s -X POST "http://localhost:8000/v1/managed-agents/${AGENT_ID}/messages" \ - -H 'Content-Type: application/json' \ - -d '{"content": "Who are the 5 people I message the most?", "stream": true}' \ - --no-buffer | head -20 -``` - -Expected: SSE chunks streaming with agent response including tool call results. - -- [ ] **Step 5: Stop server and push** - -```bash -kill %1 -git push origin feat/deep-research-setup -``` - ---- - -### Task 5: Update PR #78 title and description - -**Files:** None (GitHub CLI) - -- [ ] **Step 1: Update PR title and body** - -```bash -gh pr edit 78 --repo open-jarvis/OpenJarvis \ - --title "feat: Personal Deep Research — connectors, retrieval, agent, channels, desktop" \ - --body "$(cat <<'PREOF' -## Summary - -Full Deep Research experience: connect personal data sources → ingest → multi-hop retrieval → cited research reports via desktop app, browser, CLI, or iMessage. - -### What's built -- **13 connectors**: Gmail IMAP, Outlook, Obsidian, Apple Notes, iMessage, Slack, Notion, Granola, Google Drive*, Calendar*, Contacts*, Dropbox*, WhatsApp* -- **Ingestion pipeline**: SemanticChunker, KnowledgeStore (FTS5/BM25), incremental sync, attachment store -- **4 agent tools**: knowledge_search (BM25), knowledge_sql (SQL aggregation), scan_chunks (LM semantic grep), think (reasoning) -- **DeepResearchAgent**: multi-hop with query expansion, forced synthesis, cited reports -- **Agent template**: "Personal Deep Research" in Agents tab — create and chat via desktop/browser -- **CLI**: `jarvis deep-research-setup` / `jarvis research` — auto-detect sources, ingest, chat -- **API**: 6 connector endpoints + managed agent streaming -- **Channel gateway**: ChannelBridge, webhook routes for Twilio/BlueBubbles/WhatsApp -- **220+ tests** - -*\* = mocked tests only* - -### Live-verified with real data -- 55,302 chunks (Apple Notes + iMessage + Gmail + Slack + Notion + Granola) -- DeepResearchAgent with Qwen3.5 35B MoE producing cited reports -- Cross-source queries: found Spain trip dates from iMessages, ranked top contacts via SQL, identified recurring meetings from Granola - -## Test plan -- [ ] `uv run pytest tests/ -v` — all tests pass -- [ ] `uv run jarvis research` — end-to-end CLI flow -- [ ] Create agent from template in Agents tab -- [ ] Stream chat response with tool calls -- [ ] Verify read-only: no writes to any connector - -🤖 Generated with [Claude Code](https://claude.com/claude-code) -PREOF -)" -``` - -- [ ] **Step 2: Push our branch to the PR branch** - -```bash -git push origin feat/deep-research-setup:feat/mobile-channel-gateway --force-with-lease -``` diff --git a/docs/superpowers/specs/2026-03-25-deep-research-setup-design.md b/docs/superpowers/specs/2026-03-25-deep-research-setup-design.md deleted file mode 100644 index 5b4b276e..00000000 --- a/docs/superpowers/specs/2026-03-25-deep-research-setup-design.md +++ /dev/null @@ -1,468 +0,0 @@ -# Deep Research Setup Experience — Design Spec - -**Date:** 2026-03-25 -**Status:** Draft -**Scope:** End-to-end design for the OpenJarvis "killer install + setup" experience — from first launch to Deep Research over your personal data, accessible via desktop app, CLI, and messaging channels. - ---- - -## 1. Vision - -An Apple-like onboarding experience that connects users to their personal data (email, messaging, documents, calendar, contacts, notes), indexes it privately on-device, and provides a Deep Research agent that can navigate the full corpus with multi-hop retrieval — all accessible from the desktop app, CLI, or messaging channels like iMessage, WhatsApp, and Slack. - -## 2. Architecture Overview - -Seven layers from data sources to user interfaces: - -``` -Data Sources (12 services) - ↓ OAuth / API / Local DB access -Connector Layer (native bulk sync + MCP real-time tools) - ↓ Normalized Document objects -Ingestion Pipeline (normalize → deduplicate → chunk → index) - ↓ -On-Device Storage (SQLite/FTS5 + ColBERTv2 + attachment store) - ↓ -Multi-Layer Retrieval (BM25 recall → ColBERTv2 rerank → agent multi-hop) - ↓ -Deep Research Agent (ToolUsingAgent with knowledge_search, MCP tools, think, web_search) - ↓ -User Interfaces (Desktop App, CLI, iMessage, WhatsApp, Slack) -``` - -### Design Principles - -- **Local-first:** All data stored on-device in SQLite. No cloud dependency for storage or retrieval. -- **Hybrid connectors:** Native bulk sync for initial ingest, MCP tools for real-time agent queries. -- **Two-stage retrieval:** BM25 for fast recall, ColBERTv2 for precise reranking. Agent adds multi-hop. -- **Progressive onboarding:** Pick sources → guided OAuth for each → bulk ingest → ready to research. -- **Escalation model:** Quick answers in chat channels, full reports in desktop app. - -## 3. Data Sources - -Twelve data sources across three categories: - -| Category | Sources | Auth Type | -|----------|---------|-----------| -| **Communication** | Gmail, Slack, iMessage, WhatsApp | OAuth, OAuth, Local DB, Bridge (QR) | -| **Documents** | Google Drive, Dropbox, Notion | OAuth, OAuth, OAuth | -| **PIM** | Google Calendar, Google Contacts, Apple Notes, Obsidian/Markdown | OAuth, OAuth, Local DB, Filesystem | - -## 4. Setup Wizard (Desktop-First) - -The primary onboarding path is the Tauri desktop app. The CLI gets a simpler `jarvis connect` command that performs the same operations text-based. The desktop wizard is the "wow" experience — polished visual design with real product logos, clean progress bars, and premium feel throughout. - -### Step 1: Welcome + Hardware Detection - -Auto-detects CPU, RAM, GPU. Shows friendly capability messaging (e.g., "MacBook Pro M4 Max, 128GB RAM — you can run powerful local models"). Background: Ollama starts, recommended model begins downloading. - -**Builds on:** Existing desktop boot sequence (Ollama → model → server) and `jarvis init` hardware detection. - -**New:** Friendlier UI with hardware capability messaging, sets user expectations for what's possible on their machine. - -### Step 2: Engine Setup - -If Ollama is already installed, skip silently. If not, one-click install or "Use cloud instead" option. Model download with a polished progress bar. Option to add cloud API keys for hybrid local+cloud mode. - -**Builds on:** Existing `jarvis init` engine selection and model download logic. - -**New:** Automated Ollama install, cloud key entry UI, visual progress. - -### Step 3: Pick Your Sources - -A grid of source cards with product logos, grouped by category (Communication / Documents / PIM). User toggles on/off. "Connect what you want now — you can always add more later." - -**Entirely new.** This step and everything after it is new functionality. - -### Step 4: Connect Each Source (Guided) - -For each selected source, one at a time, with a sidebar checklist showing connected/remaining status. "Skip" option for each. Four connection patterns: - -**OAuth services** (Gmail, Drive, Slack, Calendar, Contacts, Dropbox, Notion): Show what permissions are requested and why → "Open in browser" button → OAuth consent screen → redirect back → green checkmark. OpenJarvis registers its own OAuth apps — users never create developer credentials. - -**Local access** (iMessage, Apple Notes): Request macOS Full Disk Access permission → explain why in plain language → link to System Preferences → verify access was granted. - -**Bridge services** (WhatsApp): Display QR code in the app → user scans with their phone → connection confirmed. - -**Local files** (Obsidian/Markdown): File picker → select vault directory → confirm. - -### Step 5: Ingest & Index - -Bulk sync begins for all connected sources in parallel. Live dashboard shows per-source progress with item counts and progress bars: - -``` -✉ Gmail 12,847 / ~45,000 emails ████████░░░░░░░ 28% -💬 Slack 3,201 / ~8,500 messages ██████████░░░░░ 38% -📁 Google Drive 156 / ~420 docs █████████░░░░░░ 37% -📱 iMessage ✓ Complete 24,102 messages indexed -``` - -**Key UX:** User doesn't have to wait. They can start chatting with partial data — "Your data is still syncing, but you can start asking questions now." Ingest is resumable: close the app, come back later, it picks up where it left off. - -### Step 6: Ready — First Query - -Drops user into the Deep Research chat. Suggested starter queries based on their connected sources: -- "What were the key decisions from last week's team threads?" -- "Find the proposal doc Sarah shared about the Q3 roadmap" -- "Summarize my unread emails from today" - -## 5. Connector Layer - -### New Abstraction: `BaseConnector` - -Lives in `src/openjarvis/connectors/` — separate from channels. Channels are for real-time communication (send/receive). Connectors are for data ingest (bulk download + index history). - -```python -class BaseConnector(ABC): - # Identity - connector_id: str # "gmail", "slack", etc. - display_name: str # "Gmail", "Slack", etc. - icon_url: str # Product logo for UI - auth_type: str # "oauth" | "local" | "bridge" | "filesystem" - - # Auth lifecycle - def auth_url() -> str # OAuth: generate consent URL - def handle_callback(code: str) -> None # OAuth: exchange code for tokens - def is_connected() -> bool - def disconnect() -> None - - # Bulk sync (native) - def sync(since: datetime | None) -> Iterator[Document] - def sync_status() -> SyncStatus - - # MCP real-time (agent-callable) - def mcp_tools() -> list[ToolSpec] -``` - -Registered via `@ConnectorRegistry.register("gmail")` — new registry alongside the existing ones. - -### SyncStatus - -```python -@dataclass -class SyncStatus: - state: str # "idle", "syncing", "paused", "error" - items_synced: int # items completed so far - items_total: int # estimated total (0 if unknown) - last_sync: datetime | None - cursor: str | None # opaque checkpoint for resume - error: str | None # last error message if state == "error" -``` - -### Universal Document Schema - -All connectors normalize their data into a common dataclass: - -```python -@dataclass -class Document: - doc_id: str # globally unique (connector:source_id) - source: str # "gmail", "slack", "gdrive", etc. - doc_type: str # "email", "message", "document", "event", "contact", "note" - content: str # plain text body - title: str # subject line, filename, event title - author: str # sender / creator - participants: list[str] # recipients, channel members, attendees - timestamp: datetime # when created/sent - thread_id: str | None # conversation grouping - url: str | None # deep link back to original source - attachments: list[Attachment] - metadata: dict # source-specific (labels, channel name, folder, etc.) -``` - -Six `doc_type` values: `email`, `message`, `document`, `event`, `contact`, `note`. - -### Per-Connector Details - -**Gmail** — Gmail API (REST). Scope: `gmail.readonly`. Bulk sync via `messages.list` → `messages.get`, paginated, resumable via `historyId`. MCP tools: `gmail_search_emails`, `gmail_get_thread`, `gmail_list_unread`. - -**Slack** — Slack Web API. Scopes: `channels:history`, `channels:read`, `users:read`. Bulk sync via `conversations.history` per channel, cursor-paginated. MCP tools: `slack_search_messages`, `slack_get_thread`, `slack_list_channels`. - -**iMessage** — Direct SQLite read of `~/Library/Messages/chat.db`. Requires macOS Full Disk Access. Bulk sync queries message + handle tables. MCP tools: `imessage_search_messages`, `imessage_get_conversation`. - -**WhatsApp** — Baileys Node.js bridge (existing). Auth via QR code scan. Bulk sync via `fetchMessageHistory` through Baileys protocol. MCP tools: `whatsapp_search_messages`, `whatsapp_get_chat`. - -**Google Drive** — Drive API v3. Scope: `drive.readonly`. Bulk sync via `files.list` → export/download. Google Docs/Sheets/Slides exported as text/markdown. MCP tools: `gdrive_search_files`, `gdrive_get_document`, `gdrive_list_recent`. - -**Dropbox** — Dropbox API v2. Scopes: `files.metadata.read`, `files.content.read`. Bulk sync via `list_folder` recursive → download text-extractable files. MCP tools: `dropbox_search_files`, `dropbox_get_file`, `dropbox_list_recent`. - -**Notion** — Notion API. Read content integration scope. Bulk sync via `search` → retrieve blocks → render to markdown. MCP tools: `notion_search_pages`, `notion_get_page`. - -**Google Calendar** — Calendar API v3. Scope: `calendar.readonly`. Bulk sync via `events.list` across calendars, paginated. MCP tools: `calendar_get_events_today`, `calendar_search_events`, `calendar_next_meeting`. - -**Google Contacts** — People API. Scope: `contacts.readonly`. Bulk sync via `people.connections.list`, paginated. MCP tools: `contacts_find`, `contacts_get_info`. - -**Apple Notes** — Direct SQLite read of `~/Library/Group Containers/group.com.apple.notes/NoteStore.sqlite`. Requires Full Disk Access. Bulk sync decodes `ZICNOTEDATA` (gzipped HTML → plain text). MCP tools: `notes_search`, `notes_get_note`. - -**Obsidian/Markdown** — Direct filesystem read of a user-selected vault directory. Bulk sync walks `.md` files, parses YAML frontmatter + wikilinks. MCP tools: `obsidian_search_notes`, `obsidian_get_note`, `obsidian_list_recent`. - -### SyncEngine - -A shared `SyncEngine` orchestrates all connectors: - -- **Checkpoint/Resume:** Stores last sync cursor per connector in SQLite. Interrupted syncs resume from the checkpoint. -- **Rate Limiting:** Per-connector rate limits with exponential backoff. Respects API quotas (e.g., Gmail: 250 quota units/sec). -- **Incremental Sync:** After initial bulk sync, periodic delta sync fetches only new/modified items. Configurable interval. -- **Parallel Ingest:** Multiple connectors sync concurrently. Documents stream into the ingestion pipeline as they arrive. - -## 6. Ingestion Pipeline - -Four stages from raw documents to retrieval-ready index: - -### Stage 1: Normalize - -Connectors yield `Document` objects. HTML emails converted to plain text via `html2text`. Google Docs exported as markdown. Notion blocks rendered to markdown. Apple Notes decompressed from gzipped HTML to plain text. Attachments extracted, hashed (SHA-256), stored in content-addressed blob store. - -### Stage 2: Deduplicate - -Cross-source dedup by `doc_id` (connector:source_id). If the same message appears in a Slack export and an email notification, keep both but link them via metadata. Attachments deduped by SHA-256 — a file shared across Gmail, Slack, and Drive is stored once. - -### Stage 3: Chunk - -Uses the existing `chunking.py` infrastructure with type-aware strategies. **Chunk size defaults to the active reranker's max context:** 512 tokens when ColBERTv2 is available (its WordPiece max), 256 tokens when falling back to FAISS with `all-MiniLM-L6-v2`. - -**Semantic splitting rules (never split mid-sentence):** - -- **Emails/Messages:** Keep whole if under the chunk limit. For long threads, split on message/reply boundaries — never mid-message. If a single message exceeds the limit, fall back to sentence-boundary splitting. -- **Documents:** Split on section boundaries (`## Heading`) first, then paragraph boundaries within sections. If a single paragraph exceeds the limit, split on sentence boundaries. Each chunk inherits its section heading as metadata. -- **Events:** Single chunk per event (title + attendees + description + location). No splitting needed. -- **Contacts:** Single chunk per contact (name + email + phone + org + notes). Structured for entity lookup. -- **Notes:** Same rules as documents — section boundaries first, then paragraphs, then sentences. - -**Metadata carries context:** Each chunk inherits the parent document's title, author, timestamp, source, and its section heading (if any), so retrieval results are never orphaned fragments with no context about where they came from. - -### Stage 4: Index (Dual-Write) - -Each chunk is written to both indexes simultaneously: - -**SQLite + FTS5** — durable store with BM25 ranking. Rust-backed (existing infrastructure). Stores: chunk content, doc_id, source, doc_type, title, author, participants, timestamp, thread_id, url, metadata JSON, chunk_index. - -**ColBERTv2 Index** — token-level embeddings for semantic reranking. **New: persisted to disk** (current implementation is in-memory). Memory-mapped tensor file with chunk_id→offset index. Append-only for incremental indexing. Lazy loading — only map pages touched by current query. - -### SQLite Schema - -Extends the existing `documents` table with source-aware columns: - -```sql -CREATE TABLE documents ( - id TEXT PRIMARY KEY, -- chunk_id - doc_id TEXT NOT NULL, -- parent document (connector:source_id) - content TEXT NOT NULL, -- chunk text - source TEXT NOT NULL, -- "gmail", "slack", "gdrive", ... - doc_type TEXT NOT NULL, -- "email", "message", "document", ... - title TEXT, -- subject / filename / event title - author TEXT, -- sender / creator - participants TEXT, -- JSON array - timestamp TEXT NOT NULL, -- ISO 8601 - thread_id TEXT, -- conversation grouping - url TEXT, -- deep link to original - metadata TEXT, -- JSON (labels, channel, folder, etc.) - chunk_index INTEGER, -- position within parent doc - created_at TEXT NOT NULL -- when indexed -); - -CREATE VIRTUAL TABLE documents_fts USING fts5( - content, title, author, - content=documents, content_rowid=rowid, - tokenize='porter unicode61' -); - -CREATE INDEX idx_source ON documents(source); -CREATE INDEX idx_doc_type ON documents(doc_type); -CREATE INDEX idx_author ON documents(author); -CREATE INDEX idx_timestamp ON documents(timestamp); -CREATE INDEX idx_thread ON documents(thread_id); -CREATE INDEX idx_doc_id ON documents(doc_id); -``` - -### ColBERTv2 Persistence (New) - -The current ColBERT backend is in-memory only. For a personal knowledge base with 100K+ chunks, disk persistence is required. - -- **On-disk format:** Memory-mapped tensor file + chunk_id→offset index. Lazy loading. Append-only for incremental indexing. -- **Fallback:** Systems without ColBERT dependencies (no torch) fall back to BM25 + FAISS (single-vector with `all-MiniLM-L6-v2`). The retrieval API is identical — callers don't know which reranker is active. When falling back to FAISS, chunk size adjusts to 256 tokens to match the embedding model's context. - -### Attachment Store - -Content-addressed blob store at `~/.openjarvis/blobs/{sha256[:2]}/{sha256}`. A metadata table tracks: hash, filename, mime_type, size_bytes, and source doc_ids. - -Text extraction: PDFs via pdfplumber (existing). Office docs via python-docx/openpyxl. Images skipped for now (future: OCR). Extracted text gets chunked and indexed alongside the parent document. - -## 7. Multi-Layer Retrieval - -Two-stage retrieval as the base primitive, with agent-level multi-hop on top: - -### Stage 1: BM25 Recall - -FTS5 MATCH query with Porter stemming against the `documents_fts` virtual table. Returns top-100 candidates ranked by BM25. Supports pre-filtering by source, doc_type, author, and timestamp range via SQL WHERE clauses before the FTS5 MATCH. - -### Stage 2: ColBERTv2 Rerank - -The top-100 BM25 candidates are reranked using ColBERTv2 late interaction scoring (MaxSim). Returns top-K results (default K=10). This is where semantic understanding kicks in — ColBERT's token-level matching catches results that keyword search alone would miss or misrank. - -### Stage 3: Agent Multi-Hop - -The Deep Research agent can invoke the retrieval primitive (stages 1+2) multiple times across turns, refining its subqueries based on what it's found. Configurable max turns (default: 5 hops). The multi-hop logic lives at the agent level, not the retrieval level — clean separation of concerns. - -## 8. Deep Research Agent - -### Agent Configuration - -```python -@AgentRegistry.register("deep_research") -class DeepResearchAgent(ToolUsingAgent): - agent_id = "deep_research" - # Default model: largest Qwen3.5 that fits locally, or cloud model if configured - # Default max_turns: 5 (multi-hop research turns) -``` - -### Tool Belt - -**`knowledge_search`** (core retrieval tool): BM25→ColBERT with filters: `query`, `source`, `doc_type`, `author`, `since`/`until`, `top_k`. Returns chunks with content, source, author, timestamp, url (deep link), relevance score. - -**MCP connector tools** (real-time data access): Each connected source registers MCP tools. Agent can call these for fresh data not yet in the index — `gmail_search_emails`, `gmail_get_thread`, `slack_search_messages`, `gdrive_get_document`, `calendar_get_events`, `contacts_find`, etc. - -**`think`** (reasoning scratchpad): Existing tool. The agent uses this between retrieval hops to synthesize findings and plan the next subquery. Critical for multi-hop research. - -**`web_search`** (external supplement): Existing tool (DuckDuckGo or Tavily). When personal data references external context (a news article, a product, a company), the agent supplements with public web search. - -### Multi-Hop Research Loop - -Example for "What was the context around the Kubernetes migration decision?": - -``` -Turn 1: think → "I need to find discussions about K8s migration across email and Slack" - knowledge_search(query="Kubernetes migration", top_k=10) → 8 results - -Turn 2: think → "Found a Slack thread from March. Sarah and Mike debated cost. Get full thread + related docs" - knowledge_search(query="Kubernetes cost analysis", author="sarah", doc_type="document") → 3 results - slack_get_thread(thread_ts="1709234567.123") → full thread - -Turn 3: think → "Sarah's doc references a meeting on March 5th. Check calendar + find the proposal" - calendar_get_events(query="Kubernetes", since="2024-03-01") → 1 event - knowledge_search(query="Kubernetes proposal", doc_type="document") → 2 results - -Synthesize: Comprehensive report with cross-platform citations + deep links -``` - -### Output Format - -Structured research report with: -- Narrative synthesis of findings -- Inline citations as clickable deep links back to the original source (Slack thread, Gmail message, Drive doc, Calendar event) -- Source list at the bottom with platform, title, author, and date - -### Conversational Follow-up - -After the initial report, the user can drill deeper or pivot: -- "Show me just Mike's cost concerns in detail" → new retrieval, focused summary -- "Draft a summary of this for my manager" → uses conversation context, no new retrieval needed -- "What happened after the decision?" → new time-scoped retrieval - -## 9. Channel Plugins - -### ChannelAgent - -A lightweight layer between existing channel transports and the Deep Research Agent. Classifies incoming queries automatically and routes them — the user never specifies "quick" or "deep." - -**Classification is automatic** based on query analysis: -- **Quick signals:** Single entity lookup, starts with "when"/"where"/"find", short query (<20 words), mentions a specific person/file/event. -- **Deep signals:** "summarize", "research", "what was the context", time ranges ("last month"), multi-entity queries. -- **Adaptive:** If the agent starts down the quick path but realizes mid-retrieval it needs more hops, it escalates on the fly. - -### Quick Path (Answer Inline) - -Simple lookups answered in a single retrieval hop that fit in a chat message: - -``` -User: When's my next meeting with Sarah? -Jarvis: Tomorrow at 2pm — "Q3 Planning Sync" with Sarah Chen, Mike Ross, and you. - Google Meet link: meet.google.com/abc-defg-hij -``` - -### Deep Path (Escalate to Desktop) - -Complex queries requiring multi-hop retrieval or long-form output: - -``` -User: What was the full context behind the decision to switch to Kubernetes? -Jarvis: That's a complex question spanning multiple sources — I found relevant threads - in Slack, emails from Sarah and Mike, and a proposal doc in Drive. - I'm preparing a full research report for you. - 📄 Open full report in OpenJarvis → openjarvis://research/{session_id} -``` - -Escalation flow: -1. Agent kicks off multi-hop research in the background (doesn't block the chat) -2. Sends brief preview + escalation link in the chat channel -3. When report is ready, sends follow-up notification: "Your report is ready" -4. Deep link (`openjarvis://research/{session_id}`) opens desktop app directly to the research session — Tauri registers as URL handler - -### Per-Channel Implementation - -**iMessage:** BlueBubbles bridge. Currently send-only — needs incoming message support added. Plain text + URL only. Escalation via `openjarvis://` deep link. - -**WhatsApp:** Baileys bridge (existing, already bidirectional). Supports WhatsApp markdown (*bold*, _italic_). Escalation via deep link (WhatsApp renders URLs as tappable). - -**Slack:** Socket Mode (existing, already bidirectional). Can use Block Kit for richer formatting — bullet lists, bold, links, buttons. DM the Jarvis bot or @mention in a channel. - -### Shared Auth - -For WhatsApp and Slack, the data connector and channel plugin share the same authentication. Connecting Slack as a data source automatically enables the Slack channel plugin. One OAuth flow, two capabilities. Channel plugins are configured from a "Connected Apps" settings page after initial setup. - -## 10. CLI: `jarvis connect` - -The CLI equivalent of the desktop setup wizard. Simpler, text-based, but same underlying logic: - -```bash -jarvis connect # Interactive: pick sources, walk through auth -jarvis connect gmail # Connect a specific source -jarvis connect --list # Show connected sources and sync status -jarvis connect --sync # Trigger incremental sync for all sources -jarvis connect --disconnect gmail # Disconnect a source -``` - -OAuth flows open the system browser for consent, then listen on a localhost callback URL. Local access (iMessage, Apple Notes) checks permissions and provides instructions. Filesystem sources (Obsidian) accept a path argument. - -## 11. Implementation Phases - -### Phase 1: Connector Foundation + Gmail + Obsidian -- `BaseConnector` ABC and `ConnectorRegistry` -- `Document` schema and `SyncEngine` (checkpoint/resume, rate limiting) -- Gmail connector (OAuth flow + bulk sync + MCP tools) -- Obsidian/Markdown connector (filesystem, simplest possible) -- Extended SQLite schema with source-aware columns -- `knowledge_search` tool with filtered BM25 retrieval -- CLI `jarvis connect` command -- Tests for all of the above - -### Phase 2: Desktop Wizard + More Connectors -- Desktop setup wizard UI (Steps 1-6) with product logos, progress bars, polished design -- Slack connector -- Google Drive connector -- Google Calendar + Contacts connectors -- iMessage connector (macOS local DB access) -- Parallel ingest dashboard in desktop UI - -### Phase 3: ColBERTv2 Persistence + Deep Research Agent -- ColBERTv2 on-disk persistence (memory-mapped tensors) -- Two-stage retrieval (BM25 recall → ColBERT rerank) -- `DeepResearchAgent` with multi-hop loop -- Research report output with inline citations and deep links -- Conversational follow-up support -- Desktop app research UI (report rendering, source panel) - -### Phase 4: Channel Plugins + Remaining Connectors -- `ChannelAgent` with automatic quick/deep classification -- iMessage channel plugin (add incoming message support to BlueBubbles) -- WhatsApp channel plugin (leverage existing Baileys bridge) -- Slack channel plugin (leverage existing Socket Mode) -- `openjarvis://` URL handler registration in Tauri -- Remaining connectors: WhatsApp, Dropbox, Notion, Apple Notes - -### Phase 5: Polish + Incremental Sync -- Incremental delta sync for all connectors -- Attachment store with content-addressed dedup and text extraction -- Desktop "Connected Apps" settings page (manage sources post-setup) -- Sync status dashboard (last sync time, item counts, errors) -- Error handling and retry UX across all connectors diff --git a/docs/superpowers/specs/2026-03-26-deep-research-agent-v2-design.md b/docs/superpowers/specs/2026-03-26-deep-research-agent-v2-design.md deleted file mode 100644 index 4b238c71..00000000 --- a/docs/superpowers/specs/2026-03-26-deep-research-agent-v2-design.md +++ /dev/null @@ -1,117 +0,0 @@ -# Deep Research Agent v2 — Better Tools + Prompts - -## Goal - -Improve the DeepResearchAgent so it can answer aggregation queries ("who do I talk to most"), semantic/fuzzy queries ("which VCs have I spoken with"), and factual queries ("when was my trip to Spain") by adding SQL and LM-scan tools alongside better prompts. - -## Evaluation Criteria - -Re-run these 4 queries and compare to v1 (which returned empty or "no data found" for all 4): - -1. "When was my most recent trip to Spain?" -2. "Which VCs have I spoken with since 2023?" -3. "Who are the 10 people I have spoken with the most over text?" -4. "What meetings take up most of my time based on my calendar and meeting logs?" - -Success = the agent produces substantive, cited answers for at least 3 of 4. - -## Changes - -### 1. New tool: `knowledge_sql` - -Read-only SQL queries against the `knowledge_chunks` table. The agent writes SELECT statements to aggregate, count, filter, and rank data. - -```python -class KnowledgeSQLTool(BaseTool): - tool_id = "knowledge_sql" - - def execute(self, query: str) -> ToolResult: - # Reject non-SELECT queries - # Execute against KnowledgeStore's SQLite connection - # Return formatted rows (max 50 rows) -``` - -Exposed schema (included in the tool description so the model knows the columns): -``` -knowledge_chunks(id, content, source, doc_type, doc_id, title, author, - participants, timestamp, thread_id, url, metadata, chunk_index) -``` - -### 2. New tool: `scan_chunks` - -Semantic grep — pulls chunks by filter, batches through the LM with a question. - -```python -class ScanChunksTool(BaseTool): - tool_id = "scan_chunks" - - def execute(self, question: str, source: str = "", doc_type: str = "", - since: str = "", until: str = "", max_chunks: int = 200, - batch_size: int = 20) -> ToolResult: - # 1. Pull chunks matching filters from KnowledgeStore - # 2. Batch into groups of batch_size - # 3. For each batch, call engine.generate(): - # "Extract info relevant to: {question}\n\nChunks:\n{batch_text}" - # 4. Return aggregated findings -``` - -This is the "LM combs through tokens" capability — catches semantic matches that BM25 keyword search misses. - -### 3. Wire `think` tool - -The `think` tool already exists at `src/openjarvis/tools/think.py`. Wire it into the DeepResearchAgent's tool list in `deep_research_setup_cmd.py`. - -### 4. System prompt rewrite - -Replace the current generic prompt with one that teaches query strategies: - -``` -/no_think -You are a deep research agent with access to a personal knowledge base -containing emails, messages, meeting notes, documents, and notes. - -## Your Tools - -- **knowledge_search**: BM25 keyword search. Best for finding specific topics, - names, or phrases. Use filters (source, author, since, until) to narrow results. - -- **knowledge_sql**: Run SQL queries against the knowledge_chunks table. - Schema: knowledge_chunks(id, content, source, doc_type, doc_id, title, - author, participants, timestamp, thread_id, url, metadata, chunk_index) - Best for: counting, ranking, aggregation, time-range queries. - Example: SELECT author, COUNT(*) as n FROM knowledge_chunks WHERE source='imessage' GROUP BY author ORDER BY n DESC LIMIT 10 - -- **scan_chunks**: Semantic search — feeds chunks to an LM that reads the actual - text looking for relevant information. Use when keyword search misses semantic - matches (e.g. searching for "VCs" when text says "fundraising round"). - Slower but catches what BM25 misses. - -- **think**: Reasoning scratchpad. Use between searches to plan your next query, - evaluate findings, and identify gaps. - -## Strategy - -1. Start with **think** to plan your approach — which tools suit this query? -2. For "who/what/how many" queries → start with **knowledge_sql** -3. For specific topics → start with **knowledge_search** -4. If keyword search returns nothing useful → try **scan_chunks** with broader filters -5. Cross-reference across sources — search emails, then messages, then notes -6. After gathering evidence → write a cited narrative report - -## Citation Format - -Cite sources as: [source] title -- author -Include a Sources section at the end. -``` - -### 5. Loop guard tuning - -Increase the `max_turns` default from 5 to 8, and configure the loop guard (if present) with a higher poll budget to accommodate the larger tool set. - -## What Does NOT Change - -- Agent class structure (DeepResearchAgent) -- Retriever pipeline (BM25 + optional ColBERT) -- Ingestion pipeline -- Connectors -- CLI command diff --git a/docs/superpowers/specs/2026-03-26-deep-research-vertical-slice-design.md b/docs/superpowers/specs/2026-03-26-deep-research-vertical-slice-design.md deleted file mode 100644 index 56b8cc20..00000000 --- a/docs/superpowers/specs/2026-03-26-deep-research-vertical-slice-design.md +++ /dev/null @@ -1,101 +0,0 @@ -# Deep Research Vertical Slice — End-to-End on Laptop - -## Goal - -Get the full Deep Research experience working end-to-end on a MacBook with real personal data: connect local sources → ingest → retrieve → research agent → cited report. Then wire the API router so the desktop wizard UI works too. - -## Scope - -**In scope:** -- Fix Apple Notes protobuf text extraction + broken test -- `jarvis deep-research-setup` CLI command that auto-detects and ingests local sources -- Ingest 3 local sources into shared KnowledgeStore (~/.openjarvis/knowledge.db): - - Apple Notes (42 notes, ~100 chunks) - - iMessage (52K messages, ~52K chunks) - - Obsidian vault (size varies) -- Run TwoStageRetriever queries (BM25 + optional ColBERT rerank) against combined corpus -- Run DeepResearchAgent with Qwen3.5 4B via Ollama — multi-hop research producing cited reports -- Wire connectors_router into FastAPI app (one-line fix) -- Smoke test wizard UI via browser - -**Out of scope:** -- OAuth connectors (Gmail, Drive, Calendar, Contacts, Slack, Notion, Dropbox) -- Channel plugins (iMessage/WhatsApp/Slack bot) -- Sync scheduling, performance optimization -- ColBERT disk persistence tuning - -## Architecture - -### Data Flow - -``` -Local Sources (no OAuth) - Apple Notes → NoteStore.sqlite → protobuf → plain text - iMessage → chat.db → messages with contact/conversation metadata - Obsidian → filesystem → .md files - ↓ -Connectors (AppleNotesConnector, IMessageConnector, ObsidianConnector) - yield Document(doc_id, source, doc_type, content, title, author, timestamp, ...) - ↓ -IngestionPipeline - Dedup by doc_id → SemanticChunker (512 token chunks) → KnowledgeStore - ↓ -KnowledgeStore (~/.openjarvis/knowledge.db) - SQLite + FTS5 (BM25), ~50K-55K chunks - ↓ -TwoStageRetriever - Stage 1: BM25 recall (top 100) with source/type/date filters - Stage 2: ColBERT rerank (top 10) — optional, falls back to BM25-only - ↓ -DeepResearchAgent (Qwen3.5 4B via Ollama) - Multi-hop loop (up to 5 turns): - think → knowledge_search → refine → knowledge_search → ... → synthesize - Output: cited research report with source attribution -``` - -### CLI Command: `jarvis deep-research-setup` - -Interactive CLI command that: -1. Auto-detects available local sources (checks if NoteStore.sqlite, chat.db exist; prompts for Obsidian vault path or skips if none) -2. Confirms with user which sources to connect -3. Syncs each source via SyncEngine + IngestionPipeline into ~/.openjarvis/knowledge.db -4. Prints summary (sources, document count, chunk count) -5. Drops user into `jarvis chat` with DeepResearchAgent configured - -### LLM Configuration - -- Engine: Ollama -- Model: qwen3.5:4b (pulled via `ollama pull qwen3.5:4b`) -- Agent: DeepResearchAgent with knowledge_search + think tools -- Expected latency: fast token generation on Apple Silicon, main cost is ColBERT model loading on first query - -### API Router Fix - -The `connectors_router.py` file exists with all endpoints but was never registered in the FastAPI app. Fix: add `app.include_router(create_connectors_router())` in `app.py` or `api_routes.py`. - -## Test Plan - -### Retrieval Quality Checks -1. Name search spanning Apple Notes + iMessage (person who appears in both) -2. Topical search across Obsidian vault -3. Time-bounded query ("what was I working on last week") - -### Agent Quality Checks -1. Cross-source research query requiring multi-hop -2. Check that citations point back to real source documents -3. Check that agent uses multiple search refinements (not just one query) - -### Wizard Smoke Test -1. `jarvis serve` → open localhost in browser -2. Walk through wizard: pick sources → connect → ingest → ready screen -3. Verify sync status polling shows progress -4. Verify "Ready" screen appears with suggestion queries - -## Risks - -| Risk | Impact | Mitigation | -|------|--------|------------| -| 4B model too small for multi-hop reasoning | Agent produces shallow one-hop reports | Swap to larger Ollama model or cloud engine | -| 52K iMessage chunks slow to ingest | Long initial setup time | Show progress, allow incremental use | -| ColBERT model loading slow on first query | Bad first-query UX | Fall back to BM25-only, lazy load ColBERT | -| Apple Notes protobuf extraction still garbled for some notes | Missing or corrupted content | Graceful degradation, skip notes with no extractable text | diff --git a/docs/superpowers/specs/2026-03-26-outlook-connector-and-token-sources-design.md b/docs/superpowers/specs/2026-03-26-outlook-connector-and-token-sources-design.md deleted file mode 100644 index 6606ca00..00000000 --- a/docs/superpowers/specs/2026-03-26-outlook-connector-and-token-sources-design.md +++ /dev/null @@ -1,123 +0,0 @@ -# Outlook Connector + Token-Based Source Integration - -## Goal - -Generalize the Gmail IMAP connector to support Outlook/Microsoft 365, and extend `jarvis deep-research-setup` to detect and connect token-based sources (Slack, Notion, Granola, Gmail IMAP, Outlook) alongside the existing local sources. - -## Scope - -**In scope:** -- Generalize `gmail_imap.py` to accept an `imap_host` parameter (default `imap.gmail.com`) -- Create `outlook.py` as a thin subclass with `outlook.office365.com` default -- Add `detect_token_sources()` to `deep_research_setup_cmd.py` — scans `~/.openjarvis/connectors/*.json` for already-connected token-based sources -- Add interactive prompts in `deep-research-setup` to connect new token-based sources (Slack, Notion, Granola, Gmail IMAP, Outlook) -- Extend `_instantiate_connector()` to handle all connector types -- Connect and ingest all sources in one unified flow - -**Out of scope:** -- Google OAuth token exchange (Drive, Calendar, Contacts still blocked) -- Microsoft Graph API -- Channel plugins (Slack bot, etc.) - -## Architecture - -### IMAP Generalization - -`GmailIMAPConnector` gains one new parameter: - -```python -class GmailIMAPConnector(BaseConnector): - _default_imap_host = "imap.gmail.com" - - def __init__(self, ..., imap_host: str = "") -> None: - self._imap_host = imap_host or self._default_imap_host -``` - -The `sync()` method changes from `imaplib.IMAP4_SSL("imap.gmail.com")` to `imaplib.IMAP4_SSL(self._imap_host)`. - -### Outlook Connector - -A thin subclass in `outlook.py`: - -```python -@ConnectorRegistry.register("outlook") -class OutlookConnector(GmailIMAPConnector): - connector_id = "outlook" - display_name = "Outlook / Microsoft 365" - _default_imap_host = "outlook.office365.com" - - def __init__(self, email_address="", app_password="", credentials_path="", *, max_messages=500): - super().__init__(email_address, app_password, - credentials_path or str(DEFAULT_CONFIG_DIR / "connectors" / "outlook.json"), - max_messages=max_messages) - - def auth_url(self) -> str: - return "https://account.microsoft.com/security" - - def sync(self, *, since=None, cursor=None) -> Iterator[Document]: - # Delegate to parent but override source/doc_id prefix - for doc in super().sync(since=since, cursor=cursor): - doc.source = "outlook" - doc.doc_id = doc.doc_id.replace("gmail:", "outlook:", 1) - yield doc -``` - -### Token Source Detection in deep-research-setup - -New function `detect_token_sources()`: - -```python -_TOKEN_SOURCES = [ - {"connector_id": "gmail_imap", "display_name": "Gmail (IMAP)", "creds_file": "gmail_imap.json"}, - {"connector_id": "outlook", "display_name": "Outlook", "creds_file": "outlook.json"}, - {"connector_id": "slack", "display_name": "Slack", "creds_file": "slack.json"}, - {"connector_id": "notion", "display_name": "Notion", "creds_file": "notion.json"}, - {"connector_id": "granola", "display_name": "Granola", "creds_file": "granola.json"}, -] -``` - -For each: check if `~/.openjarvis/connectors/{creds_file}` exists with valid content. If so, include it as an already-connected source. - -### Interactive Connect Flow - -For sources not yet connected, `deep-research-setup` offers to connect them: - -``` -Detected Sources: - Apple Notes ready (local) - iMessage ready (local) - -Available to Connect: - Gmail (IMAP) not connected - Outlook not connected - Slack not connected - Notion not connected - Granola not connected - -Connect additional sources? [y/N]: y -Which source? [gmail_imap/outlook/slack/notion/granola]: slack -Paste your Slack token (xoxb-... or xoxe-...): xoxe-1-... - Slack: connected! - -Connect another? [y/N]: y -... -``` - -Each connector's `handle_callback()` saves the credential. Then all connected sources are ingested together. - -### Connector Credential Formats - -| Connector | Credential Format | `handle_callback` Input | -|-----------|------------------|------------------------| -| Gmail IMAP | `{"email": "...", "password": "..."}` | `email:password` | -| Outlook | `{"email": "...", "password": "..."}` | `email:password` | -| Slack | `{"token": "..."}` | Raw token | -| Notion | `{"token": "..."}` | Raw token | -| Granola | `{"token": "..."}` | Raw token | - -## Test Plan - -- Test `OutlookConnector` inherits IMAP behavior with fake IMAP server (same pattern as gmail_imap tests) -- Test `detect_token_sources()` finds connected sources from credential files -- Test `detect_token_sources()` skips sources with missing/empty credential files -- Test interactive connect saves credentials correctly diff --git a/docs/superpowers/specs/2026-03-27-channels-and-messaging-tabs-design.md b/docs/superpowers/specs/2026-03-27-channels-and-messaging-tabs-design.md deleted file mode 100644 index 84999969..00000000 --- a/docs/superpowers/specs/2026-03-27-channels-and-messaging-tabs-design.md +++ /dev/null @@ -1,85 +0,0 @@ -# Channels + Messaging Tabs — Unified Agent Setup - -## Goal - -Replace the current disconnected setup experience with two new tabs on every agent detail page: **Channels** (data sources the agent can search) and **Messaging** (ways to talk to the agent from your phone). Clicking "+ Add" on an unconnected source opens step-by-step setup inline. - -## Tab Structure - -``` -Overview | Interact | Channels | Messaging | Tasks | Memory | Learning | Logs -``` - -### Channels Tab - -Shows all data source connectors with their connection status and chunk counts. - -**Connected sources** — green border, checkmark, chunk count: -- Gmail (1,076 chunks) ✓ -- iMessage (52,956 chunks) ✓ -- Granola (1,144 chunks) ✓ -- etc. - -**Not connected** — dashed border, "+ Add" button: -- Google Drive — Not connected [+ Add] -- Calendar — Not connected [+ Add] -- Outlook — Not connected [+ Add] - -Clicking "+ Add" expands inline with the StepByStepPanel (per-connector instructions, links to settings pages, input fields for credentials). Same flow currently in the setup wizard but embedded in the tab. - -**Data shown per connected source:** -- Icon + name -- Chunk count (from KnowledgeStore) -- Connection status -- Disconnect option (on hover/click) - -### Messaging Tab - -Shows ways to reach the agent from your phone or other platforms. - -**Active channels** — green border, clear instructions: -- iMessage: "Text +1 (408) 981-9553 from your iPhone" [Active] -- Slack: "DM @jarvis in your workspace" [Active] - -**Not set up** — dashed border, "Set Up" button: -- WhatsApp — Not set up [Set Up] -- SMS (Twilio) — Not set up [Set Up] - -The key UX principle: **tell the user what to do, don't ask them for config.** Active channels show the action ("Text this number"), not the plumbing ("Enter phone number to monitor"). - -### What Changes vs Current - -| Before | After | -|--------|-------| -| Separate "Channels" tab with confusing "phone number to monitor" prompt | **Messaging** tab with clear "Text this number" instructions | -| Setup wizard only during onboarding, hidden after | **Channels** tab shows all sources with inline "+ Add" | -| No way to see connected data sources on agent page | **Channels** tab shows all with chunk counts | -| Connector setup disconnected from agent | Unified on the agent page | - -### What Does NOT Change - -- Overview tab (stats, config, instruction) -- Interact tab (chat) -- Tasks, Memory, Learning, Logs tabs -- Backend API endpoints (already exist) -- Connector sync logic -- iMessage daemon / ChannelBridge / webhook routes - -### Backend Requirements - -One new endpoint needed: - -`GET /v1/connectors/status` — returns all connectors with connection status AND chunk counts from KnowledgeStore. The Channels tab calls this to populate the grid. - -Everything else (bind/unbind channel, connect/disconnect source, sync status) already exists. - -## Test Plan - -- Channels tab shows connected sources with chunk counts -- Channels tab shows unconnected sources with "+ Add" -- Clicking "+ Add" shows step-by-step instructions inline -- Completing setup adds the source and starts ingestion -- Messaging tab shows active channels with clear action text -- Messaging tab shows "Set Up" for inactive channels -- Setting up iMessage starts the daemon -- Both tabs appear on ALL agents, not just DeepResearch diff --git a/docs/superpowers/specs/2026-03-27-channels-tab-and-connector-setup-design.md b/docs/superpowers/specs/2026-03-27-channels-tab-and-connector-setup-design.md deleted file mode 100644 index 0c167a94..00000000 --- a/docs/superpowers/specs/2026-03-27-channels-tab-and-connector-setup-design.md +++ /dev/null @@ -1,120 +0,0 @@ -# Channels Tab + Seamless Connector Setup - -## Goal - -Add a Channels tab to every agent for bidirectional messaging (iMessage, Slack, WhatsApp, SMS), and build per-connector step-by-step setup guides in the wizard with one-click OAuth where available. - -## Sub-project 1: Channels Tab on Every Agent - -### What it does - -A new "Channels" tab on the agent detail page showing messaging interfaces to reach the agent. Users can connect/disconnect channels from the UI. Appears on ALL agents. - -### Frontend - -Add a 7th tab "Channels" to the agent detail view in `AgentsPage.tsx`. Content: - -- Header: "Talk to this agent from" + "+ Add Channel" button -- Channel list: iMessage, Slack, WhatsApp, SMS (Twilio) -- Each channel shows: icon, name, description, status badge (Active/Not connected) -- Connected channels expand to show: monitored contact/workspace, instructions, Stop/Settings buttons -- Disconnected channels show a "Connect" button - -**"+ Add Channel" / "Connect" flow:** -- Opens a modal/inline form specific to the channel type: - - **iMessage:** prompt for phone number or contact to monitor → starts daemon - - **Slack:** prompt for bot token → registers webhook - - **WhatsApp:** prompt for access token + phone number ID → registers webhook - - **SMS:** prompt for Twilio account SID + auth token + phone number → registers webhook - -### Backend - -- `POST /v1/managed-agents/{id}/channels` — bind a channel (creates binding + starts daemon/webhook) -- `DELETE /v1/managed-agents/{id}/channels/{binding_id}` — unbind (stops daemon/webhook) -- `AgentManager.add_channel_binding()` already exists — wire it to the new endpoint -- For iMessage: the endpoint starts/stops the `imessage_daemon` via the existing `run_daemon`/`stop_daemon` functions -- For Slack/WhatsApp/SMS: the endpoint updates `ChannelBridge` channel config - -### No new daemon code - -`imessage_daemon.py`, `ChannelBridge`, `webhook_routes.py`, and `channels_cmd.py` already exist. This sub-project wires them to the frontend UI. - ---- - -## Sub-project 2: Seamless Connector Setup - -### What it does - -Replace the current generic "paste token" connect flow with per-connector step-by-step guides. Two patterns: - -**Pattern A — Step-by-step guide (for token/password auth):** -Numbered steps with direct links to settings pages, then paste credentials. - -**Pattern B — One-click OAuth + manual fallback:** -Primary "Authorize" button opens the service in browser. Manual token paste as secondary option. - -### Per-connector flows - -| Connector | Pattern | UX | -|-----------|---------|-----| -| Gmail IMAP | A | Steps: enable 2FA → generate app password → paste email:password | -| Outlook | A | Steps: enable 2FA → generate app password → paste email:password | -| Slack | B | "Open Slack App Settings" button → paste bot token (xoxb-...) | -| Notion | B | "Open Notion Integrations" button → paste token (ntn_...) → remind to share pages | -| Granola | A | Steps: open Granola app → Settings → API → paste key (grn_...) | -| Google Drive | B | "Authorize" opens OAuth consent URL → localhost callback captures code → exchanges for tokens | -| Google Calendar | B | Same as Drive (shared OAuth client, different scope) | -| Google Contacts | B | Same as Drive (shared OAuth client, different scope) | -| Obsidian | Filesystem | Path input field | -| Apple Notes | Auto | Check Full Disk Access, auto-connect | -| iMessage | Auto | Check Full Disk Access, auto-connect | - -### Google OAuth proper flow (new) - -The current Gmail OAuth connector saves the raw auth code without exchanging it. Fix: - -1. Add `OAuthCallbackServer` — a tiny localhost HTTP server at `http://localhost:8789/callback` that: - - Waits for Google to redirect with `?code=...` - - Exchanges the code for access_token + refresh_token via `POST https://oauth2.googleapis.com/token` - - Stores tokens in `~/.openjarvis/connectors/{connector}.json` - - Returns a "Success! You can close this tab" page to the browser - -2. Update `oauth.py` with `exchange_google_token(code, client_id, client_secret, redirect_uri)` function - -3. Update `handle_callback()` in Gmail, Drive, Calendar, Contacts connectors to use the exchange flow - -4. Google OAuth client ID/secret stored in `~/.openjarvis/config.toml` under `[connectors.google]` — user provides their own from Google Cloud Console - -### Frontend changes - -In `SourceConnectFlow.tsx`, replace the generic OAuth panel with connector-specific panels: - -- Each connector gets its own instructions component -- Step-by-step connectors show numbered cards with links -- OAuth connectors show the "Authorize" button + manual fallback input -- All connectors show a "Read-only access · No data leaves your device" badge - -### What does NOT change - -- SourcePicker (already lists all connectors) -- IngestDashboard (already shows progress) -- ReadyScreen (already shows suggestions) -- Backend connector sync logic (already works) -- KnowledgeStore / ingestion pipeline - -## Test Plan - -### Channels Tab -- Channels tab appears on all agents (not just DeepResearch) -- Binding iMessage channel starts daemon and shows "Active" status -- Unbinding stops daemon -- Channel status updates in real-time -- Multiple agents can bind different channels - -### Connector Setup -- Gmail IMAP: step-by-step flow → paste credentials → connected -- Notion: one-click opens integration page → paste token → connected -- Slack: paste bot token → connected -- Google Drive: OAuth consent URL opens → localhost callback exchanges token → connected -- Setup wizard completes with all connected sources ingested -- Read-only guarantee maintained across all connectors diff --git a/docs/superpowers/specs/2026-03-27-personal-deep-research-template-design.md b/docs/superpowers/specs/2026-03-27-personal-deep-research-template-design.md deleted file mode 100644 index c198bc40..00000000 --- a/docs/superpowers/specs/2026-03-27-personal-deep-research-template-design.md +++ /dev/null @@ -1,162 +0,0 @@ -# Personal DeepResearch Agent Template — Desktop + Browser + CLI - -## Goal - -Register "Personal Deep Research" as a default agent template so users can create it from the Agents tab in the desktop/browser app, chat with it via streaming SSE, and use it from the CLI. All read-only — no writes to any connector. - -## Scope - -**In scope:** -- New template TOML: `personal_deep_research.toml` -- Wire DeepResearch tools in `_stream_managed_agent()` when `agent_type == "deep_research"` -- Ensure KnowledgeStore at `~/.openjarvis/knowledge.db` is loaded when the agent runs -- Verify end-to-end: create from template in Agents tab → send message → get streamed cited report - -**Out of scope:** -- Google OAuth (Sub-project 2) -- Channel listeners (Sub-project 3) -- New frontend components (existing Agents tab + chat UI work as-is) - -## Architecture - -### Template File - -`src/openjarvis/agents/templates/personal_deep_research.toml`: - -```toml -[template] -id = "personal_deep_research" -name = "Personal Deep Research" -description = "Search across your emails, messages, meeting notes, and documents with multi-hop retrieval and cited reports. Uses BM25 keyword search, SQL aggregation, and LM-powered semantic scanning." -agent_type = "deep_research" -schedule_type = "manual" -tools = ["knowledge_search", "knowledge_sql", "scan_chunks", "think"] -max_turns = 8 -temperature = 0.3 -max_tokens = 4096 -``` - -This is auto-discovered by `AgentManager.list_templates()` and appears in the Agents tab wizard alongside the existing templates (code_reviewer, inbox_triager, research_monitor). - -### Server-Side Tool Wiring - -In `_stream_managed_agent()` (agent_manager_routes.py), when the agent type is `deep_research`, build the tools from the shared KnowledgeStore: - -```python -if agent_record["agent_type"] == "deep_research": - from openjarvis.connectors.store import KnowledgeStore - from openjarvis.connectors.retriever import TwoStageRetriever - from openjarvis.tools.knowledge_search import KnowledgeSearchTool - from openjarvis.tools.knowledge_sql import KnowledgeSQLTool - from openjarvis.tools.scan_chunks import ScanChunksTool - from openjarvis.tools.think import ThinkTool - - store = KnowledgeStore() # defaults to ~/.openjarvis/knowledge.db - retriever = TwoStageRetriever(store) - model = agent_record.get("config", {}).get("model") or engine_model - tools = [ - KnowledgeSearchTool(retriever=retriever), - KnowledgeSQLTool(store=store), - ScanChunksTool(store=store, engine=engine, model=model), - ThinkTool(), - ] -``` - -These tools are passed to `DeepResearchAgent(engine=engine, model=model, tools=tools)` when instantiating the agent for this request. - -### Flow - -``` -User clicks "Personal Deep Research" template in Agents tab - → POST /v1/managed-agents {template_id: "personal_deep_research", name: "My Research"} - → AgentManager.create_from_template() → stores agent record in SQLite - -User types question in chat - → POST /v1/managed-agents/{id}/messages {content: "...", stream: true} - → _stream_managed_agent() detects agent_type == "deep_research" - → Builds 4 tools from KnowledgeStore - → Instantiates DeepResearchAgent with tools - → agent.run(query) → multi-hop tool calls - → Streams response as SSE word-by-word - → Frontend renders streamed markdown with citations -``` - -### CLI Alias - -Add `jarvis research` as a convenience alias that runs `jarvis deep-research-setup`. Same behavior, shorter command. - -### Multi-Channel Messaging — iMessage, Slack, WhatsApp, SMS - -Users can message their DeepResearch agent from their phone via iMessage, Slack, WhatsApp, or SMS. This builds on the `ChannelBridge` + webhook architecture from PR #78 (open-jarvis/OpenJarvis#78). - -**Three options for iMessage (user picks during setup):** - -1. **AppleScript daemon (free, no external services)** — Background daemon on the Mac polls `chat.db` for new messages in a designated conversation, routes to agent, responds via AppleScript. Requires Mac to be running. - -2. **BlueBubbles (free, self-hosted)** — Install BlueBubbles on the Mac, which provides a REST API + webhook for iMessage. PR #78 already has webhook handler at `/webhooks/bluebubbles`. More reliable than AppleScript, supports media. - -3. **Sendblue (paid cloud API)** — Cloud iMessage API, no Mac required for sending. Works from any server. Costs money but no local dependencies. We'd add a Sendblue webhook handler alongside the existing BlueBubbles one. - -**Architecture (from PR #78):** - -``` -Phone sends message - → Webhook hits /webhooks/{twilio|bluebubbles|whatsapp} - → ChannelBridge.handle_incoming(sender, content, channel_type) - → SessionStore tracks per-sender conversation - → Routes to DeepResearchAgent.run(query) - → Response sent back via channel adapter (same channel) -``` - -For the AppleScript daemon (option 1), the flow is: -``` -iPhone sends iMessage → Mac Messages.app → chat.db - → iMessageDaemon polls chat.db (read-only) - → ChannelBridge.handle_incoming(sender, content, "imessage") - → DeepResearchAgent.run(query) - → AppleScript sends response via Messages.app - → Response appears on iPhone -``` - -**What we build (from PR #78 + new):** -- Merge/rebase PR #78's `ChannelBridge`, `SessionStore`, `webhook_routes`, `auth_middleware` -- Add AppleScript-based iMessage daemon as an alternative to BlueBubbles -- Add `jarvis channels setup` CLI command to configure channel(s) -- Add `jarvis channels start` / `stop` / `status` for daemon lifecycle -- Wire `ChannelBridge` to route to DeepResearchAgent instead of generic `JarvisSystem.ask()` - -**Finding the agent address (shown in CLI + Desktop + Browser):** - -| Channel | How user finds the address | -|---------|---------------------------| -| iMessage (AppleScript) | `jarvis channels status` → "iMessage: listening on [chat name]" | -| iMessage (BlueBubbles) | `jarvis channels status` → "iMessage: via BlueBubbles at [url]" | -| Slack | Agent detail → Channels tab → shows Slack workspace + channel | -| WhatsApp | Agent detail → Channels tab → shows WhatsApp number | -| SMS (Twilio) | Agent detail → Channels tab → shows Twilio phone number | - -**Read-only guarantee:** All channel adapters only READ incoming messages and SEND responses. No channel adapter modifies, deletes, or marks messages as read. The iMessage daemon reads chat.db in read-only mode (`?mode=ro`). Responses go through Messages.app (AppleScript) or BlueBubbles API, not by writing to chat.db. - -## What Does NOT Change - -- Frontend Agents tab UI (already supports templates) -- Frontend chat components (already support SSE streaming) -- Agent CRUD endpoints (already work) -- DeepResearchAgent class (already has all 4 tools + prompt) -- KnowledgeStore / ingestion pipeline (already populated) -- iMessage connector (read-only data source — separate from the iMessage agent channel) - -## Test Plan - -- Template appears in `GET /v1/templates` response -- Creating from template succeeds via `POST /v1/managed-agents` -- Sending a message with `stream: true` returns SSE chunks -- Agent uses knowledge_search/knowledge_sql/scan_chunks/think tools -- Response includes citations -- `jarvis research` CLI alias works -- iMessage daemon detects new messages in designated chat -- iMessage daemon sends response via AppleScript -- `jarvis channels status` shows correct state per channel -- ChannelBridge routes incoming webhook to DeepResearchAgent -- Webhook endpoints return correct HTTP responses (TwiML for Twilio, 200 for BlueBubbles/WhatsApp) -- SessionStore tracks per-sender conversations across channels diff --git a/frontend/src/components/Chat/AudioPlayer.tsx b/frontend/src/components/Chat/AudioPlayer.tsx new file mode 100644 index 00000000..15d7f057 --- /dev/null +++ b/frontend/src/components/Chat/AudioPlayer.tsx @@ -0,0 +1,119 @@ +import { useRef, useState, useEffect, useCallback } from 'react'; +import { Play, Pause, Volume2 } from 'lucide-react'; + +interface AudioPlayerProps { + src: string; +} + +function formatTime(seconds: number): string { + const m = Math.floor(seconds / 60); + const s = Math.floor(seconds % 60); + return `${m}:${s.toString().padStart(2, '0')}`; +} + +export function AudioPlayer({ src }: AudioPlayerProps) { + const audioRef = useRef(null); + const [playing, setPlaying] = useState(false); + const [currentTime, setCurrentTime] = useState(0); + const [duration, setDuration] = useState(0); + + const toggle = useCallback(() => { + const el = audioRef.current; + if (!el) return; + if (playing) { + el.pause(); + } else { + el.play(); + } + setPlaying(!playing); + }, [playing]); + + useEffect(() => { + const el = audioRef.current; + if (!el) return; + + const onTime = () => setCurrentTime(el.currentTime); + const onMeta = () => setDuration(el.duration); + const onEnded = () => { + setPlaying(false); + setCurrentTime(0); + }; + + el.addEventListener('timeupdate', onTime); + el.addEventListener('loadedmetadata', onMeta); + el.addEventListener('ended', onEnded); + return () => { + el.removeEventListener('timeupdate', onTime); + el.removeEventListener('loadedmetadata', onMeta); + el.removeEventListener('ended', onEnded); + }; + }, []); + + const progress = duration > 0 ? (currentTime / duration) * 100 : 0; + + const seek = (e: React.MouseEvent) => { + const el = audioRef.current; + if (!el || !duration) return; + const rect = e.currentTarget.getBoundingClientRect(); + const pct = (e.clientX - rect.left) / rect.width; + el.currentTime = pct * duration; + }; + + return ( +
+
)} + {/* Audio player (e.g. morning digest) */} + {message.audio?.url && } + {/* Assistant message */} {cleanContent && (
diff --git a/frontend/src/components/setup/SourceConnectFlow.tsx b/frontend/src/components/setup/SourceConnectFlow.tsx index b977b275..c5a5b7bb 100644 --- a/frontend/src/components/setup/SourceConnectFlow.tsx +++ b/frontend/src/components/setup/SourceConnectFlow.tsx @@ -8,7 +8,8 @@ import { Loader2, } from 'lucide-react'; import { SOURCE_CATALOG } from '../../types/connectors'; -import { connectSource } from '../../lib/connectors-api'; +import { connectSource, getConnector } from '../../lib/connectors-api'; +import { getBase } from '../../lib/api'; import type { ConnectRequest, ConnectorMeta } from '../../types/connectors'; // --------------------------------------------------------------------------- @@ -138,89 +139,71 @@ function FilesystemPanel({ function OAuthPanel({ displayName, authUrl, + connectorId, onConnect, onSkip, isConnecting, }: { displayName: string; authUrl?: string; + connectorId: string; onConnect: (req: ConnectRequest) => void; onSkip: () => void; isConnecting: boolean; }) { - const [token, setToken] = useState(''); - const [phase, setPhase] = useState<'start' | 'paste'>('start'); + const [waiting, setWaiting] = useState(false); - const openBrowser = () => { - if (authUrl) { - window.open(authUrl, '_blank'); - } - setPhase('paste'); + const startOAuth = () => { + // Open the server's OAuth start endpoint which redirects to the provider + const oauthUrl = `${getBase()}/v1/connectors/${encodeURIComponent(connectorId)}/oauth/start`; + window.open(oauthUrl, '_blank', 'width=600,height=700'); + setWaiting(true); + + // Poll for connection status + const interval = setInterval(async () => { + try { + const info = await getConnector(connectorId); + if (info.connected) { + clearInterval(interval); + setWaiting(false); + onConnect({}); + } + } catch { + // ignore polling errors + } + }, 2000); + + // Stop polling after 3 minutes + setTimeout(() => { + clearInterval(interval); + setWaiting(false); + }, 180000); }; return (
- {phase === 'start' ? ( - <> -

- Authorize OpenJarvis to access your {displayName} account. -

- - +

+ {waiting + ? `Waiting for ${displayName} authorization... Complete it in the browser window.` + : `Connect your ${displayName} account with one click.`} +

+ {waiting ? ( +
+ + Waiting for authorization... +
) : ( - <> -

- After authorizing, paste the token or code below. -

-