diff --git a/.github/workflows/e2e-reusable.yml b/.github/workflows/e2e-reusable.yml index 4f11b67fe..87f4796ab 100644 --- a/.github/workflows/e2e-reusable.yml +++ b/.github/workflows/e2e-reusable.yml @@ -28,8 +28,7 @@ on: workflow_call: inputs: ref: - description: - Git ref (tag or SHA) to test. Release workflows pass the + description: Git ref (tag or SHA) to test. Release workflows pass the freshly-pushed staging/production tag here so pretest validates the exact commit the build matrix will check out, not main HEAD at workflow_dispatch time. Defaults to empty (checkout uses its @@ -379,8 +378,8 @@ jobs: image: ghcr.io/tinyhumansai/openhuman_ci:latest timeout-minutes: 60 env: - CARGO_INCREMENTAL: '0' - RUST_BACKTRACE: '1' + CARGO_INCREMENTAL: "0" + RUST_BACKTRACE: "1" steps: - name: Checkout code uses: actions/checkout@v5 @@ -612,7 +611,9 @@ jobs: shard: - { name: foundation, suites: "auth,navigation,system" } - { name: chat, suites: "chat,skills,journeys" } - - { name: integrations, suites: "providers,webhooks,notifications" } + - { name: providers, suites: "providers,notifications" } + - { name: webhooks, suites: "webhooks" } + - { name: connectors, suites: "connectors" } - { name: commerce, suites: "payments,settings" } steps: - name: Checkout code @@ -758,7 +759,9 @@ jobs: shard: - { name: foundation, suites: "auth,navigation,system" } - { name: chat, suites: "chat,skills,journeys" } - - { name: integrations, suites: "providers,webhooks,notifications" } + - { name: providers, suites: "providers,notifications" } + - { name: webhooks, suites: "webhooks" } + - { name: connectors, suites: "connectors" } - { name: commerce, suites: "payments,settings" } steps: - name: Checkout code diff --git a/app/test/e2e/helpers/composio-helpers.ts b/app/test/e2e/helpers/composio-helpers.ts index c0670cda6..7018fd16b 100644 --- a/app/test/e2e/helpers/composio-helpers.ts +++ b/app/test/e2e/helpers/composio-helpers.ts @@ -66,15 +66,41 @@ export async function assertConnectorCardVisible(name: string, timeout = 15_000) */ export async function openConnectorModal(name: string, timeout = 15_000): Promise { console.log(`${LOG} opening connector modal for "${name}"`); - // Click the connector card by name - const cardEl = await waitForText(name, timeout); - await cardEl.click(); - // @ts-expect-error -- browser global is injected by WDIO at runtime, not typed in this env - await browser.pause(1_500); + const candidates = [ + `Connect ${name}`, + `Manage ${name}`, + `Reconnect ${name}`, + `${name} authorization expired`, + `${name} is connected`, + 'Disconnect', + ]; + + const ensureModalOpen = async (): Promise => + browser.execute((connectorName: string) => { + const dialog = document.querySelector('[role="dialog"]'); + if (dialog) return true; + const exactButton = Array.from(document.querySelectorAll('button')).find(btn => { + const label = btn.getAttribute('aria-label') ?? ''; + const title = btn.getAttribute('title') ?? ''; + const text = btn.textContent ?? ''; + return ( + label.includes(connectorName) || + title.includes(connectorName) || + text.includes(connectorName) + ); + }) as HTMLButtonElement | undefined; + if (!exactButton) return false; + exactButton.click(); + return false; + }, name); + + // Click once up front. If the modal appears, stop trying to re-click the + // underlying card; the backdrop will intercept any later coordinate clicks. + await waitForText(name, timeout); + await ensureModalOpen(); - // Wait for any of the standard modal header patterns - const candidates = [`Connect ${name}`, `Manage ${name}`, `Reconnect ${name}`]; const deadline = Date.now() + timeout; + let lastReopenAt = 0; while (Date.now() < deadline) { for (const candidate of candidates) { if (await textExists(candidate)) { @@ -82,8 +108,15 @@ export async function openConnectorModal(name: string, timeout = 15_000): Promis return candidate; } } + const modalVisible = await browser + .execute(() => Boolean(document.querySelector('[role="dialog"]'))) + .catch(() => false); + if (!modalVisible && Date.now() - lastReopenAt > 1_000) { + await ensureModalOpen(); + lastReopenAt = Date.now(); + } // @ts-expect-error -- browser global is injected by WDIO at runtime, not typed in this env - await browser.pause(400); + await browser.pause(250); } console.log(`${LOG} modal for "${name}" did not open within timeout`); diff --git a/app/test/e2e/specs/harness-composio-tool-flow.spec.ts b/app/test/e2e/specs/harness-composio-tool-flow.spec.ts index 5114df80f..26645bf6f 100644 --- a/app/test/e2e/specs/harness-composio-tool-flow.spec.ts +++ b/app/test/e2e/specs/harness-composio-tool-flow.spec.ts @@ -57,6 +57,18 @@ import { const LOG_PREFIX = '[HarnessComposio]'; const USER_ID = 'e2e-harness-composio-tool-flow'; +function seedHarnessComposioState(): void { + setMockBehavior('composioToolkits', JSON.stringify(['gmail', 'github', 'linear'])); + setMockBehavior( + 'composioConnections', + JSON.stringify([ + { id: 'conn-gmail', toolkit: 'gmail', status: 'ACTIVE' }, + { id: 'conn-github', toolkit: 'github', status: 'ACTIVE' }, + { id: 'conn-linear', toolkit: 'linear', status: 'ACTIVE' }, + ]) + ); +} + // --------------------------------------------------------------------------- // Shared helpers // --------------------------------------------------------------------------- @@ -65,15 +77,32 @@ const USER_ID = 'e2e-harness-composio-tool-flow'; * message and return. The calling test is responsible for asserting outcomes. */ async function navigateChatAndSend(prompt: string): Promise { await navigateViaHash('/chat'); - await browser.waitUntil(async () => await textExists('Threads'), { - timeout: 15_000, - timeoutMsg: 'Conversations panel did not mount', - }); - expect(await clickByTitle('New thread', 8_000)).toBe(true); - await browser.waitUntil(async () => await getSelectedThreadId(), { - timeout: 8_000, - timeoutMsg: 'thread.selectedThreadId never populated', - }); + await browser.waitUntil( + async () => { + if (await getSelectedThreadId()) return true; + if (await textExists('No messages yet')) return true; + return textExists('Type a message'); + }, + { timeout: 15_000, timeoutMsg: 'Chat surface did not mount' } + ); + if (!(await getSelectedThreadId())) { + const clicked = + (await clickByTitle('New thread', 8_000)) || + (await clickByTitle('New thread (/new)', 3_000)) || + (await browser.execute(() => { + const btn = Array.from(document.querySelectorAll('button')).find( + button => (button.textContent ?? '').trim() === 'New' + ) as HTMLButtonElement | undefined; + if (!btn) return false; + btn.click(); + return true; + })); + expect(clicked).toBe(true); + await browser.waitUntil(async () => await getSelectedThreadId(), { + timeout: 8_000, + timeoutMsg: 'thread.selectedThreadId never populated', + }); + } await typeIntoComposer(prompt); const socketReady = await waitForSocketConnected(30_000); @@ -117,6 +146,7 @@ describe('Harness — Composio tool-call prompt flow', () => { clearRequestLog(); resetMockBehavior(); + seedHarnessComposioState(); // Canned inbox: 3 messages the mock Composio execute will return. const GMAIL_MESSAGES = [ @@ -195,6 +225,7 @@ describe('Harness — Composio tool-call prompt flow', () => { clearRequestLog(); resetMockBehavior(); + seedHarnessComposioState(); const GITHUB_REPOS = [ { name: 'openhuman', full_name: 'tinyhumansai/openhuman', private: false }, @@ -248,6 +279,7 @@ describe('Harness — Composio tool-call prompt flow', () => { clearRequestLog(); resetMockBehavior(); + seedHarnessComposioState(); // Inject a 400 failure for all composio execute calls. setMockBehavior('composioExecuteFails', '400'); @@ -304,6 +336,7 @@ describe('Harness — Composio tool-call prompt flow', () => { clearRequestLog(); resetMockBehavior(); + seedHarnessComposioState(); const LINEAR_RESULT = { issue: { diff --git a/docs/whatsapp-data-flow.md b/docs/whatsapp-data-flow.md index 76a9fc46f..3c45feaa2 100644 --- a/docs/whatsapp-data-flow.md +++ b/docs/whatsapp-data-flow.md @@ -42,10 +42,10 @@ Both ingest endpoints fire on every scan tick; both are `tokio::spawn` fire-and- ## Why two paths -| Path | Backing store | Strength | Use it for | -|------|---------------|----------|------------| -| **Direct** | `whatsapp_data.db` (SQLite) | Exact, structured, paginated | "List my WhatsApp chats", "show the last 50 messages with Alice", "search for `invoice` across WhatsApp" | -| **Memory tree** | Per-source memory tree + embeddings | Semantic, cross-source | "Summarise this week of WhatsApp", "find action items across email and WhatsApp", "what did the team agree on?" | +| Path | Backing store | Strength | Use it for | +| --------------- | ----------------------------------- | ---------------------------- | --------------------------------------------------------------------------------------------------------------- | +| **Direct** | `whatsapp_data.db` (SQLite) | Exact, structured, paginated | "List my WhatsApp chats", "show the last 50 messages with Alice", "search for `invoice` across WhatsApp" | +| **Memory tree** | Per-source memory tree + embeddings | Semantic, cross-source | "Summarise this week of WhatsApp", "find action items across email and WhatsApp", "what did the team agree on?" | The same scan tick populates both stores. Idempotency keys make the dual-write safe to retry: diff --git a/scripts/mock-api/routes/integrations.mjs b/scripts/mock-api/routes/integrations.mjs index 54f32cf08..00d894d62 100644 --- a/scripts/mock-api/routes/integrations.mjs +++ b/scripts/mock-api/routes/integrations.mjs @@ -142,6 +142,42 @@ export function handleIntegrations(ctx) { return true; } + if (method === "POST" && /^\/openai\/v1\/embeddings\/?$/.test(url)) { + const inputs = Array.isArray(parsedBody?.input) + ? parsedBody.input + : [parsedBody?.input ?? ""]; + const data = inputs.map((input, index) => { + const text = String(input ?? ""); + // Keep the vector tiny but deterministic so callers that cache / + // compare embeddings can still observe stable output. + const basis = text.length || index + 1; + return { + object: "embedding", + index, + embedding: [basis, basis / 10, basis / 100, 1], + }; + }); + json(res, 200, { + object: "list", + data, + model: + typeof parsedBody?.model === "string" && parsedBody.model.length > 0 + ? parsedBody.model + : "text-embedding-3-small", + usage: { + prompt_tokens: inputs.reduce( + (acc, input) => acc + String(input ?? "").length, + 0, + ), + total_tokens: inputs.reduce( + (acc, input) => acc + String(input ?? "").length, + 0, + ), + }, + }); + return true; + } + // (chat/completions is handled by routes/llm.mjs ahead of this route) // ── Composio ─────────────────────────────────────────────── @@ -313,7 +349,10 @@ export function handleIntegrations(ctx) { : ""; // composioExecuteFails → inject error response // Knob values: '400' or '1' → HTTP 400; '500' → HTTP 500 - if (mockBehavior.composioExecuteFails === "400" || mockBehavior.composioExecuteFails === "1") { + if ( + mockBehavior.composioExecuteFails === "400" || + mockBehavior.composioExecuteFails === "1" + ) { json(res, 400, { success: false, error: "Mock execute failure", @@ -325,7 +364,11 @@ export function handleIntegrations(ctx) { json(res, 500, { success: false, error: "Mock execute server error", - data: { successful: false, data: null, error: "Mock execute server error" }, + data: { + successful: false, + data: null, + error: "Mock execute server error", + }, }); return true; } @@ -369,11 +412,20 @@ export function handleIntegrations(ctx) { /^\/agent-integrations\/composio\/connections\/[^/]+\/?$/.test(url) ) { if (mockBehavior.composioDeleteFails === "400") { - json(res, 400, { success: false, error: "Mock connection delete failure" }); + json(res, 400, { + success: false, + error: "Mock connection delete failure", + }); return true; } - if (mockBehavior.composioDeleteFails === "500" || mockBehavior.composioDeleteFails === "1") { - json(res, 500, { success: false, error: "Mock connection delete failure" }); + if ( + mockBehavior.composioDeleteFails === "500" || + mockBehavior.composioDeleteFails === "1" + ) { + json(res, 500, { + success: false, + error: "Mock connection delete failure", + }); return true; } let connId = url.split("/").filter(Boolean).pop() ?? ""; @@ -381,7 +433,10 @@ export function handleIntegrations(ctx) { try { connId = decodeURIComponent(connId); } catch { - json(res, 400, { success: false, error: "Invalid connection id encoding" }); + json(res, 400, { + success: false, + error: "Invalid connection id encoding", + }); return true; } // Remove the connection from the seeded list if present @@ -404,7 +459,10 @@ export function handleIntegrations(ctx) { json(res, 400, { success: false, error: "Mock sync failure" }); return true; } - if (mockBehavior.composioSyncFails === "500" || mockBehavior.composioSyncFails === "1") { + if ( + mockBehavior.composioSyncFails === "500" || + mockBehavior.composioSyncFails === "1" + ) { json(res, 500, { success: false, error: "Mock sync failure" }); return true; } diff --git a/src/openhuman/memory_store/README.md b/src/openhuman/memory_store/README.md index ddfb83f8a..2de734a2f 100644 --- a/src/openhuman/memory_store/README.md +++ b/src/openhuman/memory_store/README.md @@ -19,18 +19,18 @@ unified/ [staging for removal] UnifiedMemory's remaining SQLite surface ## Cross-cutting modules -| Path | Role | -| --- | --- | -| [`mod.rs`](mod.rs) | Module root + public re-exports. | -| [`README.md`](README.md) | You are here. | -| [`kinds.rs`](kinds.rs) | `MemoryKind` enum — the authoritative catalog: Raw / Chunk / Entity / Tree / Vector / Kv / Contact — plus per-kind type aliases. | -| [`traits.rs`](traits.rs) | `VectorEmbeddable` + `ObsidianRepresentable` + `ObsidianFile`. Every stored kind implements both — the compiler enforces "everything in memory_store is vector and obsidian compatible". | -| [`types.rs`](types.rs) | Shared serde types used across submodules: `NamespaceDocumentInput`, `NamespaceMemoryHit`, `NamespaceQueryResult`, `NamespaceRetrievalContext`, `RetrievalScoreBreakdown`, `MemoryItemKind`, `MemoryKvRecord`. | -| [`memory_trait.rs`](memory_trait.rs) | `impl Memory for UnifiedMemory` — bridges the generic `Memory` trait surface onto the unified store. | -| [`client.rs`](client.rs) | `MemoryClient` / `MemoryClientRef` / `MemoryState`. Async wrapper over `UnifiedMemory` used by RPC controllers; owns the singleton ingestion-queue handle. | -| [`factories.rs`](factories.rs) | `create_memory*` constructors. Selects the embedding provider per the `MemoryConfig`, probes Ollama health, and builds a `Box` over `UnifiedMemory`. | -| [`retrieval/`](retrieval/) | `RetrievalFacade` — single import surface over the four retrieval modes (tree-walk, vector, keyword, param/tag). | -| [`tools/`](tools/) | Agent tools that read directly from memory_store: `memory_store_raw_search`, `memory_store_raw_chunks`, `memory_store_kinds`. | +| Path | Role | +| ------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`mod.rs`](mod.rs) | Module root + public re-exports. | +| [`README.md`](README.md) | You are here. | +| [`kinds.rs`](kinds.rs) | `MemoryKind` enum — the authoritative catalog: Raw / Chunk / Entity / Tree / Vector / Kv / Contact — plus per-kind type aliases. | +| [`traits.rs`](traits.rs) | `VectorEmbeddable` + `ObsidianRepresentable` + `ObsidianFile`. Every stored kind implements both — the compiler enforces "everything in memory_store is vector and obsidian compatible". | +| [`types.rs`](types.rs) | Shared serde types used across submodules: `NamespaceDocumentInput`, `NamespaceMemoryHit`, `NamespaceQueryResult`, `NamespaceRetrievalContext`, `RetrievalScoreBreakdown`, `MemoryItemKind`, `MemoryKvRecord`. | +| [`memory_trait.rs`](memory_trait.rs) | `impl Memory for UnifiedMemory` — bridges the generic `Memory` trait surface onto the unified store. | +| [`client.rs`](client.rs) | `MemoryClient` / `MemoryClientRef` / `MemoryState`. Async wrapper over `UnifiedMemory` used by RPC controllers; owns the singleton ingestion-queue handle. | +| [`factories.rs`](factories.rs) | `create_memory*` constructors. Selects the embedding provider per the `MemoryConfig`, probes Ollama health, and builds a `Box` over `UnifiedMemory`. | +| [`retrieval/`](retrieval/) | `RetrievalFacade` — single import surface over the four retrieval modes (tree-walk, vector, keyword, param/tag). | +| [`tools/`](tools/) | Agent tools that read directly from memory_store: `memory_store_raw_search`, `memory_store_raw_chunks`, `memory_store_kinds`. | ## Storage submodules