From 7fd935e7a974de07467ed550a61edb7e9b0b0106 Mon Sep 17 00:00:00 2001 From: Steven Enamakel <31011319+senamakel@users.noreply.github.com> Date: Sat, 4 Jul 2026 17:23:17 -0700 Subject: [PATCH] test(e2e): stabilize chat release-lane tail (#4527) --- app/src/pages/Conversations.tsx | 1 + .../specs/chat-harness-scroll-render.spec.ts | 33 +++++++++++-------- .../e2e/specs/chat-harness-subagent.spec.ts | 24 +++++++++++++- .../specs/chat-harness-wallet-flow.spec.ts | 20 ++++++++--- .../e2e/specs/user-journey-full-task.spec.ts | 14 ++++++-- 5 files changed, 70 insertions(+), 22 deletions(-) diff --git a/app/src/pages/Conversations.tsx b/app/src/pages/Conversations.tsx index 779445197..190d119ef 100644 --- a/app/src/pages/Conversations.tsx +++ b/app/src/pages/Conversations.tsx @@ -2099,6 +2099,7 @@ const Conversations = ({ }>
` in Conversations.tsx. -// There's only one in /chat so the structural class predicate is enough. async function scrollMetrics(): Promise<{ scrollTop: number; scrollHeight: number; clientHeight: number; + found: boolean; }> { return (await browser.execute(() => { - // The messages scroll container is `flex-1 min-h-0 overflow-y-auto` - // (Conversations.tsx). The former hardcoded `bg-[#f6f6f6]` class was - // replaced by a surface token, so match on the structural classes instead. - const el = document.querySelector('div.flex-1.min-h-0.overflow-y-auto') as HTMLElement | null; - if (!el) return { scrollTop: 0, scrollHeight: 0, clientHeight: 0 }; + const el = document.querySelector('[data-testid="chat-messages-scroll"]') as HTMLElement | null; + if (!el) return { scrollTop: 0, scrollHeight: 0, clientHeight: 0, found: false }; return { scrollTop: el.scrollTop, scrollHeight: el.scrollHeight, clientHeight: el.clientHeight, + found: true, }; - })) as { scrollTop: number; scrollHeight: number; clientHeight: number }; + })) as { scrollTop: number; scrollHeight: number; clientHeight: number; found: boolean }; } async function scrollMessageColumn(top: number): Promise { await browser.execute((y: number) => { - // The messages scroll container is `flex-1 min-h-0 overflow-y-auto` - // (Conversations.tsx). The former hardcoded `bg-[#f6f6f6]` class was - // replaced by a surface token, so match on the structural classes instead. - const el = document.querySelector('div.flex-1.min-h-0.overflow-y-auto') as HTMLElement | null; + const el = document.querySelector('[data-testid="chat-messages-scroll"]') as HTMLElement | null; if (el) el.scrollTo({ top: y, behavior: 'auto' }); }, top); } @@ -181,7 +174,18 @@ describe('Chat harness — scroll + markdown render', () => { // ── 3. Auto-scroll anchored to the bottom after the stream ───── // (within 40 px to absorb sub-pixel layout drift) + await browser.waitUntil( + async () => { + const metrics = await scrollMetrics(); + if (!metrics.found) return false; + return metrics.scrollHeight > metrics.clientHeight; + }, + { timeout: 10_000, timeoutMsg: 'chat messages scroll container never overflowed' } + ); const atBottom = await scrollMetrics(); + console.log( + `[chat-harness-scroll-render] bottom metrics: scrollTop=${atBottom.scrollTop}, scrollHeight=${atBottom.scrollHeight}, clientHeight=${atBottom.clientHeight}` + ); expect(atBottom.scrollHeight).toBeGreaterThan(atBottom.clientHeight); expect(atBottom.scrollHeight - (atBottom.scrollTop + atBottom.clientHeight)).toBeLessThan(40); @@ -190,6 +194,9 @@ describe('Chat harness — scroll + markdown render', () => { await scrollMessageColumn(targetTop); await browser.pause(500); // let the stick hook react const afterScrollUp = await scrollMetrics(); + console.log( + `[chat-harness-scroll-render] after manual scroll: scrollTop=${afterScrollUp.scrollTop}, scrollHeight=${afterScrollUp.scrollHeight}, clientHeight=${afterScrollUp.clientHeight}, targetTop=${targetTop}` + ); expect(Math.abs(afterScrollUp.scrollTop - targetTop)).toBeLessThan(40); expect( afterScrollUp.scrollHeight - (afterScrollUp.scrollTop + afterScrollUp.clientHeight) diff --git a/app/test/e2e/specs/chat-harness-subagent.spec.ts b/app/test/e2e/specs/chat-harness-subagent.spec.ts index ae2f8927f..b9780c155 100644 --- a/app/test/e2e/specs/chat-harness-subagent.spec.ts +++ b/app/test/e2e/specs/chat-harness-subagent.spec.ts @@ -104,6 +104,16 @@ async function snapshotRuntime(threadId: string): Promise { }, threadId)) as RuntimeSnapshot; } +async function hasRenderedSubagentTimeline(): Promise { + return (await browser.execute(() => { + const rows = Array.from(document.querySelectorAll('[data-testid="agent-timeline-row"]')); + return rows.some(row => { + const text = row.textContent ?? ''; + return /Research|Researching|subagent/i.test(text); + }); + })) as boolean; +} + describe('Chat harness — orchestrator → subagent flow', () => { before(async function beforeSuite() { this.timeout(90_000); @@ -114,6 +124,13 @@ describe('Chat harness — orchestrator → subagent flow', () => { // clean signed-in state rather than a polluted one (the source of the // intermittent "final canary never arrives" failures). await resetApp(USER_ID, { clearAuthSession: true }); + const superContext = await callOpenhumanRpc('openhuman.config_set_super_context_enabled', { + value: false, + }); + expect(superContext.ok).toBe(true); + console.log( + '[chat-harness-subagent] Disabled super context for deterministic scripted LLM calls' + ); setMockBehavior('llmForcedResponses', JSON.stringify(FORCED_RESPONSES)); // Faster streaming for non-tool-call responses so this spec doesn't @@ -170,10 +187,15 @@ describe('Chat harness — orchestrator → subagent flow', () => { sawSubagentTimeline = true; } if (sawSubagentPhase && sawSubagentTimeline) break; - if (await textExists(CANARY_FINAL)) break; + if (await textExists(CANARY_FINAL)) { + sawSubagentTimeline = sawSubagentTimeline || (await hasRenderedSubagentTimeline()); + break; + } await browser.pause(200); } + sawSubagentTimeline = sawSubagentTimeline || (await hasRenderedSubagentTimeline()); + // At least ONE of the two signals must have fired — the timeline // entry is the more durable check (the live phase can flip back to // 'thinking' or 'idle' before our 200ms poll catches it). diff --git a/app/test/e2e/specs/chat-harness-wallet-flow.spec.ts b/app/test/e2e/specs/chat-harness-wallet-flow.spec.ts index ea27d2d5f..34d2274e9 100644 --- a/app/test/e2e/specs/chat-harness-wallet-flow.spec.ts +++ b/app/test/e2e/specs/chat-harness-wallet-flow.spec.ts @@ -31,7 +31,7 @@ import { waitForSocketConnected, } from '../helpers/chat-harness'; import { callOpenhumanRpc } from '../helpers/core-rpc'; -import { clickText, clickToggle, textExists } from '../helpers/element-helpers'; +import { clickText, textExists } from '../helpers/element-helpers'; import { resetApp } from '../helpers/reset-app'; import { navigateViaHash } from '../helpers/shared-flows'; import { @@ -53,7 +53,7 @@ const FORCED_RESPONSES = [ toolCalls: [ { id: 'call_delegate_do_crypto_1', - name: 'delegate_do_crypto', + name: 'do_crypto', arguments: JSON.stringify({ prompt: `Prepare a $5 EVM transfer to John at ${JOHN_ADDRESS}.`, }), @@ -87,14 +87,17 @@ const FORCED_RESPONSES = [ ]; async function clickRecoveryConsentCheckbox(): Promise { - const checkbox = await browser.$('input[type="checkbox"]'); + const checkbox = await browser.$('#mnemonic-confirm-checkbox'); if (!(await checkbox.isExisting())) { throw new Error('Recovery phrase consent checkbox not found'); } if (!(await checkbox.isSelected())) { - await clickToggle(); + await checkbox.click(); } - await expect(checkbox).toBeSelected(); + await browser.waitUntil(async () => await checkbox.isSelected(), { + timeout: 5_000, + timeoutMsg: 'Recovery phrase consent checkbox did not become selected', + }); } describe('Chat harness — wallet flow', () => { @@ -106,6 +109,13 @@ describe('Chat harness — wallet flow', () => { // so the crypto sub-agent run starts from a clean signed-in state (a // polluted session was the source of the intermittent quote-store failures). await resetApp(USER_ID, { clearAuthSession: true }); + const superContext = await callOpenhumanRpc('openhuman.config_set_super_context_enabled', { + value: false, + }); + expect(superContext.ok).toBe(true); + console.log( + '[chat-harness-wallet-flow] Disabled super context for deterministic scripted LLM calls' + ); }); after(async () => { diff --git a/app/test/e2e/specs/user-journey-full-task.spec.ts b/app/test/e2e/specs/user-journey-full-task.spec.ts index 52fa227cb..1b484b70f 100644 --- a/app/test/e2e/specs/user-journey-full-task.spec.ts +++ b/app/test/e2e/specs/user-journey-full-task.spec.ts @@ -61,6 +61,11 @@ describe('User journey — full research task', () => { await startMockServer(); await waitForApp(); await resetApp(USER_ID); + const superContext = await callOpenhumanRpc('openhuman.config_set_super_context_enabled', { + value: false, + }); + expect(superContext.ok).toBe(true); + console.log(`${LOG_PREFIX} Disabled super context for deterministic scripted LLM calls`); setMockBehavior('llmForcedResponses', JSON.stringify(FORCED_RESPONSES)); setMockBehavior('llmStreamChunkDelayMs', '10'); @@ -111,7 +116,8 @@ describe('User journey — full research task', () => { console.log(`${LOG_PREFIX} J1.1: passed — user message visible`); }); - it('J1.2 — tool call timeline appears during execution', async () => { + it('J1.2 — tool call timeline appears during execution', async function () { + this.timeout(60_000); console.log(`${LOG_PREFIX} J1.2: watching for tool timeline entry`); let sawToolTimeline = false; const deadline = Date.now() + 45_000; @@ -142,7 +148,8 @@ describe('User journey — full research task', () => { console.log(`${LOG_PREFIX} J1.2: passed`); }); - it('J1.3 — final answer with canary text renders', async () => { + it('J1.3 — final answer with canary text renders', async function () { + this.timeout(60_000); console.log(`${LOG_PREFIX} J1.3: waiting for canary`); await browser.waitUntil(async () => await textExists(CANARY_FINAL), { timeout: 45_000, @@ -151,7 +158,8 @@ describe('User journey — full research task', () => { console.log(`${LOG_PREFIX} J1.3: passed — canary visible`); }); - it('J1.4 — after navigate away + back, thread messages still visible', async () => { + it('J1.4 — after navigate away + back, thread messages still visible', async function () { + this.timeout(60_000); console.log(`${LOG_PREFIX} J1.4: navigating away to /home`); // Ensure the IN_FLIGHT map cleared (turn is fully done) before navigating.