fix(test): eliminate flaky Playwright patterns and add CI retries (#3517)

app/playwright.config.ts

@@ -6,6 +6,7 @@ export default defineConfig({
   testDir: './test/playwright/specs',
   fullyParallel: false,
   workers: 1,
+  retries: process.env.CI ? 2 : 0,
   timeout: 60_000,
   expect: {
     timeout: 10_000,

app/test/playwright/specs/auth-access-control.spec.ts

@@ -24,12 +24,14 @@ async function mockRequests(): Promise<MockRequest[]> {
 
 async function waitForMockRequest(method: string, pathFragment: string, timeoutMs = 15_000) {
   const deadline = Date.now() + timeoutMs;
+  let delay = 200;
   while (Date.now() < deadline) {
     const match = (await mockRequests()).find(
       request => request.method === method && request.url.includes(pathFragment)
     );
     if (match) return match;
-    await new Promise(resolve => setTimeout(resolve, 300));
+    await new Promise(resolve => setTimeout(resolve, delay));
+    delay = Math.min(delay * 1.5, 1_000);
   }
   return null;
 }

app/test/playwright/specs/chat-harness-cancel.spec.ts

@@ -129,9 +129,9 @@ test.describe('Chat Harness - Cancel', () => {
 
     await page.getByRole('button', { name: 'Cancel' }).click();
 
-    await page.waitForTimeout(3_500);
+    await expect(page.getByRole('button', { name: 'Cancel' })).toHaveCount(0, { timeout: 10_000 });
     for (const piece of LATE_PIECES) {
-      await expect(page.getByText(piece, { exact: false })).toHaveCount(0);
+      await expect(page.getByText(piece, { exact: false })).toHaveCount(0, { timeout: 5_000 });
     }
 
     const composer = page.getByPlaceholder('How can I help you today?');

app/test/playwright/specs/chat-harness-scroll-render.spec.ts

@@ -163,7 +163,18 @@ test.describe('Chat Harness - Scroll Render', () => {
         column?.scrollTo({ top: nextTop, behavior: 'auto' });
       }, targetTop);
 
-      await page.waitForTimeout(500);
+      await expect
+        .poll(
+          async () =>
+            page.evaluate(expected => {
+              const column = document.querySelector(
+                'div.flex-1.overflow-y-auto.bg-\\[\\#f6f6f6\\]'
+              ) as HTMLElement | null;
+              return Math.abs((column?.scrollTop ?? 0) - expected) < 40;
+            }, targetTop),
+          { timeout: 5_000 }
+        )
+        .toBe(true);
 
       const afterScrollUp = await page.evaluate(() => {
         const column = document.querySelector(

app/test/playwright/specs/connector-gmail-composio.spec.ts

@@ -121,7 +121,6 @@ async function ensureComposioSurface(page: Page) {
         return;
       }
     }
-    await page.waitForTimeout(500);
   }
   await expect(heading).toBeVisible({ timeout: 20_000 });
 }

app/test/playwright/specs/login-flow.spec.ts

@@ -29,12 +29,14 @@ async function requests(): Promise<MockRequest[]> {
 
 async function waitForMockRequest(method: string, pathFragment: string, timeoutMs = 15_000) {
   const deadline = Date.now() + timeoutMs;
+  let delay = 200;
   while (Date.now() < deadline) {
     const match = (await requests()).find(
       request => request.method === method && request.url.includes(pathFragment)
     );
     if (match) return match;
-    await new Promise(resolve => setTimeout(resolve, 300));
+    await new Promise(resolve => setTimeout(resolve, delay));
+    delay = Math.min(delay * 1.5, 1_000);
   }
   return null;
 }

app/test/playwright/specs/memory-roundtrip.spec.ts

@@ -74,8 +74,12 @@ test.describe('Memory subsystem round-trip', () => {
       limit: 10,
     });
     let recalled = JSON.stringify(recallAfterForget ?? {});
-    if (recalled.includes(TEST_KEY) || recalled.includes(TEST_CONTENT)) {
-      await new Promise(resolve => setTimeout(resolve, 3_000));
+    for (
+      let attempt = 0;
+      attempt < 10 && (recalled.includes(TEST_KEY) || recalled.includes(TEST_CONTENT));
+      attempt++
+    ) {
+      await new Promise(resolve => setTimeout(resolve, 500));
       const retry = await callCoreRpc<unknown>('openhuman.memory_recall_memories', {
         namespace: TEST_NAMESPACE,
         limit: 10,

app/test/playwright/specs/navigation-smoothness.spec.ts

@@ -30,7 +30,7 @@ async function verifyRouteLoaded(
   route: RouteCheck
 ): Promise<void> {
   await waitForAppReady(page);
-  await expect(await rootTextLength(page)).toBeGreaterThan(50);
+  await expect.poll(() => rootTextLength(page), { timeout: 10_000 }).toBeGreaterThan(50);
 }
 
 test.describe('Navigation Smoothness', () => {
@@ -42,14 +42,12 @@ test.describe('Navigation Smoothness', () => {
     for (const route of routes) {
       await page.goto(`/#${route.hash}`);
       await verifyRouteLoaded(page, route);
-      await page.waitForTimeout(400);
     }
   });
 
   test('rapid cycle completes without blank screens', async ({ page }) => {
     for (const route of routes) {
       await page.goto(`/#${route.hash}`);
-      await page.waitForTimeout(350);
       await verifyRouteLoaded(page, route);
     }
   });

app/test/playwright/specs/onboarding-modes.spec.ts

@@ -17,25 +17,15 @@ async function clickOnboardingNext(page: Page): Promise<void> {
 }
 
 async function clickTestId(page: Page, testId: string, timeout = 10_000): Promise<boolean> {
-  const deadline = Date.now() + timeout;
-  while (Date.now() < deadline) {
-    const status = await page.evaluate(id => {
-      const el = document.querySelector<HTMLElement>(`[data-testid="${id}"]`);
-      if (!el) return 'missing';
-      if ((el as HTMLButtonElement).disabled) return 'disabled';
-      const rect = el.getBoundingClientRect();
-      if (rect.width === 0 || rect.height === 0) return 'no-layout';
-      ['mousedown', 'mouseup', 'click'].forEach(type => {
-        el.dispatchEvent(
-          new MouseEvent(type, { bubbles: true, cancelable: true, view: window, button: 0 })
-        );
-      });
-      return 'clicked';
-    }, testId);
-    if (status === 'clicked') return true;
-    await page.waitForTimeout(300);
+  const locator = page.getByTestId(testId);
+  try {
+    await locator.waitFor({ state: 'visible', timeout });
+    await expect(locator).toBeEnabled({ timeout: 5_000 });
+    await locator.click({ force: true, timeout: 5_000 });
+    return true;
+  } catch {
+    return false;
   }
-  return false;
 }
 
 async function bootIntoOnboarding(page: Page, userId: string): Promise<void> {

app/test/playwright/specs/runtime-picker-login.spec.ts

@@ -30,12 +30,14 @@ async function mockRequests(): Promise<MockRequest[]> {
 
 async function waitForMockRequest(method: string, pathFragment: string, timeoutMs = 15_000) {
   const deadline = Date.now() + timeoutMs;
+  let delay = 200;
   while (Date.now() < deadline) {
     const match = (await mockRequests()).find(
       request => request.method === method && request.url.includes(pathFragment)
     );
     if (match) return match;
-    await new Promise(resolve => setTimeout(resolve, 300));
+    await new Promise(resolve => setTimeout(resolve, delay));
+    delay = Math.min(delay * 1.5, 1_000);
   }
   return null;
 }

app/test/playwright/specs/webhooks-tunnel-flow.spec.ts

@@ -42,11 +42,13 @@ async function waitForRequest(
   timeoutMs = 10_000
 ): Promise<RequestLogEntry | undefined> {
   const deadline = Date.now() + timeoutMs;
+  let delay = 200;
   while (Date.now() < deadline) {
     const log = await getRequestLog();
     const match = log.find(entry => entry.method === method && entry.url?.includes(urlFragment));
     if (match) return match;
-    await new Promise(resolve => setTimeout(resolve, 250));
+    await new Promise(resolve => setTimeout(resolve, delay));
+    delay = Math.min(delay * 1.5, 1_000);
   }
   return undefined;
 }