mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-27 21:08:00 +00:00
617 lines
27 KiB
TypeScript
617 lines
27 KiB
TypeScript
// @ts-nocheck
|
||
/**
|
||
* E2E: global push-to-talk (PTT) end-to-end flow with mocked STT.
|
||
*
|
||
* What this spec exercises (top to bottom):
|
||
*
|
||
* UI:
|
||
* 1. Navigate to /settings/voice → PttSettingsPanel mounts (data-testid
|
||
* "ptt-settings-panel").
|
||
* 2. Programmatically dispatch `setPttShortcut('F13')` against the exposed
|
||
* Redux store to simulate the user binding a hotkey. Using a Redux
|
||
* dispatch (rather than driving the readonly capture input via
|
||
* chromedriver) sidesteps two fragile layers:
|
||
* a. The keyboard-capture input intercepts native keydown events
|
||
* that CDP would otherwise inject into the textarea.
|
||
* b. F13 is reliably passable through chromedriver to a generic
|
||
* input but the panel-level interception logic is unit-tested
|
||
* elsewhere (PttSettingsPanel.test.tsx). We test the *binding
|
||
* effect*, not the capture UX.
|
||
* 3. Assert `usePttHotkey` reacts and Redux state settles with a non-null
|
||
* shortcut. Registration may succeed (no error) or fail with a non-
|
||
* empty error string on headless Linux runners with no real keyboard
|
||
* — both are acceptable signals that the binding path was driven; we
|
||
* log the failure for follow-up but don't make CI red on it.
|
||
*
|
||
* PTT session:
|
||
* 4. Mock navigator.mediaDevices.getUserMedia + MediaRecorder so the
|
||
* renderer-side audio capture (pttAudio.ts) can run without a real
|
||
* microphone (headless CEF has no audio device).
|
||
* 5. Configure the mock backend (audioTranscriptionText) so the core's
|
||
* cloud STT path returns a known transcript "hello from PTT".
|
||
* 6. Simulate the hotkey hold by emitting `ptt://start`/`ptt://stop` via
|
||
* Tauri's internal event plugin (`__TAURI_INTERNALS__.invoke('plugin:
|
||
* event|emit', ...)`). This is the same path `@tauri-apps/api/event`'s
|
||
* `emit()` uses; we go through the internal because direct dynamic
|
||
* imports of `@tauri-apps/api/event` don't resolve under Chromium-
|
||
* driver (see core-rpc.ts).
|
||
* 7. Wait long enough between start/stop (≥ 250 ms — pttService's
|
||
* `minAudioMs`) so the recording isn't dropped as an accidental tap.
|
||
*
|
||
* Assertions:
|
||
* 8. The overlay window is created (window-handle count went from 1 →
|
||
* 2 when register_ptt_hotkey called ptt_overlay::ensure_window).
|
||
* 9. The transcribed text appears as a user message in the chat thread.
|
||
* 10. The core_rpc_relay invocation for `channel_web_chat` carried
|
||
* `speak_reply: true` (the user's PTT setting was honoured on the
|
||
* wire). We spy on `__TAURI_INTERNALS__.invoke` before the press to
|
||
* capture the call payload.
|
||
*
|
||
* Spec: docs/superpowers/specs/2026-06-02-global-ptt-design.md.
|
||
*
|
||
* Limitations / notes for follow-up sessions:
|
||
* - The OS-level global-shortcut emit can't be triggered by the Chromium
|
||
* driver (CDP injects events into the renderer, not the OS keyboard
|
||
* subsystem). Step 6 above is the correct workaround in a unit-test
|
||
* sense, but it does not exercise the rdev → tauri global-shortcut
|
||
* pipeline on the way in. That layer is covered by Rust unit tests
|
||
* in `ptt_hotkeys.rs` and integration coverage in PttHotkeyManager
|
||
* tests.
|
||
* - MediaRecorder availability under CEF headless: present but won't
|
||
* produce real opus frames. We mock it entirely so the buffer reaches
|
||
* the transcribe RPC as a zero-byte blob; the mock backend doesn't care
|
||
* about the actual audio bytes (it just returns the configured
|
||
* transcript text).
|
||
*/
|
||
import { waitForApp } from '../helpers/app-helpers';
|
||
import {
|
||
chatMounted,
|
||
getSelectedThreadId,
|
||
waitForAssistantReplyContaining,
|
||
waitForSocketConnected,
|
||
} from '../helpers/chat-harness';
|
||
import { callOpenhumanRpc } from '../helpers/core-rpc';
|
||
import { resetApp } from '../helpers/reset-app';
|
||
import { navigateViaHash } from '../helpers/shared-flows';
|
||
import {
|
||
clearRequestLog,
|
||
getRequestLog,
|
||
setMockBehavior,
|
||
startMockServer,
|
||
stopMockServer,
|
||
} from '../mock-server';
|
||
|
||
const USER_ID = 'e2e-ptt-flow';
|
||
const SHORTCUT = 'F13';
|
||
const STT_TRANSCRIPT = 'hello from PTT';
|
||
|
||
const OVERLAY_WINDOW_LABEL = 'ptt-overlay';
|
||
// pttService.minAudioMs is 250; we hold for 800 ms to be comfortably above the
|
||
// floor and tolerant of slow CI scheduling.
|
||
const HOLD_DURATION_MS = 800;
|
||
|
||
describe('PTT — global push-to-talk flow', function () {
|
||
this.timeout(180_000);
|
||
|
||
before(async function beforeSuite() {
|
||
this.timeout(120_000);
|
||
await startMockServer();
|
||
await waitForApp();
|
||
await resetApp(USER_ID);
|
||
|
||
// The cloud STT path goes through /openai/v1/audio/transcriptions in the
|
||
// mock backend; set the deterministic transcript before any PTT press.
|
||
setMockBehavior('audioTranscriptionText', STT_TRANSCRIPT);
|
||
});
|
||
|
||
after(async () => {
|
||
setMockBehavior('audioTranscriptionText', '');
|
||
await stopMockServer();
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Step 1: settings → voice → PttSettingsPanel.
|
||
// ---------------------------------------------------------------------------
|
||
it('renders the PTT settings panel under /settings/voice', async () => {
|
||
await navigateViaHash('/settings/voice');
|
||
|
||
// The panel may take a beat to mount as VoicePanel hydrates its providers.
|
||
const panel = await browser.$('[data-testid="ptt-settings-panel"]');
|
||
await panel.waitForExist({
|
||
timeout: 20_000,
|
||
timeoutMsg: 'ptt-settings-panel did not mount under /settings/voice',
|
||
});
|
||
|
||
// The hotkey input + the two switches must all be present (T13 contract).
|
||
const shortcutInput = await browser.$('[data-testid="ptt-shortcut-input"]');
|
||
await shortcutInput.waitForExist({ timeout: 5_000 });
|
||
const speakSwitch = await browser.$('[data-testid="ptt-speak-replies-switch"]');
|
||
await speakSwitch.waitForExist({ timeout: 5_000 });
|
||
const overlaySwitch = await browser.$('[data-testid="ptt-show-overlay-switch"]');
|
||
await overlaySwitch.waitForExist({ timeout: 5_000 });
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Step 2 + 3: bind the shortcut, observe Redux + register_ptt_hotkey.
|
||
//
|
||
// We drive Redux directly. The shortcut-capture input is exhaustively
|
||
// covered by PttSettingsPanel.test.tsx; here we test the *binding effect*
|
||
// — that setting the shortcut triggers the manager hook which calls
|
||
// register_ptt_hotkey in the Tauri shell.
|
||
// ---------------------------------------------------------------------------
|
||
it('binds the F13 hotkey via Redux + the manager hook forwards to the Tauri shell', async () => {
|
||
// Sanity: store handle is exposed (gated on E2E build flag).
|
||
const storePresent = await browser.execute(
|
||
() =>
|
||
typeof (window as unknown as { __OPENHUMAN_STORE__?: unknown }).__OPENHUMAN_STORE__ !==
|
||
'undefined'
|
||
);
|
||
expect(storePresent).toBe(true);
|
||
|
||
// Speak replies must be true so the chat-send carries speak_reply: true.
|
||
// showOverlay must be true so the manager invokes show_ptt_overlay on
|
||
// the start edge (overlay window check below depends on it).
|
||
await browser.execute(() => {
|
||
const store = (
|
||
window as unknown as {
|
||
__OPENHUMAN_STORE__: { dispatch: (a: { type: string; payload: unknown }) => unknown };
|
||
}
|
||
).__OPENHUMAN_STORE__;
|
||
store.dispatch({ type: 'ptt/setSpeakReplies', payload: true });
|
||
store.dispatch({ type: 'ptt/setShowOverlay', payload: true });
|
||
});
|
||
|
||
// Dispatch the binding.
|
||
await browser.execute((shortcut: string) => {
|
||
const store = (
|
||
window as unknown as {
|
||
__OPENHUMAN_STORE__: { dispatch: (a: { type: string; payload: string }) => unknown };
|
||
}
|
||
).__OPENHUMAN_STORE__;
|
||
store.dispatch({ type: 'ptt/setPttShortcut', payload: shortcut });
|
||
}, SHORTCUT);
|
||
|
||
// Wait until the slice settles with the bound shortcut.
|
||
await browser.waitUntil(
|
||
async () => {
|
||
return (
|
||
(await browser.execute(() => {
|
||
const state = (
|
||
window as unknown as {
|
||
__OPENHUMAN_STORE__: { getState: () => { ptt?: { shortcut?: string | null } } };
|
||
}
|
||
).__OPENHUMAN_STORE__.getState();
|
||
return state.ptt?.shortcut ?? null;
|
||
})) === SHORTCUT
|
||
);
|
||
},
|
||
{ timeout: 5_000, timeoutMsg: 'ptt.shortcut never settled to F13' }
|
||
);
|
||
|
||
// Give usePttHotkey a beat to call register_ptt_hotkey, then read the
|
||
// registration-error slice. A null (or empty) error means the Tauri
|
||
// shell registered the OS shortcut successfully. A non-null error is
|
||
// acceptable in headless Linux containers where the global-shortcut
|
||
// plugin can't talk to a real X11 / Wayland socket — we log and
|
||
// continue rather than fail the spec on env-specific gaps.
|
||
await browser.pause(2_000);
|
||
const registrationError = await browser.execute(() => {
|
||
const state = (
|
||
window as unknown as {
|
||
__OPENHUMAN_STORE__: { getState: () => { ptt?: { registrationError?: string | null } } };
|
||
}
|
||
).__OPENHUMAN_STORE__.getState();
|
||
return state.ptt?.registrationError ?? null;
|
||
});
|
||
if (registrationError) {
|
||
console.warn(
|
||
`[ptt-flow] register_ptt_hotkey returned error in this environment: ${registrationError}. ` +
|
||
'Continuing — the binding-side wiring was driven and the failure is the OS shortcut path.'
|
||
);
|
||
} else {
|
||
console.log('[ptt-flow] register_ptt_hotkey succeeded — overlay window should now exist');
|
||
}
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Step 8: overlay window is created lazily by register_ptt_hotkey.
|
||
//
|
||
// We check getWindowHandles. The handle count goes from 1 (main app) →
|
||
// 2 (main + ptt-overlay) once ensure_window has run. We tolerate either
|
||
// outcome: if the OS shortcut failed earlier (headless container), the
|
||
// overlay might still be created (ensure_window is best-effort and runs
|
||
// before the shortcut registration), but we don't *require* it to assert
|
||
// success.
|
||
// ---------------------------------------------------------------------------
|
||
it('lazy-creates the overlay webview window once the hotkey is bound', async () => {
|
||
// Poll briefly — window creation is async after register_ptt_hotkey returns.
|
||
const deadline = Date.now() + 10_000;
|
||
let handles: string[] = [];
|
||
while (Date.now() < deadline) {
|
||
handles = await browser.getWindowHandles();
|
||
if (handles.length >= 2) break;
|
||
await browser.pause(300);
|
||
}
|
||
console.log(`[ptt-flow] window handles after bind: ${handles.length}`);
|
||
if (handles.length < 2) {
|
||
console.warn(
|
||
'[ptt-flow] overlay window did not appear — likely register_ptt_hotkey failed on this OS ' +
|
||
'(see registrationError log above). Skipping overlay-window assertion.'
|
||
);
|
||
return;
|
||
}
|
||
// Confirm at least one of the new handles loads the ptt-overlay route.
|
||
const mainHandle = await browser.getWindowHandle();
|
||
let foundOverlay = false;
|
||
for (const handle of handles) {
|
||
if (handle === mainHandle) continue;
|
||
try {
|
||
await browser.switchToWindow(handle);
|
||
const url = await browser.getUrl();
|
||
console.log(`[ptt-flow] inspecting non-main window: ${url}`);
|
||
if (url.includes('ptt-overlay') || url.includes(OVERLAY_WINDOW_LABEL)) {
|
||
foundOverlay = true;
|
||
break;
|
||
}
|
||
} catch (err) {
|
||
console.warn('[ptt-flow] switchToWindow threw — continuing', err);
|
||
}
|
||
}
|
||
// Switch back to the main window before the next test runs.
|
||
try {
|
||
await browser.switchToWindow(mainHandle);
|
||
} catch (err) {
|
||
console.warn('[ptt-flow] could not switch back to main window', err);
|
||
}
|
||
expect(foundOverlay).toBe(true);
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Step 4–7 + 9–10: simulate the hold, observe the commit.
|
||
// ---------------------------------------------------------------------------
|
||
it('simulates a PTT hold, captures audio, transcribes via mock, sends with speak_reply: true', async function () {
|
||
this.timeout(120_000);
|
||
|
||
// Make sure the user is signed in + the socket is connected so the
|
||
// channel_web_chat RPC has a real client_id to route on.
|
||
const socketReady = await waitForSocketConnected(30_000);
|
||
if (!socketReady) {
|
||
console.warn('[ptt-flow] socket did not connect within 30s — chat send may fail');
|
||
}
|
||
|
||
// Navigate to /chat so the chat runtime is hydrated and we land on a
|
||
// resolvable thread. pttThread.ts will resolve the active thread or
|
||
// create one as needed; this just makes the assertion at step 9
|
||
// easier (we can read selectedThreadId and assert message presence).
|
||
await navigateViaHash('/chat');
|
||
await browser.waitUntil(async () => await chatMounted(), {
|
||
timeout: 15_000,
|
||
timeoutMsg: 'Conversations did not mount under /chat',
|
||
});
|
||
|
||
// -------------------------------------------------------------------------
|
||
// 4a. Mock getUserMedia + MediaRecorder so pttAudio.ts succeeds.
|
||
//
|
||
// We replace getUserMedia with a fake that returns a MediaStream-shaped
|
||
// object; we replace MediaRecorder with a minimal stub that fires
|
||
// 'dataavailable' (empty Blob) and 'stop' synchronously when .stop() is
|
||
// called. The audio buffer ends up zero-byte — the mock STT endpoint
|
||
// returns the fixed transcript regardless.
|
||
// -------------------------------------------------------------------------
|
||
await browser.execute(() => {
|
||
const w = window as unknown as Record<string, unknown>;
|
||
w.__e2e_ptt_real_gum = navigator.mediaDevices?.getUserMedia?.bind(navigator.mediaDevices);
|
||
w.__e2e_ptt_real_mr = (window as unknown as { MediaRecorder?: unknown }).MediaRecorder;
|
||
|
||
class FakeMediaRecorder {
|
||
public state: 'inactive' | 'recording' = 'inactive';
|
||
public mimeType: string;
|
||
private listeners = new Map<string, Set<(e: unknown) => void>>();
|
||
constructor(_stream: unknown, opts?: { mimeType?: string }) {
|
||
this.mimeType = opts?.mimeType || 'audio/webm;codecs=opus';
|
||
}
|
||
static isTypeSupported(_mime: string): boolean {
|
||
return true;
|
||
}
|
||
addEventListener(type: string, fn: (e: unknown) => void): void {
|
||
if (!this.listeners.has(type)) this.listeners.set(type, new Set());
|
||
this.listeners.get(type)!.add(fn);
|
||
}
|
||
removeEventListener(type: string, fn: (e: unknown) => void): void {
|
||
this.listeners.get(type)?.delete(fn);
|
||
}
|
||
dispatchEvent(type: string, payload: unknown): void {
|
||
const set = this.listeners.get(type);
|
||
if (!set) return;
|
||
for (const fn of set) {
|
||
try {
|
||
fn(payload);
|
||
} catch (err) {
|
||
// swallow — listener failures shouldn't break the test
|
||
console.warn('[e2e-ptt-mock] listener threw', err);
|
||
}
|
||
}
|
||
}
|
||
start(): void {
|
||
this.state = 'recording';
|
||
}
|
||
stop(): void {
|
||
// Emit a tiny synthetic chunk + a stop event. pttAudio expects
|
||
// dataavailable with .data:Blob and then stop.
|
||
const blob = new Blob([new Uint8Array(8)], { type: this.mimeType });
|
||
this.dispatchEvent('dataavailable', { data: blob });
|
||
this.state = 'inactive';
|
||
this.dispatchEvent('stop', new Event('stop'));
|
||
}
|
||
}
|
||
|
||
const fakeStream = {
|
||
getTracks: () => [
|
||
{
|
||
stop() {
|
||
/* noop */
|
||
},
|
||
kind: 'audio' as const,
|
||
},
|
||
],
|
||
};
|
||
|
||
Object.defineProperty(navigator, 'mediaDevices', {
|
||
configurable: true,
|
||
value: {
|
||
...(navigator.mediaDevices || {}),
|
||
getUserMedia: () => Promise.resolve(fakeStream as unknown as MediaStream),
|
||
},
|
||
});
|
||
(window as unknown as { MediaRecorder: unknown }).MediaRecorder =
|
||
FakeMediaRecorder as unknown;
|
||
});
|
||
|
||
// -------------------------------------------------------------------------
|
||
// 10a. Spy on Tauri invocations so we can capture the channel_web_chat
|
||
// payload and assert speak_reply: true was forwarded on the wire.
|
||
//
|
||
// __TAURI_INTERNALS__.invoke is the underlying channel every Tauri
|
||
// command (and `core_rpc_relay`) flows through. We wrap it to push
|
||
// relay calls into a module-window-scoped list.
|
||
// -------------------------------------------------------------------------
|
||
await browser.execute(() => {
|
||
const w = window as unknown as {
|
||
__TAURI_INTERNALS__?: {
|
||
invoke?: (...args: unknown[]) => Promise<unknown>;
|
||
[k: string]: unknown;
|
||
};
|
||
__e2e_ptt_relay_calls?: Array<{ cmd: string; args: unknown }>;
|
||
__e2e_ptt_real_invoke?: (...args: unknown[]) => Promise<unknown>;
|
||
};
|
||
if (!w.__TAURI_INTERNALS__ || typeof w.__TAURI_INTERNALS__.invoke !== 'function') {
|
||
console.warn('[e2e-ptt-spy] __TAURI_INTERNALS__.invoke missing — spy not installed');
|
||
return;
|
||
}
|
||
w.__e2e_ptt_relay_calls = [];
|
||
w.__e2e_ptt_real_invoke = w.__TAURI_INTERNALS__.invoke;
|
||
const original = w.__e2e_ptt_real_invoke;
|
||
w.__TAURI_INTERNALS__.invoke = async function spied(
|
||
cmd: string,
|
||
args?: unknown,
|
||
...rest: unknown[]
|
||
): Promise<unknown> {
|
||
try {
|
||
if (cmd === 'core_rpc_relay') {
|
||
w.__e2e_ptt_relay_calls!.push({ cmd, args });
|
||
}
|
||
} catch {
|
||
/* ignore */
|
||
}
|
||
// Forward to the original implementation, preserving binding.
|
||
return (original as Function).call(w.__TAURI_INTERNALS__, cmd, args, ...rest);
|
||
};
|
||
});
|
||
|
||
clearRequestLog();
|
||
const threadIdBefore = await getSelectedThreadId();
|
||
console.log(`[ptt-flow] selectedThreadId before press: ${threadIdBefore}`);
|
||
|
||
// -------------------------------------------------------------------------
|
||
// 6. Simulate the hotkey hold by emitting ptt://start and ptt://stop
|
||
// via Tauri's internal event plugin. PttHotkeyManager's listen()
|
||
// handlers pick these up and drive pttService through onStart/onStop.
|
||
// -------------------------------------------------------------------------
|
||
const sessionId = 1;
|
||
const emitOk = await browser.execute(
|
||
async ({ event, payloadJson }) => {
|
||
const w = window as unknown as {
|
||
__TAURI_INTERNALS__?: { invoke?: (...args: unknown[]) => Promise<unknown> };
|
||
};
|
||
const invoke = w.__TAURI_INTERNALS__?.invoke;
|
||
if (!invoke) return { ok: false, err: 'no __TAURI_INTERNALS__.invoke' };
|
||
try {
|
||
// plugin:event|emit accepts a JSON-string payload for arbitrary
|
||
// event types (the listener side is generic-typed).
|
||
await invoke('plugin:event|emit', { event, payload: payloadJson });
|
||
return { ok: true };
|
||
} catch (e) {
|
||
return { ok: false, err: e instanceof Error ? e.message : String(e) };
|
||
}
|
||
},
|
||
{ event: 'ptt://start', payloadJson: JSON.stringify({ session_id: sessionId }) }
|
||
);
|
||
if (!emitOk?.ok) {
|
||
console.warn(`[ptt-flow] emit ptt://start failed: ${emitOk?.err}`);
|
||
}
|
||
|
||
// Hold for HOLD_DURATION_MS so the recording isn't dropped as a tap.
|
||
await browser.pause(HOLD_DURATION_MS);
|
||
|
||
const stopOk = await browser.execute(
|
||
async ({ event, payloadJson }) => {
|
||
const w = window as unknown as {
|
||
__TAURI_INTERNALS__?: { invoke?: (...args: unknown[]) => Promise<unknown> };
|
||
};
|
||
const invoke = w.__TAURI_INTERNALS__?.invoke;
|
||
if (!invoke) return { ok: false, err: 'no __TAURI_INTERNALS__.invoke' };
|
||
try {
|
||
await invoke('plugin:event|emit', { event, payload: payloadJson });
|
||
return { ok: true };
|
||
} catch (e) {
|
||
return { ok: false, err: e instanceof Error ? e.message : String(e) };
|
||
}
|
||
},
|
||
{ event: 'ptt://stop', payloadJson: JSON.stringify({ session_id: sessionId }) }
|
||
);
|
||
if (!stopOk?.ok) {
|
||
console.warn(`[ptt-flow] emit ptt://stop failed: ${stopOk?.err}`);
|
||
}
|
||
|
||
// -------------------------------------------------------------------------
|
||
// 9. The transcript should appear as a user message in the chat thread.
|
||
// -------------------------------------------------------------------------
|
||
const sawTranscript = await waitForAssistantReplyContaining(STT_TRANSCRIPT, {
|
||
timeoutMs: 30_000,
|
||
logPrefix: '[ptt-flow]',
|
||
});
|
||
if (!sawTranscript) {
|
||
console.warn(
|
||
`[ptt-flow] transcript "${STT_TRANSCRIPT}" did not appear in DOM — ` +
|
||
'this is often caused by getUserMedia mock injection failing under headless CEF, ' +
|
||
'or by register_ptt_hotkey having failed earlier so pttService never received ptt://start.'
|
||
);
|
||
}
|
||
|
||
// -------------------------------------------------------------------------
|
||
// 10b. Assert at least one core_rpc_relay invocation included
|
||
// method: 'openhuman.channel_web_chat' with speak_reply: true.
|
||
// -------------------------------------------------------------------------
|
||
const relayCalls = (await browser.execute(() => {
|
||
return (window as unknown as { __e2e_ptt_relay_calls?: unknown[] }).__e2e_ptt_relay_calls;
|
||
})) as Array<{ cmd: string; args: unknown }> | undefined;
|
||
console.log(`[ptt-flow] captured ${relayCalls?.length ?? 0} core_rpc_relay invocations`);
|
||
|
||
let sawSpeakReplyChat = false;
|
||
for (const call of relayCalls ?? []) {
|
||
try {
|
||
// Tauri's invoke signature is (cmd, args) where args is a record.
|
||
// For core_rpc_relay the renderer passes either a record like
|
||
// { method, params, body } or a single string — we coerce robustly.
|
||
const args = call.args as Record<string, unknown> | undefined;
|
||
const payload = args && typeof args === 'object' ? JSON.stringify(args) : String(args);
|
||
if (
|
||
payload.includes('openhuman.channel_web_chat') &&
|
||
payload.includes('"speak_reply":true')
|
||
) {
|
||
sawSpeakReplyChat = true;
|
||
break;
|
||
}
|
||
} catch {
|
||
/* ignore non-stringifiable payloads */
|
||
}
|
||
}
|
||
if (!sawSpeakReplyChat) {
|
||
console.warn(
|
||
'[ptt-flow] did not observe a channel_web_chat call with speak_reply:true. ' +
|
||
'Dumping the captured payloads for diagnosis:\n' +
|
||
JSON.stringify(relayCalls ?? [], null, 2).slice(0, 4_000)
|
||
);
|
||
}
|
||
|
||
// Restore the spy + getUserMedia/MediaRecorder so any later spec in the
|
||
// session sees a clean window.
|
||
await browser.execute(() => {
|
||
const w = window as unknown as {
|
||
__TAURI_INTERNALS__?: { invoke?: unknown };
|
||
__e2e_ptt_real_invoke?: unknown;
|
||
__e2e_ptt_real_gum?: unknown;
|
||
__e2e_ptt_real_mr?: unknown;
|
||
};
|
||
if (w.__TAURI_INTERNALS__ && w.__e2e_ptt_real_invoke) {
|
||
w.__TAURI_INTERNALS__.invoke = w.__e2e_ptt_real_invoke;
|
||
}
|
||
if (w.__e2e_ptt_real_gum && navigator.mediaDevices) {
|
||
Object.defineProperty(navigator.mediaDevices, 'getUserMedia', {
|
||
configurable: true,
|
||
value: w.__e2e_ptt_real_gum,
|
||
});
|
||
}
|
||
if (w.__e2e_ptt_real_mr) {
|
||
(window as unknown as { MediaRecorder: unknown }).MediaRecorder = w.__e2e_ptt_real_mr;
|
||
}
|
||
delete (w as Record<string, unknown>).__e2e_ptt_relay_calls;
|
||
delete (w as Record<string, unknown>).__e2e_ptt_real_invoke;
|
||
delete (w as Record<string, unknown>).__e2e_ptt_real_gum;
|
||
delete (w as Record<string, unknown>).__e2e_ptt_real_mr;
|
||
});
|
||
|
||
// Soft-assert: in a fully green environment both flags are true. We
|
||
// expect both, but the warnings above explain the env paths where one
|
||
// might come back false. Asserting hard would gate CI on shaky pieces.
|
||
expect(sawTranscript).toBe(true);
|
||
expect(sawSpeakReplyChat).toBe(true);
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Step 5 corroboration: the mock STT endpoint was hit.
|
||
//
|
||
// We assert the request log contains a POST to
|
||
// /openai/v1/audio/transcriptions. This is independent of the spy above —
|
||
// it confirms the audio bytes actually traversed the Rust STT pipeline
|
||
// (voice_transcribe_bytes RPC → cloud provider → mock).
|
||
// ---------------------------------------------------------------------------
|
||
it('the mock backend received the audio-transcriptions request', async () => {
|
||
const log = getRequestLog() as Array<{ method: string; url: string }>;
|
||
const sttCalls = log.filter(
|
||
r => r.method === 'POST' && r.url.includes('/openai/v1/audio/transcriptions')
|
||
);
|
||
console.log(`[ptt-flow] /openai/v1/audio/transcriptions calls observed: ${sttCalls.length}`);
|
||
// The earlier "PTT session" test logs a warning rather than failing if the
|
||
// OS shortcut couldn't register. In that case the audio path may never
|
||
// have triggered — log and move on rather than make CI red on env gaps.
|
||
if (sttCalls.length === 0) {
|
||
console.warn(
|
||
'[ptt-flow] no audio-transcriptions calls observed. ' +
|
||
'Most likely cause: the renderer-side audio capture mock or the ptt://start emit ' +
|
||
'did not fully exercise the pttService path. The earlier in-flight steps log ' +
|
||
'their specific failures.'
|
||
);
|
||
}
|
||
expect(sttCalls.length).toBeGreaterThan(0);
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Optional sanity: the conversation persists with the transcript text.
|
||
//
|
||
// Uses the same test_support_read_workspace_file mechanism as the chat-
|
||
// harness specs (see chat-harness-send-stream.spec.ts).
|
||
// ---------------------------------------------------------------------------
|
||
it('the chat thread JSONL contains the transcribed text on disk', async () => {
|
||
const threadId = await getSelectedThreadId();
|
||
if (typeof threadId !== 'string' || threadId.length === 0) {
|
||
console.warn('[ptt-flow] no selectedThreadId after press — skipping JSONL check');
|
||
return;
|
||
}
|
||
const hex = Array.from(new TextEncoder().encode(threadId))
|
||
.map(b => b.toString(16).padStart(2, '0'))
|
||
.join('');
|
||
const relPath = `memory/conversations/threads/${hex}.jsonl`;
|
||
let content = '';
|
||
const deadline = Date.now() + 10_000;
|
||
while (Date.now() < deadline) {
|
||
const read = await callOpenhumanRpc<{ result: { content_utf8: string } }>(
|
||
'openhuman.test_support_read_workspace_file',
|
||
{ rel_path: relPath, max_bytes: 65_536 }
|
||
);
|
||
if (read.ok && read.result?.result?.content_utf8) {
|
||
content = read.result.result.content_utf8;
|
||
if (content.includes(STT_TRANSCRIPT)) break;
|
||
}
|
||
await browser.pause(300);
|
||
}
|
||
if (!content.includes(STT_TRANSCRIPT)) {
|
||
console.warn(
|
||
`[ptt-flow] thread JSONL did not contain "${STT_TRANSCRIPT}". This corroborates ` +
|
||
'an earlier failure in the press path; the earlier `it` logs the specific cause.'
|
||
);
|
||
}
|
||
expect(content).toContain(STT_TRANSCRIPT);
|
||
});
|
||
});
|