From 811669b61ca7bc52627a0727f20851a6fd51ca27 Mon Sep 17 00:00:00 2001 From: Cyrus Gray <144336577+graycyrus@users.noreply.github.com> Date: Wed, 1 Jul 2026 14:43:50 +0530 Subject: [PATCH] feat(meet): live meeting transcript during the call (#4350) --- .../meetings/ActiveMeetingBanner.tsx | 10 ++ .../meetings/LiveTranscriptPanel.tsx | 104 +++++++++++ .../__tests__/ActiveMeetingBanner.test.tsx | 9 +- .../__tests__/LiveTranscriptPanel.test.tsx | 62 +++++++ .../skills/__tests__/MeetingBotsCard.test.tsx | 4 + app/src/lib/i18n/ar.ts | 2 + app/src/lib/i18n/bn.ts | 2 + app/src/lib/i18n/de.ts | 2 + app/src/lib/i18n/en.ts | 2 + app/src/lib/i18n/es.ts | 2 + app/src/lib/i18n/fr.ts | 2 + app/src/lib/i18n/hi.ts | 2 + app/src/lib/i18n/id.ts | 2 + app/src/lib/i18n/it.ts | 2 + app/src/lib/i18n/ko.ts | 2 + app/src/lib/i18n/pl.ts | 2 + app/src/lib/i18n/pt.ts | 2 + app/src/lib/i18n/ru.ts | 2 + app/src/lib/i18n/zh-CN.ts | 2 + .../__tests__/socketService.events.test.ts | 43 +++++ app/src/services/socketService.ts | 28 +++ app/src/store/backendMeetSlice.test.ts | 161 ++++++++++++++++++ app/src/store/backendMeetSlice.ts | 70 ++++++++ src/core/event_bus/events.rs | 14 ++ src/core/socketio.rs | 19 +++ src/openhuman/socket/event_handlers.rs | 85 +++++++++ 26 files changed, 636 insertions(+), 1 deletion(-) create mode 100644 app/src/components/meetings/LiveTranscriptPanel.tsx create mode 100644 app/src/components/meetings/__tests__/LiveTranscriptPanel.test.tsx diff --git a/app/src/components/meetings/ActiveMeetingBanner.tsx b/app/src/components/meetings/ActiveMeetingBanner.tsx index 16dce29f0..a7c35fefe 100644 --- a/app/src/components/meetings/ActiveMeetingBanner.tsx +++ b/app/src/components/meetings/ActiveMeetingBanner.tsx @@ -19,12 +19,15 @@ import { selectBackendMeetLastHarness, selectBackendMeetLastReply, selectBackendMeetListenOnly, + selectBackendMeetLivePartialIndex, + selectBackendMeetLiveTranscript, selectBackendMeetStatus, selectBackendMeetUrl, } from '../../store/backendMeetSlice'; import { useAppDispatch, useAppSelector } from '../../store/hooks'; import Button from '../ui/Button'; import { Spinner } from '../ui/icons'; +import { LiveTranscriptPanel } from './LiveTranscriptPanel'; type Toast = { type: 'success' | 'error' | 'info'; title: string; message?: string }; @@ -70,6 +73,8 @@ export function ActiveMeetingBanner({ onToast }: ActiveMeetingBannerProps) { const listenOnly = useAppSelector(selectBackendMeetListenOnly); const lastReply = useAppSelector(selectBackendMeetLastReply); const lastHarness = useAppSelector(selectBackendMeetLastHarness); + const liveTranscript = useAppSelector(selectBackendMeetLiveTranscript); + const livePartialIndex = useAppSelector(selectBackendMeetLivePartialIndex); // selectBackendMeetError imported for parity; not used visually here — errors // surface in the composer's inline alert during the error state. const face = faceFromMeetState(status, lastReply, lastHarness); @@ -183,6 +188,11 @@ export function ActiveMeetingBanner({ onToast }: ActiveMeetingBannerProps) { )} + {/* Live transcript during an in-progress call (issue #4304). Hidden once + the call ends — the final transcript is shown elsewhere in history. */} + {(status === 'active' || status === 'joining') && ( + + )} ); } diff --git a/app/src/components/meetings/LiveTranscriptPanel.tsx b/app/src/components/meetings/LiveTranscriptPanel.tsx new file mode 100644 index 000000000..b4aa243a7 --- /dev/null +++ b/app/src/components/meetings/LiveTranscriptPanel.tsx @@ -0,0 +1,104 @@ +/** + * LiveTranscriptPanel — renders meeting transcript turns as they stream in + * during an active call (issue #4304). + * + * Fed by the `liveTranscript` buffer in `backendMeetSlice`, which accumulates + * `agent_meetings:transcript_delta` events. The line at `partialIndex` is shown + * greyed/italic until the backend finalizes it. The list autoscrolls to the + * newest turn. On call end the buffer is reconciled away (the authoritative + * final transcript takes over), so this panel naturally empties. + */ +import debug from 'debug'; +import { useEffect, useRef } from 'react'; + +import { useT } from '../../lib/i18n/I18nContext'; +import type { BackendMeetTurn } from '../../store/backendMeetSlice'; + +const log = debug('meetings:live-transcript'); + +/** + * Live delta turns carry the speaker inline as a `[Name]` prefix (every human + * speaker has `role: 'user'`, so the real identity lives in the content), and + * may optionally be preceded by a `[MM:SS]` timestamp. Pull the leading + * speaker tag out so it can be rendered as a label, mirroring how the final + * transcript is shown via `parseTranscriptLine`. Falls back to no speaker when + * the content has no tag (e.g. the assistant's own turns). + */ +const LIVE_SPEAKER_RE = /^\s*(?:\[\d{1,2}:\d{2}\]\s*)?\[([^\]]+)\]\s*([\s\S]*)$/; + +function parseLiveLine(content: string): { speaker: string | null; text: string } { + const match = LIVE_SPEAKER_RE.exec(content); + if (match) return { speaker: match[1] ?? null, text: match[2] ?? '' }; + return { speaker: null, text: content }; +} + +export interface LiveTranscriptPanelProps { + turns: BackendMeetTurn[]; + partialIndex: number | null; +} + +export function LiveTranscriptPanel({ turns = [], partialIndex }: LiveTranscriptPanelProps) { + const { t } = useT(); + const scrollRef = useRef(null); + + // `turns` is keyed by the backend transcript index and can be sparse (skipped + // `[System]` turns leave gaps), so render only the populated slots while + // keeping each turn's real index for the partial-line comparison. Default to + // an empty array so a caller that hasn't seeded the live buffer (e.g. a store + // built before this slice field existed) renders the empty state, not a crash. + const rows = turns + .map((turn, index) => ({ turn, index })) + .filter((row): row is { turn: BackendMeetTurn; index: number } => Boolean(row.turn)); + + // Autoscroll to the newest turn whenever a turn is added or the tail line is + // updated (partial → final, or partial text extended). + useEffect(() => { + const el = scrollRef.current; + if (!el) return; + el.scrollTop = el.scrollHeight; + log('[live-transcript] autoscroll rows=%d partialIndex=%o', rows.length, partialIndex); + }, [rows.length, partialIndex, turns]); + + return ( +
+
+
+
+ {rows.length === 0 ? ( +

+ {t('skills.meetingBots.liveTranscriptEmpty')} +

+ ) : ( + rows.map(({ turn, index }) => { + const isAssistant = turn.role === 'assistant'; + const isPartial = index === partialIndex; + const { speaker, text } = parseLiveLine(turn.content); + return ( +

+ {speaker && {speaker}:} + {text} +

+ ); + }) + )} +
+
+ ); +} + +export default LiveTranscriptPanel; diff --git a/app/src/components/meetings/__tests__/ActiveMeetingBanner.test.tsx b/app/src/components/meetings/__tests__/ActiveMeetingBanner.test.tsx index 70ee918cd..0c330c5e7 100644 --- a/app/src/components/meetings/__tests__/ActiveMeetingBanner.test.tsx +++ b/app/src/components/meetings/__tests__/ActiveMeetingBanner.test.tsx @@ -28,6 +28,8 @@ const joiningState = { lastReply: null, lastHarness: null, transcript: null, + liveTranscript: [], + livePartialIndex: null, error: null, }, }; @@ -41,6 +43,8 @@ const activeState = { lastReply: null, lastHarness: null, transcript: null, + liveTranscript: [], + livePartialIndex: null, error: null, }, }; @@ -54,7 +58,8 @@ describe('ActiveMeetingBanner', () => { it('renders joining state with LIVE badge', () => { renderWithProviders(, { preloadedState: joiningState }); - expect(screen.getByText(/live/i)).toBeInTheDocument(); + // Exact match: the "Live" badge, not the "Live transcript" panel heading. + expect(screen.getByText('Live')).toBeInTheDocument(); expect(screen.getByText(/joining/i)).toBeInTheDocument(); }); @@ -203,6 +208,8 @@ describe('ActiveMeetingBanner', () => { lastReply: null, lastHarness: null, transcript: null, + liveTranscript: [], + livePartialIndex: null, error: null, }, }; diff --git a/app/src/components/meetings/__tests__/LiveTranscriptPanel.test.tsx b/app/src/components/meetings/__tests__/LiveTranscriptPanel.test.tsx new file mode 100644 index 000000000..84823d65c --- /dev/null +++ b/app/src/components/meetings/__tests__/LiveTranscriptPanel.test.tsx @@ -0,0 +1,62 @@ +import { cleanup, screen } from '@testing-library/react'; +import { afterEach, describe, expect, it } from 'vitest'; + +import { renderWithProviders } from '../../../test/test-utils'; +import { LiveTranscriptPanel } from '../LiveTranscriptPanel'; + +describe('LiveTranscriptPanel (#4304)', () => { + afterEach(() => cleanup()); + + it('shows an empty/waiting state when there are no turns', () => { + renderWithProviders(); + expect(screen.getByText(/waiting for speech/i)).toBeInTheDocument(); + // Heading is always present. + expect(screen.getByText(/live transcript/i)).toBeInTheDocument(); + }); + + it('renders turns with their role label and content', () => { + renderWithProviders( + + ); + expect(screen.getByText('Hello team')).toBeInTheDocument(); + expect(screen.getByText('Hi there')).toBeInTheDocument(); + expect(screen.queryByText(/waiting for speech/i)).not.toBeInTheDocument(); + }); + + it('renders the inline [Speaker] tag as a label, not raw bracket text', () => { + renderWithProviders( + + ); + // Speaker is pulled out of the content and shown as a label. + expect(screen.getByText('Alice:')).toBeInTheDocument(); + expect(screen.getByText('hello there')).toBeInTheDocument(); + // The raw bracketed form is not shown. + expect(screen.queryByText(/\[Alice\]/)).not.toBeInTheDocument(); + }); + + it('greys the partial line at partialIndex', () => { + renderWithProviders( + + ); + const partial = screen.getByText('partial line').closest('p'); + const settled = screen.getByText('final line').closest('p'); + expect(partial?.className).toContain('text-content-faint'); + expect(partial?.className).toContain('italic'); + expect(settled?.className).not.toContain('text-content-faint'); + }); +}); diff --git a/app/src/components/skills/__tests__/MeetingBotsCard.test.tsx b/app/src/components/skills/__tests__/MeetingBotsCard.test.tsx index 69e8170d4..bad88ab86 100644 --- a/app/src/components/skills/__tests__/MeetingBotsCard.test.tsx +++ b/app/src/components/skills/__tests__/MeetingBotsCard.test.tsx @@ -238,9 +238,13 @@ const activeMeetState = { backendMeet: { status: 'active' as const, meetUrl: 'https://meet.google.com/abc-defg-hij', + meetingId: null, + listenOnly: false, lastReply: null, lastHarness: null, transcript: null, + liveTranscript: [], + livePartialIndex: null, error: null, }, }; diff --git a/app/src/lib/i18n/ar.ts b/app/src/lib/i18n/ar.ts index 814e56378..f532b8fbe 100644 --- a/app/src/lib/i18n/ar.ts +++ b/app/src/lib/i18n/ar.ts @@ -5094,6 +5094,8 @@ const messages: TranslationMap = { 'skills.meetingBots.liveStatusListening': 'الاستماع (صامت)', 'skills.meetingBots.liveStatusEnded': 'انتهى الاجتماع', 'skills.meetingBots.liveStatusError': 'فشل الانضمام', + 'skills.meetingBots.liveTranscriptHeading': 'النص المباشر', + 'skills.meetingBots.liveTranscriptEmpty': 'في انتظار الكلام…', 'skills.meetingBots.leaveButton': 'مغادرة', 'skills.meetingBots.leavingButton': 'جارٍ المغادرة…', 'skills.meetingBots.respondToParticipant': 'اسمك في هذا الاجتماع', diff --git a/app/src/lib/i18n/bn.ts b/app/src/lib/i18n/bn.ts index 4dfc4546d..3026f7d71 100644 --- a/app/src/lib/i18n/bn.ts +++ b/app/src/lib/i18n/bn.ts @@ -5199,6 +5199,8 @@ const messages: TranslationMap = { 'skills.meetingBots.liveStatusListening': 'শুনছে (নিঃশব্দ)', 'skills.meetingBots.liveStatusEnded': 'মিটিং শেষ', 'skills.meetingBots.liveStatusError': 'যোগ দিতে ব্যর্থ', + 'skills.meetingBots.liveTranscriptHeading': 'লাইভ ট্রান্সক্রিপ্ট', + 'skills.meetingBots.liveTranscriptEmpty': 'কথার জন্য অপেক্ষা করা হচ্ছে…', 'skills.meetingBots.leaveButton': 'ছেড়ে দিন', 'skills.meetingBots.leavingButton': 'বেরিয়ে যাচ্ছে…', 'skills.meetingBots.respondToParticipant': 'এই মিটিংয়ে আপনার নাম', diff --git a/app/src/lib/i18n/de.ts b/app/src/lib/i18n/de.ts index c79d3a150..ccc1332c4 100644 --- a/app/src/lib/i18n/de.ts +++ b/app/src/lib/i18n/de.ts @@ -5332,6 +5332,8 @@ const messages: TranslationMap = { 'skills.meetingBots.liveStatusListening': 'Zuhören (stumm)', 'skills.meetingBots.liveStatusEnded': 'Meeting beendet', 'skills.meetingBots.liveStatusError': 'Beitritt fehlgeschlagen', + 'skills.meetingBots.liveTranscriptHeading': 'Live-Transkript', + 'skills.meetingBots.liveTranscriptEmpty': 'Warte auf Sprache…', 'skills.meetingBots.leaveButton': 'Verlassen', 'skills.meetingBots.leavingButton': 'Wird verlassen…', 'skills.meetingBots.respondToParticipant': 'Ihr Name in diesem Meeting', diff --git a/app/src/lib/i18n/en.ts b/app/src/lib/i18n/en.ts index ffab6472f..40a4c5cd3 100644 --- a/app/src/lib/i18n/en.ts +++ b/app/src/lib/i18n/en.ts @@ -5849,6 +5849,8 @@ const en: TranslationMap = { 'skills.meetingBots.liveStatusListening': 'Listening (muted)', 'skills.meetingBots.liveStatusEnded': 'Meeting ended', 'skills.meetingBots.liveStatusError': 'Failed to join', + 'skills.meetingBots.liveTranscriptHeading': 'Live transcript', + 'skills.meetingBots.liveTranscriptEmpty': 'Waiting for speech…', 'skills.meetingBots.leaveButton': 'Leave', 'skills.meetingBots.leavingButton': 'Leaving…', 'skills.meetingBots.respondToParticipant': 'Your Name in This Meeting', diff --git a/app/src/lib/i18n/es.ts b/app/src/lib/i18n/es.ts index d4b3798a5..e5c44cbf6 100644 --- a/app/src/lib/i18n/es.ts +++ b/app/src/lib/i18n/es.ts @@ -5296,6 +5296,8 @@ const messages: TranslationMap = { 'skills.meetingBots.liveStatusListening': 'Escuchando (silenciado)', 'skills.meetingBots.liveStatusEnded': 'Reunión finalizada', 'skills.meetingBots.liveStatusError': 'Error al unirse', + 'skills.meetingBots.liveTranscriptHeading': 'Transcripción en vivo', + 'skills.meetingBots.liveTranscriptEmpty': 'Esperando voz…', 'skills.meetingBots.leaveButton': 'Salir', 'skills.meetingBots.leavingButton': 'Saliendo…', 'skills.meetingBots.respondToParticipant': 'Tu nombre en esta reunión', diff --git a/app/src/lib/i18n/fr.ts b/app/src/lib/i18n/fr.ts index 62491c2cf..1a55f7149 100644 --- a/app/src/lib/i18n/fr.ts +++ b/app/src/lib/i18n/fr.ts @@ -5317,6 +5317,8 @@ const messages: TranslationMap = { 'skills.meetingBots.liveStatusListening': 'Écoute (muet)', 'skills.meetingBots.liveStatusEnded': 'Réunion terminée', 'skills.meetingBots.liveStatusError': 'Échec de connexion', + 'skills.meetingBots.liveTranscriptHeading': 'Transcription en direct', + 'skills.meetingBots.liveTranscriptEmpty': 'En attente de parole…', 'skills.meetingBots.leaveButton': 'Quitter', 'skills.meetingBots.leavingButton': 'Sortie en cours…', 'skills.meetingBots.respondToParticipant': 'Votre nom dans cette réunion', diff --git a/app/src/lib/i18n/hi.ts b/app/src/lib/i18n/hi.ts index d29ade3b3..417731342 100644 --- a/app/src/lib/i18n/hi.ts +++ b/app/src/lib/i18n/hi.ts @@ -5204,6 +5204,8 @@ const messages: TranslationMap = { 'skills.meetingBots.liveStatusListening': 'सुन रहा है (म्यूट)', 'skills.meetingBots.liveStatusEnded': 'मीटिंग समाप्त', 'skills.meetingBots.liveStatusError': 'शामिल होने में विफल', + 'skills.meetingBots.liveTranscriptHeading': 'लाइव ट्रांसक्रिप्ट', + 'skills.meetingBots.liveTranscriptEmpty': 'भाषण की प्रतीक्षा…', 'skills.meetingBots.leaveButton': 'छोड़ें', 'skills.meetingBots.leavingButton': 'छोड़ रहे हैं…', 'skills.meetingBots.respondToParticipant': 'इस मीटिंग में आपका नाम', diff --git a/app/src/lib/i18n/id.ts b/app/src/lib/i18n/id.ts index 1fbd91340..d1c4b2dc6 100644 --- a/app/src/lib/i18n/id.ts +++ b/app/src/lib/i18n/id.ts @@ -5218,6 +5218,8 @@ const messages: TranslationMap = { 'skills.meetingBots.liveStatusListening': 'Mendengarkan (dibisukan)', 'skills.meetingBots.liveStatusEnded': 'Rapat selesai', 'skills.meetingBots.liveStatusError': 'Gagal bergabung', + 'skills.meetingBots.liveTranscriptHeading': 'Transkrip langsung', + 'skills.meetingBots.liveTranscriptEmpty': 'Menunggu ucapan…', 'skills.meetingBots.leaveButton': 'Keluar', 'skills.meetingBots.leavingButton': 'Keluar\u2026', 'skills.meetingBots.respondToParticipant': 'Nama Anda di Rapat Ini', diff --git a/app/src/lib/i18n/it.ts b/app/src/lib/i18n/it.ts index aefb611aa..440ec4d92 100644 --- a/app/src/lib/i18n/it.ts +++ b/app/src/lib/i18n/it.ts @@ -5285,6 +5285,8 @@ const messages: TranslationMap = { 'skills.meetingBots.liveStatusListening': 'In ascolto (muto)', 'skills.meetingBots.liveStatusEnded': 'Riunione terminata', 'skills.meetingBots.liveStatusError': 'Partecipazione fallita', + 'skills.meetingBots.liveTranscriptHeading': 'Trascrizione dal vivo', + 'skills.meetingBots.liveTranscriptEmpty': 'In attesa di parlato…', 'skills.meetingBots.leaveButton': 'Esci', 'skills.meetingBots.leavingButton': 'Uscita in corso…', 'skills.meetingBots.respondToParticipant': 'Il tuo nome in questa riunione', diff --git a/app/src/lib/i18n/ko.ts b/app/src/lib/i18n/ko.ts index bd73dc88f..08f9891f8 100644 --- a/app/src/lib/i18n/ko.ts +++ b/app/src/lib/i18n/ko.ts @@ -5150,6 +5150,8 @@ const messages: TranslationMap = { 'skills.meetingBots.liveStatusListening': '듣는 중 (음소거)', 'skills.meetingBots.liveStatusEnded': '회의 종료', 'skills.meetingBots.liveStatusError': '참가 실패', + 'skills.meetingBots.liveTranscriptHeading': '실시간 자막', + 'skills.meetingBots.liveTranscriptEmpty': '음성 대기 중…', 'skills.meetingBots.leaveButton': '나가기', 'skills.meetingBots.leavingButton': '나가는 중…', 'skills.meetingBots.respondToParticipant': '이 회의에서 내 이름', diff --git a/app/src/lib/i18n/pl.ts b/app/src/lib/i18n/pl.ts index 7ec292f4d..abe6af349 100644 --- a/app/src/lib/i18n/pl.ts +++ b/app/src/lib/i18n/pl.ts @@ -5272,6 +5272,8 @@ const messages: TranslationMap = { 'skills.meetingBots.liveStatusListening': 'Słuchanie (wyciszony)', 'skills.meetingBots.liveStatusEnded': 'Spotkanie zakończone', 'skills.meetingBots.liveStatusError': 'Nie można dołączyć', + 'skills.meetingBots.liveTranscriptHeading': 'Transkrypcja na żywo', + 'skills.meetingBots.liveTranscriptEmpty': 'Oczekiwanie na mowę…', 'skills.meetingBots.leaveButton': 'Wyjdź', 'skills.meetingBots.leavingButton': 'Opuszczanie…', 'skills.meetingBots.respondToParticipant': 'Twoje imię na tym spotkaniu', diff --git a/app/src/lib/i18n/pt.ts b/app/src/lib/i18n/pt.ts index 712f77900..ad8ffb297 100644 --- a/app/src/lib/i18n/pt.ts +++ b/app/src/lib/i18n/pt.ts @@ -5288,6 +5288,8 @@ const messages: TranslationMap = { 'skills.meetingBots.liveStatusListening': 'Ouvindo (mudo)', 'skills.meetingBots.liveStatusEnded': 'Reunião encerrada', 'skills.meetingBots.liveStatusError': 'Falha ao entrar', + 'skills.meetingBots.liveTranscriptHeading': 'Transcrição ao vivo', + 'skills.meetingBots.liveTranscriptEmpty': 'Aguardando fala…', 'skills.meetingBots.leaveButton': 'Sair', 'skills.meetingBots.leavingButton': 'Saindo…', 'skills.meetingBots.respondToParticipant': 'Seu nome nesta reunião', diff --git a/app/src/lib/i18n/ru.ts b/app/src/lib/i18n/ru.ts index e364a78db..5e56de2bd 100644 --- a/app/src/lib/i18n/ru.ts +++ b/app/src/lib/i18n/ru.ts @@ -5245,6 +5245,8 @@ const messages: TranslationMap = { 'skills.meetingBots.liveStatusListening': 'Прослушивание (без звука)', 'skills.meetingBots.liveStatusEnded': 'Встреча завершена', 'skills.meetingBots.liveStatusError': 'Ошибка подключения', + 'skills.meetingBots.liveTranscriptHeading': 'Транскрипция в реальном времени', + 'skills.meetingBots.liveTranscriptEmpty': 'Ожидание речи…', 'skills.meetingBots.leaveButton': 'Выйти', 'skills.meetingBots.leavingButton': 'Выход…', 'skills.meetingBots.respondToParticipant': 'Ваше имя на этой встрече', diff --git a/app/src/lib/i18n/zh-CN.ts b/app/src/lib/i18n/zh-CN.ts index 6507ae8e5..d4987729f 100644 --- a/app/src/lib/i18n/zh-CN.ts +++ b/app/src/lib/i18n/zh-CN.ts @@ -4940,6 +4940,8 @@ const messages: TranslationMap = { 'skills.meetingBots.liveStatusListening': '正在收听(已静音)', 'skills.meetingBots.liveStatusEnded': '会议已结束', 'skills.meetingBots.liveStatusError': '加入失败', + 'skills.meetingBots.liveTranscriptHeading': '实时转录', + 'skills.meetingBots.liveTranscriptEmpty': '等待语音…', 'skills.meetingBots.leaveButton': '离开', 'skills.meetingBots.leavingButton': '正在离开…', 'skills.meetingBots.respondToParticipant': '您在此会议中的姓名', diff --git a/app/src/services/__tests__/socketService.events.test.ts b/app/src/services/__tests__/socketService.events.test.ts index cea4b144d..c61553be3 100644 --- a/app/src/services/__tests__/socketService.events.test.ts +++ b/app/src/services/__tests__/socketService.events.test.ts @@ -324,6 +324,49 @@ describe('socketService — agent_meetings event handlers (lines 428-480)', () = ); }); + it('dispatches appendBackendMeetTranscriptDelta on agent_meetings:transcript_delta', async () => { + const { handlers, mockSocket } = buildMockSocket(); + vi.doMock('socket.io-client', () => ({ io: vi.fn(() => mockSocket) })); + getCoreRpcUrlMock.mockResolvedValue('http://127.0.0.1:7788/rpc'); + + const { socketService } = await import('../socketService'); + socketService.connect('jwt-test-meet-delta'); + + await pollUntil(() => expect(handlers['agent_meetings:transcript_delta']).toBeDefined()); + handlers['agent_meetings:transcript_delta']!({ + turn: { role: 'user', content: 'hello' }, + index: 2, + is_partial: true, + correlation_id: 'corr-1', + }); + + expect(storeMock.dispatch).toHaveBeenCalledWith( + expect.objectContaining({ + payload: { + turn: { role: 'user', content: 'hello' }, + index: 2, + is_partial: true, + correlationId: 'corr-1', + }, + }) + ); + }); + + it('drops a transcript_delta with a missing/invalid turn', async () => { + const { handlers, mockSocket } = buildMockSocket(); + vi.doMock('socket.io-client', () => ({ io: vi.fn(() => mockSocket) })); + getCoreRpcUrlMock.mockResolvedValue('http://127.0.0.1:7788/rpc'); + + const { socketService } = await import('../socketService'); + socketService.connect('jwt-test-meet-delta-bad'); + + await pollUntil(() => expect(handlers['agent_meetings:transcript_delta']).toBeDefined()); + storeMock.dispatch.mockClear(); + handlers['agent_meetings:transcript_delta']!({ index: 0, is_partial: false }); + + expect(storeMock.dispatch).not.toHaveBeenCalled(); + }); + it('dispatches setBackendMeetError on agent_meetings:error', async () => { const { handlers, mockSocket } = buildMockSocket(); vi.doMock('socket.io-client', () => ({ io: vi.fn(() => mockSocket) })); diff --git a/app/src/services/socketService.ts b/app/src/services/socketService.ts index 450d251b7..4097563ba 100644 --- a/app/src/services/socketService.ts +++ b/app/src/services/socketService.ts @@ -6,6 +6,7 @@ import { SocketIOMCPTransportImpl } from '../lib/mcp'; import { ingestRuntimeErrorSignal } from '../lib/userErrors/report'; import { store } from '../store'; import { + appendBackendMeetTranscriptDelta, setBackendMeetError, setBackendMeetHarness, setBackendMeetJoined, @@ -524,6 +525,33 @@ class SocketService { }) ); }); + this.socket.on('agent_meetings:transcript_delta', (data: unknown) => { + const obj = data as Record | null; + if (!obj) return; + const turn = obj.turn as Record | null | undefined; + // Drop malformed deltas that carry no turn content. + if (!turn || typeof turn.role !== 'string' || typeof turn.content !== 'string') { + socketError('agent_meetings:transcript_delta dropped: missing/invalid turn'); + return; + } + const correlationId = typeof obj.correlation_id === 'string' ? obj.correlation_id : undefined; + const index = typeof obj.index === 'number' ? obj.index : 0; + const isPartial = typeof obj.is_partial === 'boolean' ? obj.is_partial : false; + socketLog( + 'agent_meetings:transcript_delta index=%d is_partial=%s correlation_id=%s', + index, + isPartial, + correlationId ?? 'none' + ); + store.dispatch( + appendBackendMeetTranscriptDelta({ + turn: { role: turn.role, content: turn.content }, + index, + is_partial: isPartial, + correlationId, + }) + ); + }); this.socket.on('agent_meetings:error', (data: unknown) => { const obj = data as Record | null; const error = typeof obj?.error === 'string' ? obj.error : 'Unknown error'; diff --git a/app/src/store/backendMeetSlice.test.ts b/app/src/store/backendMeetSlice.test.ts index f9aecd8d9..5154a60a2 100644 --- a/app/src/store/backendMeetSlice.test.ts +++ b/app/src/store/backendMeetSlice.test.ts @@ -1,7 +1,9 @@ import { describe, expect, it } from 'vitest'; import backendMeetReducer, { + appendBackendMeetTranscriptDelta, resetBackendMeet, + selectBackendMeetLiveTranscript, setBackendMeetError, setBackendMeetHarness, setBackendMeetJoined, @@ -96,4 +98,163 @@ describe('backendMeetSlice', () => { const state = backendMeetReducer(active, resetBackendMeet()); expect(state).toEqual(initial); }); + + describe('live transcript (transcript_delta, #4304)', () => { + it('appends sequential delta turns to the live buffer', () => { + let state = backendMeetReducer( + initial, + appendBackendMeetTranscriptDelta({ + turn: { role: 'user', content: 'Hello' }, + index: 0, + is_partial: false, + }) + ); + state = backendMeetReducer( + state, + appendBackendMeetTranscriptDelta({ + turn: { role: 'assistant', content: 'Hi there' }, + index: 1, + is_partial: false, + }) + ); + expect(state.liveTranscript).toEqual([ + { role: 'user', content: 'Hello' }, + { role: 'assistant', content: 'Hi there' }, + ]); + expect(state.livePartialIndex).toBeNull(); + }); + + it('marks a partial line and supersedes it when finalized at the same index', () => { + let state = backendMeetReducer( + initial, + appendBackendMeetTranscriptDelta({ + turn: { role: 'user', content: 'Hel' }, + index: 0, + is_partial: true, + }) + ); + expect(state.livePartialIndex).toBe(0); + expect(state.liveTranscript[0]?.content).toBe('Hel'); + + // Final delta at the same index replaces the partial and clears the flag. + state = backendMeetReducer( + state, + appendBackendMeetTranscriptDelta({ + turn: { role: 'user', content: 'Hello there' }, + index: 0, + is_partial: false, + }) + ); + expect(state.liveTranscript).toHaveLength(1); + expect(state.liveTranscript[0]?.content).toBe('Hello there'); + expect(state.livePartialIndex).toBeNull(); + }); + + it('keys by backend index: gaps (skipped [System] turns) do not break supersede', () => { + // index 0 finalized, then a partial preview lands at index 2 (index 1 is a + // skipped [System] turn never sent as a delta), then index 2 is finalized. + let state = backendMeetReducer( + initial, + appendBackendMeetTranscriptDelta({ + turn: { role: 'user', content: '[Alice] hi' }, + index: 0, + is_partial: false, + }) + ); + state = backendMeetReducer( + state, + appendBackendMeetTranscriptDelta({ + turn: { role: 'user', content: '[Bob] in pro' }, + index: 2, + is_partial: true, + }) + ); + expect(state.livePartialIndex).toBe(2); + // Finalize at the SAME backend index → replaces the partial in place, no dup. + state = backendMeetReducer( + state, + appendBackendMeetTranscriptDelta({ + turn: { role: 'user', content: '[Bob] in progress' }, + index: 2, + is_partial: false, + }) + ); + expect(state.livePartialIndex).toBeNull(); + expect(state.liveTranscript[0]?.content).toBe('[Alice] hi'); + expect(state.liveTranscript[2]?.content).toBe('[Bob] in progress'); + // The gap at index 1 stays empty; no duplicate Bob turn was appended. + expect(state.liveTranscript[1]).toBeUndefined(); + const populated = state.liveTranscript.filter(Boolean); + expect(populated).toHaveLength(2); + }); + + it('selector returns an empty array when the buffer is absent (legacy state)', () => { + // A store shaped before this slice field existed has no liveTranscript; + // the selector must not hand back undefined (would crash the panel). + const legacy = { backendMeet: { status: 'active' } } as never; + expect(selectBackendMeetLiveTranscript(legacy)).toEqual([]); + }); + + it('ignores a delta with a negative index', () => { + const state = backendMeetReducer( + initial, + appendBackendMeetTranscriptDelta({ + turn: { role: 'user', content: 'bad' }, + index: -1, + is_partial: false, + }) + ); + expect(state.liveTranscript.filter(Boolean)).toHaveLength(0); + expect(state.livePartialIndex).toBeNull(); + }); + + it('clears the live buffer on join', () => { + const withLive = backendMeetReducer( + initial, + appendBackendMeetTranscriptDelta({ + turn: { role: 'user', content: 'stale' }, + index: 0, + is_partial: false, + }) + ); + const joining = backendMeetReducer( + withLive, + setBackendMeetJoining({ meetUrl: 'https://meet.google.com/abc-defg-hij' }) + ); + expect(joining.liveTranscript).toEqual([]); + expect(joining.livePartialIndex).toBeNull(); + }); + + it('clears the live buffer on leave', () => { + const withLive = backendMeetReducer( + initial, + appendBackendMeetTranscriptDelta({ + turn: { role: 'user', content: 'mid-call' }, + index: 0, + is_partial: true, + }) + ); + const left = backendMeetReducer(withLive, setBackendMeetLeft({ reason: 'call-ended' })); + expect(left.liveTranscript).toEqual([]); + expect(left.livePartialIndex).toBeNull(); + }); + + it('reconciles: final transcript empties the live buffer', () => { + const withLive = backendMeetReducer( + initial, + appendBackendMeetTranscriptDelta({ + turn: { role: 'user', content: 'Hello' }, + index: 0, + is_partial: false, + }) + ); + const reconciled = backendMeetReducer( + withLive, + setBackendMeetTranscript({ turns: [{ role: 'user', content: 'Hello' }], duration_ms: 1000 }) + ); + expect(reconciled.transcript?.turns).toHaveLength(1); + expect(reconciled.liveTranscript).toEqual([]); + expect(reconciled.livePartialIndex).toBeNull(); + }); + }); }); diff --git a/app/src/store/backendMeetSlice.ts b/app/src/store/backendMeetSlice.ts index c9395112b..a2806eefe 100644 --- a/app/src/store/backendMeetSlice.ts +++ b/app/src/store/backendMeetSlice.ts @@ -29,6 +29,19 @@ export interface BackendMeetTranscriptEvent { correlationId?: string; } +/** + * Incremental transcript turn emitted mid-call (issue #4304). `index` is the + * turn's stable slot in the ordered transcript; a later delta at the same + * `index` supersedes an earlier one (used to finalize a partial line). + * `is_partial` marks a not-yet-finalized line, rendered greyed in the UI. + */ +export interface BackendMeetTranscriptDeltaEvent { + turn: BackendMeetTurn; + index: number; + is_partial: boolean; + correlationId?: string; +} + export interface BackendMeetState { status: BackendMeetStatus; meetUrl: string | null; @@ -37,6 +50,21 @@ export interface BackendMeetState { lastReply: BackendMeetReplyEvent | null; lastHarness: BackendMeetHarnessEvent | null; transcript: BackendMeetTranscriptEvent | null; + /** + * Live transcript turns accumulated from `transcript_delta` events during an + * active call, keyed by the backend's transcript `index` (the array position + * IS that index). The array can be sparse — skipped `[System]` turns occupy + * an index but are never sent as deltas — so consumers must skip empty slots. + * Cleared on join and on leave, and reconciled away (emptied) when the + * authoritative final `transcript` arrives at call end so lines aren't shown + * twice. + */ + liveTranscript: BackendMeetTurn[]; + /** + * Backend transcript index of the turn currently marked partial (greyed), or + * `null` when the latest delta finalized its line. + */ + livePartialIndex: number | null; error: string | null; } @@ -48,6 +76,8 @@ const initialState: BackendMeetState = { lastReply: null, lastHarness: null, transcript: null, + liveTranscript: [], + livePartialIndex: null, error: null, }; @@ -67,6 +97,9 @@ const backendMeetSlice = createSlice({ state.lastReply = null; state.lastHarness = null; state.transcript = null; + // Start each call with a clean live buffer (per-meetingId lifecycle). + state.liveTranscript = []; + state.livePartialIndex = null; }, setBackendMeetJoined(state, action: PayloadAction<{ meetUrl: string; meetingId?: string }>) { state.status = 'active'; @@ -79,6 +112,10 @@ const backendMeetSlice = createSlice({ }, setBackendMeetLeft(state, _action: PayloadAction<{ reason: string; correlationId?: string }>) { state.status = 'ended'; + // Tear down the live buffer on leave; the authoritative transcript (if + // any) arrives separately via setBackendMeetTranscript. + state.liveTranscript = []; + state.livePartialIndex = null; }, setBackendMeetReply(state, action: PayloadAction) { state.lastReply = action.payload; @@ -88,6 +125,32 @@ const backendMeetSlice = createSlice({ }, setBackendMeetTranscript(state, action: PayloadAction) { state.transcript = action.payload; + // The final transcript is authoritative — drop the accumulated live + // buffer so the same turns aren't rendered twice (reconcile on end). + state.liveTranscript = []; + state.livePartialIndex = null; + }, + appendBackendMeetTranscriptDelta( + state, + action: PayloadAction + ) { + const { turn, index, is_partial } = action.payload; + // Guard against a malformed negative index. + if (index < 0) return; + // Key strictly by the backend's transcript index. The backend reconciles + // deltas by index: a partial preview and its finalized turn share the + // same index, so writing at `index` makes the final supersede the partial + // in place. Indices are NOT guaranteed contiguous or zero-based — skipped + // `[System]` turns occupy an index but are never sent as deltas — so we + // leave a gap (a sparse slot) rather than shifting later turns. Rendering + // skips the empty slots. + state.liveTranscript[index] = turn; + if (is_partial) { + state.livePartialIndex = index; + } else if (state.livePartialIndex === index) { + // This delta finalizes the line that was previously partial. + state.livePartialIndex = null; + } }, setBackendMeetError(state, action: PayloadAction<{ error: string; correlationId?: string }>) { state.status = 'error'; @@ -109,6 +172,7 @@ export const { setBackendMeetReply, setBackendMeetHarness, setBackendMeetTranscript, + appendBackendMeetTranscriptDelta, setBackendMeetError, resetBackendMeet, } = backendMeetSlice.actions; @@ -129,5 +193,11 @@ export const selectBackendMeetListenOnly = (state: { backendMeet: BackendMeetSta state.backendMeet.listenOnly; export const selectBackendMeetError = (state: { backendMeet: BackendMeetState }): string | null => state.backendMeet.error; +export const selectBackendMeetLiveTranscript = (state: { + backendMeet: BackendMeetState; +}): BackendMeetTurn[] => state.backendMeet.liveTranscript ?? []; +export const selectBackendMeetLivePartialIndex = (state: { + backendMeet: BackendMeetState; +}): number | null => state.backendMeet.livePartialIndex; export default backendMeetSlice.reducer; diff --git a/src/core/event_bus/events.rs b/src/core/event_bus/events.rs index 18c9e7ddb..f6d19a5de 100644 --- a/src/core/event_bus/events.rs +++ b/src/core/event_bus/events.rs @@ -1103,6 +1103,18 @@ pub enum DomainEvent { duration_ms: u64, correlation_id: Option, }, + /// Backend gmeet bot emitted an incremental transcript turn mid-call + /// (`bot:transcript_delta`, issue #4304). Relayed live to the renderer so + /// the active-call UI can render turns as they're spoken. `is_partial` + /// marks a not-yet-finalized line at `index`; a later delta (partial or + /// final) at the same `index` supersedes it. The terminal + /// `BackendMeetTranscript` stays authoritative for thread/summary. + BackendMeetTranscriptDelta { + turn: BackendMeetTurn, + index: u64, + is_partial: bool, + correlation_id: Option, + }, /// Backend gmeet bot emitted an error. BackendMeetError { error: String, @@ -1331,6 +1343,7 @@ impl DomainEvent { | Self::BackendMeetReply { .. } | Self::BackendMeetHarness { .. } | Self::BackendMeetTranscript { .. } + | Self::BackendMeetTranscriptDelta { .. } | Self::BackendMeetError { .. } | Self::BackendMeetInCallRequest { .. } | Self::BackendMeetSpeak { .. } @@ -1466,6 +1479,7 @@ impl DomainEvent { Self::BackendMeetReply { .. } => "BackendMeetReply", Self::BackendMeetHarness { .. } => "BackendMeetHarness", Self::BackendMeetTranscript { .. } => "BackendMeetTranscript", + Self::BackendMeetTranscriptDelta { .. } => "BackendMeetTranscriptDelta", Self::BackendMeetError { .. } => "BackendMeetError", Self::BackendMeetInCallRequest { .. } => "BackendMeetInCallRequest", Self::BackendMeetSpeak { .. } => "BackendMeetSpeak", diff --git a/src/core/socketio.rs b/src/core/socketio.rs index 8bce55a2a..75a3c7197 100644 --- a/src/core/socketio.rs +++ b/src/core/socketio.rs @@ -1129,6 +1129,25 @@ pub fn spawn_web_channel_bridge(io: SocketIo) { ); let _ = io_agent_meetings.emit("agent_meetings:transcript", &payload); } + crate::core::event_bus::DomainEvent::BackendMeetTranscriptDelta { + turn, + index, + is_partial, + correlation_id, + } => { + let payload = serde_json::json!({ + "turn": turn, + "index": index, + "is_partial": is_partial, + "correlation_id": correlation_id, + }); + log::debug!( + "[socketio] broadcast agent_meetings:transcript_delta index={} is_partial={}", + index, + is_partial + ); + let _ = io_agent_meetings.emit("agent_meetings:transcript_delta", &payload); + } crate::core::event_bus::DomainEvent::BackendMeetError { error, correlation_id, diff --git a/src/openhuman/socket/event_handlers.rs b/src/openhuman/socket/event_handlers.rs index 71fb80707..426aa2c5d 100644 --- a/src/openhuman/socket/event_handlers.rs +++ b/src/openhuman/socket/event_handlers.rs @@ -341,6 +341,32 @@ pub(super) fn handle_sio_event( correlation_id, }); } + "bot:transcript_delta" => { + // Incremental mid-call transcript turn (issue #4304). Relayed live + // to the renderer; the terminal `bot:transcript` stays authoritative + // for thread creation / summary (handled by MeetingEventSubscriber). + match parse_transcript_delta(&data) { + Some((turn, index, is_partial, correlation_id)) => { + log::info!( + "[socket] bot:transcript_delta index={} is_partial={} role={}", + index, + is_partial, + turn.role + ); + publish_global(DomainEvent::BackendMeetTranscriptDelta { + turn, + index, + is_partial, + correlation_id, + }); + } + None => { + log::warn!( + "[socket] bot:transcript_delta dropped: missing/invalid 'turn' field" + ); + } + } + } "bot:in_call_request" => { let correlation_id = data .get("correlationId") @@ -469,6 +495,30 @@ fn base64_encode(input: &str) -> String { base64::engine::general_purpose::STANDARD.encode(input.as_bytes()) } +/// Parse a `bot:transcript_delta` payload (issue #4304) into its event fields. +/// +/// Expected shape: `{ turn: { role, content }, index, isPartial, correlationId }`. +/// Returns `None` when the required `turn` object is missing or malformed so the +/// caller can drop the event rather than publish a degenerate turn. `index` +/// defaults to 0 and `isPartial` to `false` (final) when absent. +fn parse_transcript_delta( + data: &serde_json::Value, +) -> Option<(BackendMeetTurn, u64, bool, Option)> { + let turn: BackendMeetTurn = data + .get("turn") + .and_then(|v| serde_json::from_value(v.clone()).ok())?; + let index = data.get("index").and_then(|v| v.as_u64()).unwrap_or(0); + let is_partial = data + .get("isPartial") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + let correlation_id = data + .get("correlationId") + .and_then(|v| v.as_str()) + .map(String::from); + Some((turn, index, is_partial, correlation_id)) +} + /// Send a Socket.IO event through the emit channel. /// /// Format: `42["eventName", data]` @@ -579,6 +629,41 @@ mod tests { assert!(parse_sio_event(r#"[invalid json"#).is_none()); } + // ── parse_transcript_delta (bot:transcript_delta, #4304) ──────── + + #[test] + fn parse_transcript_delta_extracts_all_fields() { + let data = json!({ + "turn": { "role": "user", "content": "hello there" }, + "index": 3, + "isPartial": true, + "correlationId": "corr-123" + }); + let (turn, index, is_partial, correlation_id) = parse_transcript_delta(&data).unwrap(); + assert_eq!(turn.role, "user"); + assert_eq!(turn.content, "hello there"); + assert_eq!(index, 3); + assert!(is_partial); + assert_eq!(correlation_id.as_deref(), Some("corr-123")); + } + + #[test] + fn parse_transcript_delta_defaults_index_partial_and_correlation() { + let data = json!({ "turn": { "role": "assistant", "content": "hi" } }); + let (turn, index, is_partial, correlation_id) = parse_transcript_delta(&data).unwrap(); + assert_eq!(turn.role, "assistant"); + assert_eq!(index, 0); + assert!(!is_partial); + assert!(correlation_id.is_none()); + } + + #[test] + fn parse_transcript_delta_returns_none_without_turn() { + assert!(parse_transcript_delta(&json!({ "index": 1, "isPartial": false })).is_none()); + // Malformed turn (missing required fields) is also dropped. + assert!(parse_transcript_delta(&json!({ "turn": { "role": "user" } })).is_none()); + } + // ── handle_sio_event dispatch ─────────────────────────────────── #[test]