feat(app): memory citations UI, respond queue, silent ingest refresh (#800)

Co-authored-by: Jwalin Shah <jshah1331@gmail.com>
2026-07-28 05:12:33 +00:00 · 2026-04-24 09:45:54 -07:00
parent 116fa86838
commit 5b3dec4ded
15 changed files with 416 additions and 7 deletions
@@ -0,0 +1,98 @@
+import type { RespondQueueItem } from '../../types/providerSurfaces';
+import { openUrl } from '../../utils/openUrl';
+
+interface RespondQueuePanelProps {
+  items: RespondQueueItem[];
+  count: number;
+  status: 'idle' | 'loading' | 'succeeded' | 'failed';
+  error: string | null;
+  onRefresh: () => void;
+}
+
+function relativeTime(iso: string): string {
+  const ts = new Date(iso).getTime();
+  if (!Number.isFinite(ts)) return 'unknown time';
+  const deltaMs = Date.now() - ts;
+  if (deltaMs < 60_000) return 'just now';
+  const mins = Math.floor(deltaMs / 60_000);
+  if (mins < 60) return `${mins}m ago`;
+  const hours = Math.floor(mins / 60);
+  if (hours < 24) return `${hours}h ago`;
+  const days = Math.floor(hours / 24);
+  return `${days}d ago`;
+}
+
+function queueTitle(item: RespondQueueItem): string {
+  return item.title || item.senderName || item.eventKind || item.provider;
+}
+
+export default function RespondQueuePanel({
+  items,
+  count,
+  status,
+  error,
+  onRefresh,
+}: RespondQueuePanelProps) {
+  return (
+    <aside className="flex w-80 flex-none flex-col border-l border-stone-200 bg-white">
+      <div className="flex flex-none items-center justify-between border-b border-stone-100 px-4 py-3">
+        <div>
+          <h3 className="text-sm font-semibold text-stone-800">Respond queue</h3>
+          <p className="text-xs text-stone-500">{count} pending</p>
+        </div>
+        <button
+          type="button"
+          onClick={onRefresh}
+          className="rounded-lg border border-stone-200 px-2 py-1 text-xs text-stone-600 hover:bg-stone-50">
+          Refresh
+        </button>
+      </div>
+      <div className="flex-1 overflow-y-auto px-3 py-3">
+        {status === 'loading' && items.length === 0 ? (
+          <p className="rounded-lg bg-stone-50 px-3 py-2 text-xs text-stone-500">Loading queue…</p>
+        ) : null}
+
+        {status === 'failed' ? (
+          <p className="rounded-lg bg-coral-50 px-3 py-2 text-xs text-coral-600">
+            {error ?? 'Failed to load respond queue'}
+          </p>
+        ) : null}
+
+        {items.length === 0 && status !== 'loading' ? (
+          <p className="rounded-lg bg-stone-50 px-3 py-2 text-xs text-stone-500">
+            No pending provider events yet.
+          </p>
+        ) : null}
+
+        <div className="space-y-2">
+          {items.slice(0, 30).map(item => (
+            <button
+              key={item.id}
+              type="button"
+              onClick={() => {
+                if (item.deepLink) {
+                  void openUrl(item.deepLink);
+                }
+              }}
+              className="w-full rounded-xl border border-stone-200 bg-white px-3 py-2 text-left transition-colors hover:bg-stone-50 disabled:cursor-default"
+              disabled={!item.deepLink}>
+              <div className="flex items-center justify-between gap-2">
+                <p className="truncate text-xs font-medium text-stone-800">{queueTitle(item)}</p>
+                <span className="rounded-full bg-stone-100 px-2 py-0.5 text-[10px] uppercase text-stone-600">
+                  {item.provider}
+                </span>
+              </div>
+              {item.snippet ? (
+                <p className="mt-1 line-clamp-2 text-xs text-stone-600">{item.snippet}</p>
+              ) : null}
+              <div className="mt-1 flex items-center justify-between text-[10px] text-stone-500">
+                <span>{item.senderName ?? item.senderHandle ?? item.accountId}</span>
+                <span>{relativeTime(item.timestamp)}</span>
+              </div>
+            </button>
+          ))}
+        </div>
+      </div>
+    </aside>
+  );
+}
@@ -2,6 +2,7 @@ import { useEffect, useMemo, useState } from 'react';

 import AddAccountModal from '../components/accounts/AddAccountModal';
 import { AgentIcon, ProviderIcon } from '../components/accounts/providerIcons';
+import RespondQueuePanel from '../components/accounts/RespondQueuePanel';
 import WebviewHost from '../components/accounts/WebviewHost';
 import {
  hideWebviewAccount,
@@ -11,6 +12,7 @@ import {
 } from '../services/webviewAccountService';
 import { addAccount, removeAccount, setActiveAccount } from '../store/accountsSlice';
 import { useAppDispatch, useAppSelector } from '../store/hooks';
+import { fetchRespondQueue } from '../store/providerSurfaceSlice';
 import type { Account, AccountProvider, ProviderDescriptor } from '../types/accounts';
 import { AGENT_ACCOUNT_ID as AGENT_ID } from '../utils/accountsFullscreen';
 import { AgentChatPanel } from './Conversations';
@@ -75,6 +77,10 @@ const Accounts = () => {
  const order = useAppSelector(state => state.accounts.order);
  const activeAccountId = useAppSelector(state => state.accounts.activeAccountId);
  const unreadByAccount = useAppSelector(state => state.accounts.unread);
+  const respondQueue = useAppSelector(state => state.providerSurfaces.queue);
+  const respondQueueCount = useAppSelector(state => state.providerSurfaces.count);
+  const respondQueueStatus = useAppSelector(state => state.providerSurfaces.status);
+  const respondQueueError = useAppSelector(state => state.providerSurfaces.error);

  const [addOpen, setAddOpen] = useState(false);
  const [ctxMenu, setCtxMenu] = useState<ContextMenuState | null>(null);
@@ -83,6 +89,14 @@ const Accounts = () => {
    startWebviewAccountService();
  }, []);

+  useEffect(() => {
+    void dispatch(fetchRespondQueue());
+    const id = window.setInterval(() => {
+      void dispatch(fetchRespondQueue({ silent: true }));
+    }, 10_000);
+    return () => window.clearInterval(id);
+  }, [dispatch]);
+
  const accounts: Account[] = useMemo(
    () => order.map(id => accountsById[id]).filter((a): a is Account => Boolean(a)),
    [order, accountsById]
@@ -203,7 +217,20 @@ const Accounts = () => {
      {/* Main pane */}
      <main className="flex min-w-0 flex-1 flex-col">
        {isAgentSelected ? (
-          <AgentChatPanel />
+          <div className="flex h-full min-w-0">
+            <div className="min-w-0 flex-1">
+              <AgentChatPanel />
+            </div>
+            <RespondQueuePanel
+              items={respondQueue}
+              count={respondQueueCount}
+              status={respondQueueStatus}
+              error={respondQueueError}
+              onRefresh={() => {
+                void dispatch(fetchRespondQueue());
+              }}
+            />
+          </div>
        ) : active ? (
          <div className="flex-1">
            <WebviewHost accountId={active.id} provider={active.provider} />
@@ -43,6 +43,7 @@ import {
 } from '../utils/tauriCommands';
 import { formatTimelineEntry } from '../utils/toolTimelineFormatting';
 import { AgentMessageBubble, BubbleMarkdown } from './conversations/components/AgentMessageBubble';
+import { CitationChips, type MessageCitation } from './conversations/components/CitationChips';
 import { LimitPill } from './conversations/components/LimitPill';
 import { ToolTimelineBlock } from './conversations/components/ToolTimelineBlock';
 import {
@@ -932,6 +933,21 @@ const Conversations = ({ variant = 'page' }: ConversationsProps = {}) => {
                              );
                            }
                          )}
+                          {(() => {
+                            const raw = msg.extraMetadata?.citations;
+                            if (!Array.isArray(raw)) return null;
+                            const citations = raw.filter(
+                              (item): item is MessageCitation =>
+                                typeof item === 'object' &&
+                                item !== null &&
+                                typeof (item as MessageCitation).id === 'string' &&
+                                typeof (item as MessageCitation).key === 'string' &&
+                                typeof (item as MessageCitation).snippet === 'string' &&
+                                typeof (item as MessageCitation).timestamp === 'string'
+                            );
+                            if (citations.length === 0) return null;
+                            return <CitationChips citations={citations} />;
+                          })()}
                          {latestVisibleMessage?.id === msg.id && (
                            <p className="px-1 text-[10px] text-stone-400">
                              {formatRelativeTime(msg.createdAt)}
@@ -0,0 +1,39 @@
+/**
+ * Compact memory citation chips for assistant messages (wired from
+ * `extraMetadata.citations` populated on `chat_done` / segment events).
+ */
+export type MessageCitation = {
+  id: string;
+  key: string;
+  namespace?: string;
+  score?: number;
+  timestamp: string;
+  snippet: string;
+};
+
+export function CitationChips({ citations }: { citations: MessageCitation[] }) {
+  if (citations.length === 0) return null;
+  return (
+    <div className="mt-1.5 flex flex-wrap gap-1">
+      {citations.map(citation => {
+        const scoreLabel =
+          typeof citation.score === 'number' ? ` ${Math.round(citation.score * 100)}%` : '';
+        const title = `${citation.key}${citation.namespace ? ` (${citation.namespace})` : ''}\n${citation.snippet}`;
+        return (
+          <details key={citation.id} className="group">
+            <summary
+              className="list-none cursor-pointer rounded-full border border-stone-300 bg-stone-100 px-2 py-0.5 text-[10px] text-stone-600 hover:bg-stone-200"
+              aria-label={title}
+              title={title}>
+              {citation.namespace ?? citation.key}
+              {scoreLabel}
+            </summary>
+            <div className="mt-1 max-w-md rounded-md border border-stone-200 bg-white px-2 py-1 text-[11px] text-stone-600 shadow-sm">
+              {citation.snippet}
+            </div>
+          </details>
+        );
+      })}
+    </div>
+  );
+}
@@ -37,12 +37,13 @@ import {
  setActiveThread,
  setSelectedThread,
 } from '../store/threadSlice';
+import { IS_PROD } from '../utils/config';
 import { formatTimelineEntry, promptFromArgsBuffer } from '../utils/toolTimelineFormatting';

 const logChatRuntime = debug('openhuman:chat-runtime');

 function rtLog(message: string, fields?: Record<string, string | number | null | undefined>) {
-  if (import.meta.env.PROD) return;
+  if (IS_PROD) return;
  if (fields && Object.keys(fields).length > 0) {
    const parts = Object.entries(fields)
      .filter(([, v]) => v !== undefined && v !== '' && v !== null)
@@ -386,7 +387,11 @@ const ChatRuntimeProvider = ({ children }: { children: React.ReactNode }) => {
        )
          return;
        void dispatch(
-          addInferenceResponse({ content: segmentText(event), threadId: event.thread_id })
+          addInferenceResponse({
+            content: segmentText(event),
+            threadId: event.thread_id,
+            extraMetadata: event.citations?.length ? { citations: event.citations } : undefined,
+          })
        );
      },
      onTextDelta: event => {
@@ -524,7 +529,13 @@ const ChatRuntimeProvider = ({ children }: { children: React.ReactNode }) => {
          void (async () => {
            try {
              await dispatch(
-                addInferenceResponse({ content: event.full_response, threadId: event.thread_id })
+                addInferenceResponse({
+                  content: event.full_response,
+                  threadId: event.thread_id,
+                  extraMetadata: event.citations?.length
+                    ? { citations: event.citations }
+                    : undefined,
+                })
              ).unwrap();
              void dispatch(
                generateThreadTitleIfNeeded({
@@ -0,0 +1,32 @@
+import type { RespondQueueList } from '../../types/providerSurfaces';
+import { callCoreRpc } from '../coreRpcClient';
+
+interface ProviderSurfacesQueueEnvelope {
+  data?: RespondQueueList;
+  result?: { data?: RespondQueueList };
+}
+
+const EMPTY_QUEUE: RespondQueueList = { items: [], count: 0 };
+
+function parseQueueEnvelope(raw: unknown): RespondQueueList {
+  if (!raw || typeof raw !== 'object') {
+    throw new Error('provider_surfaces_list_queue: unexpected empty response');
+  }
+
+  const envelope = raw as ProviderSurfacesQueueEnvelope & { error?: { message?: string } };
+  if (envelope.error) {
+    throw new Error(envelope.error.message ?? 'Core RPC returned an error');
+  }
+  const candidate = envelope.result?.data ?? envelope.data;
+  if (!candidate || !Array.isArray(candidate.items) || typeof candidate.count !== 'number') {
+    return EMPTY_QUEUE;
+  }
+  return candidate;
+}
+
+export const providerSurfacesApi = {
+  async listQueue(): Promise<RespondQueueList> {
+    const raw = await callCoreRpc<unknown>({ method: 'openhuman.provider_surfaces_list_queue' });
+    return parseQueueEnvelope(raw);
+  },
+};
@@ -52,6 +52,17 @@ export interface ChatDoneEvent {
  reaction_emoji?: string | null;
  /** Total segments when the response was split into bubbles by Rust. */
  segment_total?: number | null;
+  /** Memory citations captured during retrieval for this response. */
+  citations?: ChatCitation[] | null;
+}
+
+export interface ChatCitation {
+  id: string;
+  key: string;
+  namespace?: string;
+  score?: number;
+  timestamp: string;
+  snippet: string;
 }

 /** A single segment of a multi-bubble response, emitted before `chat_done`. */
@@ -67,6 +78,7 @@ export interface ChatSegmentEvent {
  segment_index: number;
  segment_total: number;
  reaction_emoji?: string | null;
+  citations?: ChatCitation[] | null;
 }

 /** Return the segment text from a {@link ChatSegmentEvent} (avoids the misleading wire name). */
@@ -10,6 +10,7 @@ import {
  setActiveAccount,
 } from '../store/accountsSlice';
 import { addNotification } from '../store/notificationsSlice';
+import { fetchRespondQueue } from '../store/providerSurfaceSlice';
 import type { AccountProvider, IngestedMessage } from '../types/accounts';
 import { threadApi } from './api/threadApi';
 import { chatSend } from './chatService';
@@ -188,6 +189,9 @@ function handleRecipeEvent(evt: RecipeEventPayload) {

    store.dispatch(appendMessages({ accountId, messages, unread: ingest.unread }));

+    // Tauri already forwarded this ingest to core; refresh queue immediately for Agent pane.
+    void store.dispatch(fetchRespondQueue({ silent: true }));
+
    // Fire-and-forget memory write via the existing core RPC.
    // Namespace mirrors the skill-sync convention so the recall pipeline
    // can find these alongside other ingested context.
@@ -0,0 +1,65 @@
+import { describe, expect, it, vi } from 'vitest';
+
+import { providerSurfacesApi } from '../../services/api/providerSurfacesApi';
+import reducer, { fetchRespondQueue } from '../providerSurfaceSlice';
+
+vi.mock('../../services/api/providerSurfacesApi', () => ({
+  providerSurfacesApi: { listQueue: vi.fn() },
+}));
+
+describe('providerSurfaceSlice', () => {
+  it('stores queue payload on success', async () => {
+    vi.mocked(providerSurfacesApi.listQueue).mockResolvedValue({
+      items: [
+        {
+          id: 'linkedin:acct-1:message:entity-1',
+          provider: 'linkedin',
+          accountId: 'acct-1',
+          eventKind: 'message',
+          entityId: 'entity-1',
+          timestamp: '2026-04-22T17:00:00Z',
+          requiresAttention: true,
+          status: 'pending',
+        },
+      ],
+      count: 1,
+    });
+
+    const pending = reducer(undefined, fetchRespondQueue.pending('', undefined));
+    expect(pending.status).toBe('loading');
+
+    const fulfilledAction = fetchRespondQueue.fulfilled(
+      await providerSurfacesApi.listQueue(),
+      '',
+      undefined
+    );
+    const state = reducer(pending, fulfilledAction);
+    expect(state.status).toBe('succeeded');
+    expect(state.count).toBe(1);
+    expect(state.queue[0]?.provider).toBe('linkedin');
+    expect(state.lastSyncedAt).not.toBeNull();
+  });
+
+  it('stores rejected message on failure', () => {
+    const state = reducer(
+      undefined,
+      fetchRespondQueue.rejected(new Error('boom'), '', undefined, 'boom')
+    );
+    expect(state.status).toBe('failed');
+    expect(state.error).toBe('boom');
+  });
+
+  it('silent pending does not switch status to loading', () => {
+    const state = reducer(undefined, fetchRespondQueue.pending('', { silent: true }));
+    expect(state.status).toBe('idle');
+  });
+
+  it('silent rejection does not change status', () => {
+    const state = reducer(
+      undefined,
+      fetchRespondQueue.rejected(new Error('network'), '', { silent: true }, 'network')
+    );
+    expect(state.status).toBe('idle');
+    expect(state.error).toBeNull();
+  });
+});
@@ -18,6 +18,7 @@ import channelConnectionsReducer from './channelConnectionsSlice';
 import chatRuntimeReducer from './chatRuntimeSlice';
 import notificationReducer from './notificationSlice';
 import notificationsReducer from './notificationsSlice';
+import providerSurfacesReducer from './providerSurfaceSlice';
 import socketReducer from './socketSlice';
 import threadReducer from './threadSlice';

@@ -56,6 +57,7 @@ export const store = configureStore({
    accounts: persistedAccountsReducer,
    notifications: persistedNotificationReducer,
    integrationNotifications: notificationsReducer,
+    providerSurfaces: providerSurfacesReducer,
  },
  middleware: getDefaultMiddleware => {
    const middleware = getDefaultMiddleware({
@@ -0,0 +1,64 @@
+import { createAsyncThunk, createSlice } from '@reduxjs/toolkit';
+
+import { providerSurfacesApi } from '../services/api/providerSurfacesApi';
+import type { RespondQueueItem } from '../types/providerSurfaces';
+
+interface ProviderSurfaceState {
+  queue: RespondQueueItem[];
+  count: number;
+  status: 'idle' | 'loading' | 'succeeded' | 'failed';
+  error: string | null;
+  lastSyncedAt: number | null;
+}
+
+const initialState: ProviderSurfaceState = {
+  queue: [],
+  count: 0,
+  status: 'idle',
+  error: null,
+  lastSyncedAt: null,
+};
+
+/** Pass `{ silent: true }` for background refresh (no loading flicker). */
+export const fetchRespondQueue = createAsyncThunk(
+  'providerSurfaces/fetchRespondQueue',
+  async (_options: { silent?: boolean } | undefined, { rejectWithValue }) => {
+    try {
+      return await providerSurfacesApi.listQueue();
+    } catch (error) {
+      return rejectWithValue(
+        error instanceof Error ? error.message : 'Failed to load provider respond queue'
+      );
+    }
+  }
+);
+
+const providerSurfaceSlice = createSlice({
+  name: 'providerSurfaces',
+  initialState,
+  reducers: {},
+  extraReducers: builder => {
+    builder
+      .addCase(fetchRespondQueue.pending, (state, action) => {
+        if (!action.meta.arg?.silent) {
+          state.status = 'loading';
+          state.error = null;
+        }
+      })
+      .addCase(fetchRespondQueue.fulfilled, (state, action) => {
+        state.status = 'succeeded';
+        state.queue = action.payload.items;
+        state.count = action.payload.count;
+        state.lastSyncedAt = Date.now();
+      })
+      .addCase(fetchRespondQueue.rejected, (state, action) => {
+        if (!action.meta.arg?.silent) {
+          state.status = 'failed';
+          state.error = (action.payload as string) ?? 'Failed to load provider respond queue';
+        }
+        // silent failures: leave status/error as-is; a subsequent successful poll will clear
+      });
+  },
+});
+
+export default providerSurfaceSlice.reducer;
@@ -2,6 +2,7 @@ import { createAsyncThunk, createSlice } from '@reduxjs/toolkit';

 import { threadApi } from '../services/api/threadApi';
 import type { Thread, ThreadMessage } from '../types/thread';
+import { IS_DEV } from '../utils/config';
 import { isTauri, openhumanLocalAiSuggestQuestions } from '../utils/tauriCommands';

 interface ThreadState {
@@ -123,7 +124,13 @@ export const addMessageLocal = createAsyncThunk(
 export const addInferenceResponse = createAsyncThunk(
  'thread/addInferenceResponse',
  async (
-    payload: { content: string; threadId?: string; messageId?: string; type?: string },
+    payload: {
+      content: string;
+      threadId?: string;
+      messageId?: string;
+      type?: string;
+      extraMetadata?: Record<string, unknown>;
+    },
    { getState, rejectWithValue }
  ) => {
    const state = getState() as { thread: ThreadState };
@@ -134,7 +141,7 @@ export const addInferenceResponse = createAsyncThunk(
      id: payload.messageId ?? `inference-${Date.now()}-${Math.random()}`,
      content: payload.content,
      type: payload.type ?? 'text',
-      extraMetadata: {},
+      extraMetadata: payload.extraMetadata ?? {},
      sender: 'agent',
      createdAt: new Date().toISOString(),
    };
@@ -166,7 +173,7 @@ export const generateThreadTitleIfNeeded = createAsyncThunk(
    try {
      await dispatch(loadThreads()).unwrap();
    } catch (error) {
-      if (import.meta.env.DEV) {
+      if (IS_DEV) {
        console.debug('[threadSlice] generateThreadTitleIfNeeded refresh failed', {
          threadId: payload.threadId,
          error,
@@ -73,6 +73,7 @@ vi.mock('../utils/tauriCommands', () => ({
 vi.mock('../utils/config', () => ({
  CORE_RPC_URL: 'http://127.0.0.1:7788/rpc',
  IS_DEV: true,
+  IS_PROD: false,
  DEV_FORCE_ONBOARDING: false,
  SKILLS_GITHUB_REPO: 'test/skills',
  SENTRY_DSN: undefined,
@@ -0,0 +1,21 @@
+export interface RespondQueueItem {
+  id: string;
+  provider: string;
+  accountId: string;
+  eventKind: string;
+  entityId: string;
+  threadId?: string;
+  title?: string;
+  snippet?: string;
+  senderName?: string;
+  senderHandle?: string;
+  timestamp: string;
+  deepLink?: string;
+  requiresAttention: boolean;
+  status: string;
+}
+
+export interface RespondQueueList {
+  items: RespondQueueItem[];
+  count: number;
+}
@@ -27,11 +27,21 @@ function parseToolTimeoutSecs(): number {
 export const TOOL_TIMEOUT_SECS = parseToolTimeoutSecs();

 export const IS_DEV = import.meta.env.DEV;
+export const IS_PROD = import.meta.env.PROD;

 /** Dev only: skip `.skip_onboarding` workspace check and ignore onboarded state so `/onboarding` always shows. Set `VITE_DEV_FORCE_ONBOARDING=true` in `.env.local`. */
 export const DEV_FORCE_ONBOARDING =
  import.meta.env.DEV && import.meta.env.VITE_DEV_FORCE_ONBOARDING === 'true';

+/**
+ * Consumer-first-session UX (intent picker, home IA, trust affordances).
+ * **Default off** so `main` stays unchanged until slices ship behind this flag.
+ * Opt in locally or in staging: `VITE_CONSUMER_FIRST_SESSION=true` in `app/.env.local`.
+ * Spec: `docs/plans/consumer-first-session-spec.md`.
+ */
+export const CONSUMER_FIRST_SESSION_ENABLED =
+  import.meta.env.VITE_CONSUMER_FIRST_SESSION === 'true';
+
 export const SKILLS_GITHUB_REPO =
  import.meta.env.VITE_SKILLS_GITHUB_REPO || 'tinyhumansai/openhuman-skills';