feat: local model auto-reaction on user messages by channel type (#181)

* feat(conversations): implement auto-reaction feature for user messages

- Added functionality to automatically react to user messages with emojis based on the local AI's evaluation of the message content and channel type.
- Introduced `maybeAutoReact` function to handle the decision-making process for emoji reactions.
- Updated the `Conversations` component to store the last user message and trigger reactions accordingly.
- Enhanced the `tauriCommands` with a new method `openhumanLocalAiShouldReact` to facilitate local model evaluations for reactions.
- Updated local AI operations to include reaction decision logic, ensuring efficient and context-aware responses.

This feature enhances user engagement by adding a personal touch to interactions.

* style: fix prettier formatting in Conversations.tsx

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* style: apply cargo fmt to ops.rs

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix: address PR review — race condition, invisible reactions, flag emoji, PII log

- Replace lastUserMessageRef with per-thread pendingReactionRef Map so
  cancellation/retry clears stale entries and onDone only applies to
  the matching request round
- Render reactions on user message bubbles (not just agent messages)
  so auto-reactions are actually visible; manual picker stays agent-only
- Fix extract_first_emoji to consume consecutive regional indicator
  symbols as a single flag emoji (e.g. 🇺🇸); add regression test
- Replace raw_output in should_react debug log with output_len to
  avoid persisting user PII in traces

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* style: apply cargo fmt

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* refactor(conversations): remove unused imports for local AI transcription and TTS

- Cleaned up the Conversations component by removing unused imports related to local AI transcription and text-to-speech functionalities, streamlining the codebase and improving maintainability.es

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Steven Enamakel
2026-04-01 16:11:32 -07:00
committed by GitHub
co-authored by Claude Opus 4.6
parent cf344facf9
commit 207ec7d45c
4 changed files with 374 additions and 43 deletions
+93 -43
View File
@@ -35,6 +35,7 @@ import {
openhumanAutocompleteAccept,
openhumanAutocompleteCurrent,
openhumanLocalAiChat,
openhumanLocalAiShouldReact,
openhumanVoiceStatus,
openhumanVoiceTranscribeBytes,
openhumanVoiceTts,
@@ -125,6 +126,12 @@ const Conversations = () => {
const [isDelivering, setIsDelivering] = useState(false);
const deliveryActiveRef = useRef(false);
const [reactionPickerMsgId, setReactionPickerMsgId] = useState<string | null>(null);
const defaultChannelType = useAppSelector(
state => state.channelConnections?.defaultMessagingChannel ?? 'web'
);
const pendingReactionRef = useRef<
Map<string, { msgId: string; content: string; threadId: string }>
>(new Map());
const selectedThreadIdRef = useRef(selectedThreadId);
useEffect(() => {
@@ -377,6 +384,13 @@ const Conversations = () => {
};
});
// Fire-and-forget: auto-react to the user's message
const pending = pendingReactionRef.current.get(event.thread_id);
if (pending) {
maybeAutoReact(pending.msgId, pending.content, pending.threadId);
pendingReactionRef.current.delete(event.thread_id);
}
// Multi-bubble delivery gate: only when local model is active
if (!isLocalModelActiveRef.current) {
dispatch(
@@ -437,6 +451,9 @@ const Conversations = () => {
};
});
// Clear pending reaction so stale callbacks are ignored
pendingReactionRef.current.delete(event.thread_id);
if (event.error_type !== 'cancelled') {
// Deduplicate: skip if the last message is already an error
const currentState = store.getState() as {
@@ -499,6 +516,26 @@ const Conversations = () => {
setIsDelivering(false);
};
/**
* Fire-and-forget: ask the local model if we should auto-react to the
* user's message with an emoji. Adds a personal touch based on channel type.
*/
const maybeAutoReact = (userMessageId: string, messageContent: string, threadId: string) => {
if (!isTauri() || !isLocalModelActiveRef.current) return;
void openhumanLocalAiShouldReact(messageContent, defaultChannelType)
.then(response => {
const decision = response.result;
if (decision?.should_react && decision.emoji) {
console.debug('[conversations:auto-react] reacting with', decision.emoji);
dispatch(addReaction({ threadId, messageId: userMessageId, emoji: decision.emoji }));
}
})
.catch(err => {
console.debug('[conversations:auto-react] failed:', err);
});
};
const handleSendMessage = async (text?: string) => {
const normalized = text ?? inputValue;
const trimmed = normalized.trim();
@@ -525,6 +562,11 @@ const Conversations = () => {
};
dispatch(addMessageLocal({ threadId: sendingThreadId, message: userMessage }));
pendingReactionRef.current.set(sendingThreadId, {
msgId: userMessage.id,
content: trimmed,
threadId: sendingThreadId,
});
setInputValue('');
setSendError(null);
@@ -567,7 +609,10 @@ const Conversations = () => {
}
await deliverLocalResponse(reply, sendingThreadId);
pendingReactionRef.current.delete(sendingThreadId);
maybeAutoReact(userMessage.id, trimmed, sendingThreadId);
} catch (err) {
pendingReactionRef.current.delete(sendingThreadId);
const msg = err instanceof Error ? err.message : String(err);
setSendError(msg);
dispatch(
@@ -900,12 +945,16 @@ const Conversations = () => {
</svg>
)}
</button>
{msg.sender === 'agent' && (
<div className="mt-1 flex items-center gap-1 flex-wrap min-h-[20px]">
{(() => {
const myReactions =
(msg.extraMetadata?.myReactions as string[] | undefined) ?? [];
return myReactions.map(emoji => (
{(() => {
const myReactions =
(msg.extraMetadata?.myReactions as string[] | undefined) ?? [];
const hasReactions = myReactions.length > 0;
// Show reaction row if there are existing reactions (any sender)
// or if this is an agent message (manual picker available)
if (!hasReactions && msg.sender !== 'agent') return null;
return (
<div className="mt-1 flex items-center gap-1 flex-wrap min-h-[20px]">
{myReactions.map(emoji => (
<button
key={emoji}
onClick={() =>
@@ -922,46 +971,47 @@ const Conversations = () => {
title={`Remove ${emoji}`}>
{emoji}
</button>
));
})()}
{reactionPickerMsgId === msg.id ? (
<div className="flex items-center gap-0.5 px-1 py-0.5 rounded-full bg-white/10">
{['👍', '❤️', '😂', '🔥', '👀', '🎯'].map(emoji => (
))}
{msg.sender === 'agent' &&
(reactionPickerMsgId === msg.id ? (
<div className="flex items-center gap-0.5 px-1 py-0.5 rounded-full bg-white/10">
{['👍', '❤️', '😂', '🔥', '👀', '🎯'].map(emoji => (
<button
key={emoji}
onClick={() => {
if (selectedThreadId) {
dispatch(
addReaction({
threadId: selectedThreadId,
messageId: msg.id,
emoji,
})
);
}
setReactionPickerMsgId(null);
}}
className="px-0.5 rounded text-sm hover:scale-125 transition-transform"
title={emoji}>
{emoji}
</button>
))}
<button
onClick={() => setReactionPickerMsgId(null)}
className="ml-0.5 text-stone-600 hover:text-stone-400 text-xs px-0.5">
</button>
</div>
) : (
<button
key={emoji}
onClick={() => {
if (selectedThreadId) {
dispatch(
addReaction({
threadId: selectedThreadId,
messageId: msg.id,
emoji,
})
);
}
setReactionPickerMsgId(null);
}}
className="px-0.5 rounded text-sm hover:scale-125 transition-transform"
title={emoji}>
{emoji}
onClick={() => setReactionPickerMsgId(msg.id)}
className="opacity-0 group-hover/msg:opacity-100 flex items-center px-1.5 py-0.5 rounded-full bg-white/5 hover:bg-white/15 text-stone-500 hover:text-stone-300 text-xs transition-all"
title="Add reaction">
+
</button>
))}
<button
onClick={() => setReactionPickerMsgId(null)}
className="ml-0.5 text-stone-600 hover:text-stone-400 text-xs px-0.5">
</button>
</div>
) : (
<button
onClick={() => setReactionPickerMsgId(msg.id)}
className="opacity-0 group-hover/msg:opacity-100 flex items-center px-1.5 py-0.5 rounded-full bg-white/5 hover:bg-white/15 text-stone-500 hover:text-stone-300 text-xs transition-all"
title="Add reaction">
+
</button>
)}
</div>
)}
</div>
);
})()}
</div>
</div>
))}
+22
View File
@@ -1373,6 +1373,28 @@ export async function openhumanLocalAiChat(
});
}
// --- Reaction decision (local model) ---
export interface ReactionDecision {
should_react: boolean;
emoji: string | null;
}
/**
* Ask the local model whether the assistant should react to a user message
* with an emoji, based on the channel type. Designed to be fire-and-forget.
* Zero cloud cost — runs entirely on the local Ollama model.
*/
export async function openhumanLocalAiShouldReact(
message: string,
channelType: string
): Promise<CommandResponse<ReactionDecision>> {
return await callCoreRpc<CommandResponse<ReactionDecision>>({
method: 'openhuman.local_ai_should_react',
params: { message, channel_type: channelType },
});
}
export async function openhumanLocalAiAssetsStatus(): Promise<
CommandResponse<LocalAiAssetsStatus>
> {
+223
View File
@@ -479,3 +479,226 @@ pub async fn local_ai_chat(
);
Ok(RpcOutcome::single_log(reply, "local ai chat completed"))
}
/// Result of the reaction-decision prompt.
#[derive(Debug, serde::Serialize)]
pub struct ReactionDecision {
/// Whether the model thinks a reaction is appropriate.
pub should_react: bool,
/// The emoji to use (only meaningful when `should_react` is true).
pub emoji: Option<String>,
}
/// Ask the local model whether the assistant should add an emoji reaction to
/// the user's message, based on channel type and message content.
/// Designed to be called fire-and-forget — fast, lightweight, no cloud cost.
pub async fn local_ai_should_react(
config: &Config,
message: &str,
channel_type: &str,
) -> Result<RpcOutcome<ReactionDecision>, String> {
tracing::debug!(
channel_type,
msg_len = message.len(),
"[local_ai:should_react] evaluating reaction"
);
if message.trim().is_empty() {
return Ok(RpcOutcome::single_log(
ReactionDecision {
should_react: false,
emoji: None,
},
"empty message — no reaction",
));
}
let service = local_ai::global(config);
let status = service.status();
if !matches!(status.state.as_str(), "ready") {
tracing::debug!("[local_ai:should_react] local model not ready, skipping");
return Ok(RpcOutcome::single_log(
ReactionDecision {
should_react: false,
emoji: None,
},
"local model not ready",
));
}
let prompt = format!(
"You decide whether an AI assistant should react to a user message with a single emoji. \
Consider the channel context: casual channels (discord, telegram) get more frequent \
reactions with playful emojis, while professional channels (web, slack, email) are more \
reserved — only react to clearly emotional or noteworthy messages.\n\n\
Channel: {channel_type}\nUser message: {message}\n\n\
Reply with EXACTLY one word: either NONE (no reaction) or a single emoji character."
);
let output = service.prompt(config, &prompt, Some(8), true).await;
let decision = match output {
Ok(raw) => {
let trimmed = raw.trim();
tracing::debug!(
output_len = trimmed.len(),
"[local_ai:should_react] model response"
);
if trimmed.eq_ignore_ascii_case("NONE") || trimmed.is_empty() {
ReactionDecision {
should_react: false,
emoji: None,
}
} else {
// Extract the first emoji-like character(s) from the response
let emoji = extract_first_emoji(trimmed);
match emoji {
Some(e) => ReactionDecision {
should_react: true,
emoji: Some(e),
},
None => ReactionDecision {
should_react: false,
emoji: None,
},
}
}
}
Err(e) => {
tracing::debug!(error = %e, "[local_ai:should_react] inference failed, skipping");
ReactionDecision {
should_react: false,
emoji: None,
}
}
};
tracing::debug!(
should_react = decision.should_react,
emoji = ?decision.emoji,
"[local_ai:should_react] decision"
);
Ok(RpcOutcome::single_log(
decision,
"reaction decision completed",
))
}
/// Extract the first emoji from a string. Handles common emoji codepoints
/// including flag sequences (pairs of regional indicator symbols).
fn extract_first_emoji(text: &str) -> Option<String> {
let mut chars = text.chars();
while let Some(ch) = chars.next() {
// Regional indicator pair → flag emoji (e.g. 🇺🇸 = U+1F1FA U+1F1F8)
if is_regional_indicator(ch) {
let mut emoji = String::new();
emoji.push(ch);
// Consume consecutive regional indicators (flags are pairs)
for next in chars.by_ref() {
if is_regional_indicator(next) {
emoji.push(next);
} else {
break;
}
}
return Some(emoji);
}
if is_emoji_start(ch) {
let mut emoji = String::new();
emoji.push(ch);
// Consume joiners and variation selectors that extend the emoji
for next in chars.by_ref() {
if next == '\u{FE0F}' // variation selector
|| next == '\u{200D}' // zero-width joiner
|| ('\u{1F3FB}'..='\u{1F3FF}').contains(&next) // skin tones
|| is_emoji_start(next) && emoji.contains('\u{200D}')
{
emoji.push(next);
} else {
break;
}
}
return Some(emoji);
}
}
None
}
fn is_regional_indicator(ch: char) -> bool {
('\u{1F1E6}'..='\u{1F1FF}').contains(&ch)
}
fn is_emoji_start(ch: char) -> bool {
matches!(ch,
'\u{203C}' | '\u{2049}' // exclamation marks
| '\u{2139}' // information
| '\u{2194}'..='\u{2199}' // arrows
| '\u{21A9}'..='\u{21AA}' // arrows
| '\u{231A}'..='\u{231B}' // watch, hourglass
| '\u{23E9}'..='\u{23F3}' // media controls
| '\u{23F8}'..='\u{23FA}' // media controls
| '\u{24C2}' // circled M
| '\u{25AA}'..='\u{25AB}' // squares
| '\u{25B6}' | '\u{25C0}' // play buttons
| '\u{25FB}'..='\u{25FE}' // squares
| '\u{2328}' | '\u{23CF}' // keyboard, eject
| '\u{2600}'..='\u{27BF}' // misc symbols, dingbats
| '\u{2934}'..='\u{2935}' // arrows
| '\u{2B05}'..='\u{2B07}' // arrows
| '\u{2B1B}'..='\u{2B1C}' // squares
| '\u{2B50}' | '\u{2B55}' // star, circle
| '\u{FE00}'..='\u{FE0F}' // variation selectors
| '\u{1F300}'..='\u{1F9FF}' // misc symbols, emoticons, transport, supplemental
| '\u{1FA00}'..='\u{1FA6F}' // chess symbols, extended-A
| '\u{1FA70}'..='\u{1FAFF}' // symbols extended-A
| '\u{200D}' // ZWJ
)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn extract_emoji_from_simple_string() {
assert_eq!(extract_first_emoji("👍"), Some("👍".to_string()));
assert_eq!(extract_first_emoji("🔥"), Some("🔥".to_string()));
assert_eq!(extract_first_emoji("❤️"), Some("❤️".to_string()));
}
#[test]
fn extract_emoji_with_surrounding_text() {
assert_eq!(extract_first_emoji("Sure! 😂"), Some("😂".to_string()));
assert_eq!(
extract_first_emoji("I think 👀 fits here"),
Some("👀".to_string())
);
}
#[test]
fn extract_none_when_no_emoji() {
assert_eq!(extract_first_emoji("NONE"), None);
assert_eq!(extract_first_emoji("no reaction"), None);
assert_eq!(extract_first_emoji(""), None);
}
#[test]
fn extract_flag_emoji_keeps_pair_together() {
assert_eq!(extract_first_emoji("🇺🇸"), Some("🇺🇸".to_string()));
assert_eq!(
extract_first_emoji("🇬🇧 Great Britain"),
Some("🇬🇧".to_string())
);
}
#[test]
fn is_emoji_start_recognizes_common_emojis() {
assert!(is_emoji_start('👍'));
assert!(is_emoji_start('🔥'));
assert!(is_emoji_start('😂'));
assert!(is_emoji_start('⭐'));
assert!(!is_emoji_start('A'));
assert!(!is_emoji_start('1'));
}
}
+36
View File
@@ -111,6 +111,12 @@ struct LocalAiChatParams {
max_tokens: Option<u32>,
}
#[derive(Debug, Deserialize)]
struct LocalAiShouldReactParams {
message: String,
channel_type: String,
}
pub fn all_controller_schemas() -> Vec<ControllerSchema> {
vec![
schemas("agent_chat"),
@@ -138,6 +144,7 @@ pub fn all_controller_schemas() -> Vec<ControllerSchema> {
schemas("local_ai_set_ollama_path"),
schemas("local_ai_diagnostics"),
schemas("local_ai_chat"),
schemas("local_ai_should_react"),
]
}
@@ -243,6 +250,10 @@ pub fn all_registered_controllers() -> Vec<RegisteredController> {
schema: schemas("local_ai_chat"),
handler: handle_local_ai_chat,
},
RegisteredController {
schema: schemas("local_ai_should_react"),
handler: handle_local_ai_should_react,
},
]
}
@@ -488,6 +499,16 @@ pub fn schemas(function: &str) -> ControllerSchema {
],
outputs: vec![json_output("reply", "Assistant reply text.")],
},
"local_ai_should_react" => ControllerSchema {
namespace: "local_ai",
function: "should_react",
description: "Ask the local model whether the assistant should add an emoji reaction to a user message, based on channel type.",
inputs: vec![
required_string("message", "User message content to evaluate."),
required_string("channel_type", "Channel type: web, telegram, discord, slack, etc."),
],
outputs: vec![json_output("decision", "Reaction decision: {should_react, emoji}.")],
},
_ => ControllerSchema {
namespace: "local_ai",
function: "unknown",
@@ -852,6 +873,21 @@ fn handle_local_ai_set_ollama_path(params: Map<String, Value>) -> ControllerFutu
})
}
fn handle_local_ai_should_react(params: Map<String, Value>) -> ControllerFuture {
Box::pin(async move {
let p = deserialize_params::<LocalAiShouldReactParams>(params)?;
let config = config_rpc::load_config_with_timeout().await?;
to_json(
crate::openhuman::local_ai::rpc::local_ai_should_react(
&config,
&p.message,
&p.channel_type,
)
.await?,
)
})
}
fn handle_local_ai_chat(params: Map<String, Value>) -> ControllerFuture {
Box::pin(async move {
let p = deserialize_params::<LocalAiChatParams>(params)?;