mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-30 23:14:37 +00:00
feat(local_ai): sentiment analysis, GIF decision & Tenor search (#373)
* feat(local_ai): add sentiment analysis, GIF decision, and Tenor search Extend the local model with two new capabilities: - Emotion/sentiment analysis (joy/sadness/anger/etc + valence + confidence) via a lightweight prompt, designed to run periodically (~hourly) - GIF decision + Tenor search: local model decides when a GIF response fits, generates a search query, and proxies through the backend's new Tenor API (POST /agent-integrations/tenor/search) New RPC endpoints: - openhuman.local_ai_analyze_sentiment - openhuman.local_ai_should_send_gif - openhuman.local_ai_tenor_search Frontend integrates with cadence-based invocation: - Reactions: every message (unchanged) - GIF decisions: every ~7 messages - Sentiment analysis: every ~1 hour Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * style: apply formatter fixes Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * merge g Please enter the commit message for your changes. Lines starting --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
a36535330b
commit
264bfe4081
@@ -33,6 +33,9 @@ import {
|
||||
isTauri,
|
||||
openhumanAutocompleteAccept,
|
||||
openhumanAutocompleteCurrent,
|
||||
openhumanLocalAiAnalyzeSentiment,
|
||||
openhumanLocalAiShouldSendGif,
|
||||
openhumanLocalAiTenorSearch,
|
||||
openhumanVoiceStatus,
|
||||
openhumanVoiceTranscribeBytes,
|
||||
openhumanVoiceTts,
|
||||
@@ -138,11 +141,21 @@ const Conversations = () => {
|
||||
Record<string, ToolTimelineEntry[]>
|
||||
>({});
|
||||
const rustChat = useRustChat();
|
||||
const defaultChannelType = useAppSelector(
|
||||
state => state.channelConnections?.defaultMessagingChannel ?? 'web'
|
||||
);
|
||||
const [reactionPickerMsgId, setReactionPickerMsgId] = useState<string | null>(null);
|
||||
const pendingReactionRef = useRef<
|
||||
Map<string, { msgId: string; content: string; threadId: string }>
|
||||
>(new Map());
|
||||
|
||||
/** Message counter for GIF cadence — check every ~7 messages. */
|
||||
const gifCadenceCountRef = useRef(0);
|
||||
const GIF_CADENCE_MESSAGES = 7;
|
||||
/** Timestamp (ms) of last sentiment analysis — run roughly every hour. */
|
||||
const lastSentimentAtRef = useRef(0);
|
||||
const SENTIMENT_INTERVAL_MS = 60 * 60 * 1000; // 1 hour
|
||||
|
||||
const selectedThreadIdRef = useRef(selectedThreadId);
|
||||
useEffect(() => {
|
||||
selectedThreadIdRef.current = selectedThreadId;
|
||||
@@ -431,6 +444,13 @@ const Conversations = () => {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Fire-and-forget: GIF decision + sentiment analysis (cadence-based)
|
||||
const pendingMsg = pendingReactionRef.current.get(event.thread_id);
|
||||
if (pendingMsg) {
|
||||
maybeCheckGif(pendingMsg.content, pendingMsg.threadId);
|
||||
maybeSentimentAnalysis(pendingMsg.content);
|
||||
}
|
||||
pendingReactionRef.current.delete(event.thread_id);
|
||||
|
||||
// Only add the response bubble if Rust didn't already deliver it
|
||||
@@ -495,6 +515,73 @@ const Conversations = () => {
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [rustChat, socketStatus]);
|
||||
|
||||
/**
|
||||
* Fire-and-forget: periodically check if a GIF response is appropriate
|
||||
* (every ~GIF_CADENCE_MESSAGES messages). If the model says yes, search
|
||||
* Tenor and dispatch the top result as a gif-type message.
|
||||
*/
|
||||
const maybeCheckGif = (messageContent: string, threadId: string) => {
|
||||
if (!isTauri()) return;
|
||||
|
||||
gifCadenceCountRef.current += 1;
|
||||
if (gifCadenceCountRef.current < GIF_CADENCE_MESSAGES) return;
|
||||
gifCadenceCountRef.current = 0;
|
||||
|
||||
console.debug('[conversations:gif] cadence reached, evaluating gif decision');
|
||||
|
||||
void openhumanLocalAiShouldSendGif(messageContent, defaultChannelType)
|
||||
.then(async response => {
|
||||
const decision = response.result;
|
||||
if (!decision?.should_send_gif || !decision.search_query) return;
|
||||
|
||||
console.debug('[conversations:gif] searching tenor for:', decision.search_query);
|
||||
const tenorResponse = await openhumanLocalAiTenorSearch(decision.search_query, 5);
|
||||
const results = tenorResponse.result?.results;
|
||||
if (!results || results.length === 0) return;
|
||||
|
||||
// Pick a random GIF from top results
|
||||
const picked = results[Math.floor(Math.random() * Math.min(results.length, 3))];
|
||||
const gifUrl =
|
||||
picked.media?.mediumgif?.url || picked.media?.gif?.url || picked.media?.tinygif?.url;
|
||||
if (!gifUrl) return;
|
||||
|
||||
console.debug('[conversations:gif] sending gif:', picked.title || picked.id);
|
||||
dispatch(addInferenceResponse({ content: gifUrl, threadId }));
|
||||
})
|
||||
.catch(err => {
|
||||
console.debug('[conversations:gif] failed:', err);
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Fire-and-forget: periodically analyze user sentiment (~every hour).
|
||||
* Stores the result in debug logs for now.
|
||||
*/
|
||||
const maybeSentimentAnalysis = (messageContent: string) => {
|
||||
if (!isTauri()) return;
|
||||
|
||||
const now = Date.now();
|
||||
if (now - lastSentimentAtRef.current < SENTIMENT_INTERVAL_MS) return;
|
||||
lastSentimentAtRef.current = now;
|
||||
|
||||
console.debug('[conversations:sentiment] interval reached, analyzing sentiment');
|
||||
|
||||
void openhumanLocalAiAnalyzeSentiment(messageContent)
|
||||
.then(response => {
|
||||
const sentiment = response.result;
|
||||
if (!sentiment) return;
|
||||
console.debug(
|
||||
'[conversations:sentiment] result:',
|
||||
sentiment.emotion,
|
||||
sentiment.valence,
|
||||
`(${sentiment.confidence})`
|
||||
);
|
||||
})
|
||||
.catch(err => {
|
||||
console.debug('[conversations:sentiment] failed:', err);
|
||||
});
|
||||
};
|
||||
|
||||
const handleSendMessage = async (text?: string) => {
|
||||
const normalized = text ?? inputValue;
|
||||
const trimmed = normalized.trim();
|
||||
|
||||
@@ -1613,6 +1613,89 @@ export async function openhumanLocalAiShouldReact(
|
||||
});
|
||||
}
|
||||
|
||||
// --- Sentiment analysis (local model) ---
|
||||
|
||||
export interface SentimentResult {
|
||||
emotion: string;
|
||||
valence: string;
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify the emotion and sentiment of a user message via the local model.
|
||||
* Designed to be called periodically (~every hour), not on every message.
|
||||
*/
|
||||
export async function openhumanLocalAiAnalyzeSentiment(
|
||||
message: string
|
||||
): Promise<CommandResponse<SentimentResult>> {
|
||||
return await callCoreRpc<CommandResponse<SentimentResult>>({
|
||||
method: 'openhuman.local_ai_analyze_sentiment',
|
||||
params: { message },
|
||||
});
|
||||
}
|
||||
|
||||
// --- GIF decision (local model) + Tenor search ---
|
||||
|
||||
export interface GifDecision {
|
||||
should_send_gif: boolean;
|
||||
search_query: string | null;
|
||||
}
|
||||
|
||||
export interface TenorMediaFormat {
|
||||
url: string;
|
||||
dims: [number, number];
|
||||
size: number;
|
||||
duration?: number;
|
||||
}
|
||||
|
||||
export interface TenorGifResult {
|
||||
id: string;
|
||||
title: string;
|
||||
contentDescription: string;
|
||||
url: string;
|
||||
media: {
|
||||
gif?: TenorMediaFormat;
|
||||
tinygif?: TenorMediaFormat;
|
||||
mediumgif?: TenorMediaFormat;
|
||||
mp4?: TenorMediaFormat;
|
||||
tinymp4?: TenorMediaFormat;
|
||||
};
|
||||
created: number;
|
||||
}
|
||||
|
||||
export interface TenorSearchResult {
|
||||
results: TenorGifResult[];
|
||||
next: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask the local model whether a GIF response is appropriate for this message.
|
||||
* Designed to be called every ~5-10 messages, not on every message.
|
||||
*/
|
||||
export async function openhumanLocalAiShouldSendGif(
|
||||
message: string,
|
||||
channelType: string
|
||||
): Promise<CommandResponse<GifDecision>> {
|
||||
return await callCoreRpc<CommandResponse<GifDecision>>({
|
||||
method: 'openhuman.local_ai_should_send_gif',
|
||||
params: { message, channel_type: channelType },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Search for GIFs via the backend Tenor proxy.
|
||||
* Requires a valid session (charges against user budget).
|
||||
*/
|
||||
export async function openhumanLocalAiTenorSearch(
|
||||
query: string,
|
||||
limit?: number
|
||||
): Promise<CommandResponse<TenorSearchResult>> {
|
||||
return await callCoreRpc<CommandResponse<TenorSearchResult>>({
|
||||
method: 'openhuman.local_ai_tenor_search',
|
||||
params: { query, limit },
|
||||
});
|
||||
}
|
||||
|
||||
export async function openhumanLocalAiAssetsStatus(): Promise<
|
||||
CommandResponse<LocalAiAssetsStatus>
|
||||
> {
|
||||
|
||||
@@ -525,6 +525,28 @@ impl BackendOAuthClient {
|
||||
.await
|
||||
}
|
||||
|
||||
/// `POST /agent-integrations/tenor/search` — Search for GIFs via Tenor.
|
||||
pub async fn search_tenor_gifs(
|
||||
&self,
|
||||
bearer_jwt: &str,
|
||||
query: &str,
|
||||
limit: Option<u32>,
|
||||
) -> Result<Value> {
|
||||
anyhow::ensure!(!query.trim().is_empty(), "query is required");
|
||||
let body = serde_json::json!({
|
||||
"query": query.trim(),
|
||||
"limit": limit.unwrap_or(5),
|
||||
"contentFilter": "medium",
|
||||
});
|
||||
self.authed_json(
|
||||
bearer_jwt,
|
||||
Method::POST,
|
||||
"agent-integrations/tenor/search",
|
||||
Some(body),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// `POST /channels/:channel/threads` — Create a thread in a channel.
|
||||
pub async fn create_channel_thread(
|
||||
&self,
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
//! GIF decision via local AI model + Tenor search via the backend API.
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::api::config::effective_api_url;
|
||||
use crate::api::jwt::get_session_token;
|
||||
use crate::api::rest::BackendOAuthClient;
|
||||
use crate::openhuman::config::Config;
|
||||
use crate::openhuman::local_ai;
|
||||
use crate::rpc::RpcOutcome;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// GIF decision — local model decides whether a GIF response is appropriate
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Result of the GIF-decision prompt.
|
||||
#[derive(Debug, serde::Serialize)]
|
||||
pub struct GifDecision {
|
||||
/// Whether the model thinks sending a GIF is appropriate right now.
|
||||
pub should_send_gif: bool,
|
||||
/// Tenor search query (only meaningful when `should_send_gif` is true).
|
||||
pub search_query: Option<String>,
|
||||
}
|
||||
|
||||
/// Ask the local model whether the assistant should respond with a GIF,
|
||||
/// based on channel type and message content. Designed to be called every
|
||||
/// ~5-10 messages, not on every message. Lightweight: ~12 output tokens.
|
||||
pub async fn local_ai_should_send_gif(
|
||||
config: &Config,
|
||||
message: &str,
|
||||
channel_type: &str,
|
||||
) -> Result<RpcOutcome<GifDecision>, String> {
|
||||
tracing::debug!(
|
||||
channel_type,
|
||||
msg_len = message.len(),
|
||||
"[local_ai:gif] evaluating gif decision"
|
||||
);
|
||||
|
||||
if message.trim().is_empty() {
|
||||
return Ok(RpcOutcome::single_log(
|
||||
GifDecision {
|
||||
should_send_gif: false,
|
||||
search_query: None,
|
||||
},
|
||||
"empty message — no gif",
|
||||
));
|
||||
}
|
||||
|
||||
let service = local_ai::global(config);
|
||||
let status = service.status();
|
||||
if !matches!(status.state.as_str(), "ready") {
|
||||
tracing::debug!("[local_ai:gif] local model not ready, skipping");
|
||||
return Ok(RpcOutcome::single_log(
|
||||
GifDecision {
|
||||
should_send_gif: false,
|
||||
search_query: None,
|
||||
},
|
||||
"local model not ready",
|
||||
));
|
||||
}
|
||||
|
||||
let prompt = format!(
|
||||
"You decide whether an AI assistant should respond with a GIF.\n\
|
||||
GIFs are appropriate for: humor, celebration, empathy, reactions to exciting news, \
|
||||
casual banter in friendly channels.\n\
|
||||
GIFs are NOT appropriate for: technical questions, serious topics, first messages, \
|
||||
professional channels (slack, email), or when the user seems upset or frustrated.\n\n\
|
||||
Channel: {channel_type}\nUser message: {message}\n\n\
|
||||
Reply with EXACTLY one line:\n\
|
||||
NONE (no GIF) OR a 2-4 word Tenor search query for a fitting GIF."
|
||||
);
|
||||
|
||||
let output = service.prompt(config, &prompt, Some(12), true).await;
|
||||
|
||||
let decision = match output {
|
||||
Ok(raw) => {
|
||||
let trimmed = raw.trim();
|
||||
tracing::debug!(
|
||||
response = %trimmed,
|
||||
"[local_ai:gif] model response"
|
||||
);
|
||||
parse_gif_response(trimmed)
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::debug!(error = %e, "[local_ai:gif] inference failed, skipping");
|
||||
GifDecision {
|
||||
should_send_gif: false,
|
||||
search_query: None,
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
tracing::debug!(
|
||||
should_send = decision.should_send_gif,
|
||||
query = ?decision.search_query,
|
||||
"[local_ai:gif] decision"
|
||||
);
|
||||
Ok(RpcOutcome::single_log(decision, "gif decision completed"))
|
||||
}
|
||||
|
||||
/// Parse the model's response into a `GifDecision`.
|
||||
fn parse_gif_response(text: &str) -> GifDecision {
|
||||
let trimmed = text.trim();
|
||||
|
||||
if trimmed.is_empty()
|
||||
|| trimmed.eq_ignore_ascii_case("NONE")
|
||||
|| trimmed.eq_ignore_ascii_case("no gif")
|
||||
{
|
||||
return GifDecision {
|
||||
should_send_gif: false,
|
||||
search_query: None,
|
||||
};
|
||||
}
|
||||
|
||||
// The model should return a short search query. Sanity-check length:
|
||||
// reject anything too long (probably the model rambled) or too short.
|
||||
let word_count = trimmed.split_whitespace().count();
|
||||
if word_count > 8 || trimmed.len() > 80 {
|
||||
tracing::debug!(
|
||||
words = word_count,
|
||||
len = trimmed.len(),
|
||||
"[local_ai:gif] response too long, treating as NONE"
|
||||
);
|
||||
return GifDecision {
|
||||
should_send_gif: false,
|
||||
search_query: None,
|
||||
};
|
||||
}
|
||||
|
||||
GifDecision {
|
||||
should_send_gif: true,
|
||||
search_query: Some(trimmed.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tenor search — proxy through the backend API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// A single GIF result from Tenor.
|
||||
#[derive(Debug, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TenorGifResult {
|
||||
pub id: String,
|
||||
pub title: String,
|
||||
#[serde(default)]
|
||||
pub content_description: String,
|
||||
pub url: String,
|
||||
#[serde(default)]
|
||||
pub media: Value,
|
||||
#[serde(default)]
|
||||
pub created: i64,
|
||||
}
|
||||
|
||||
/// Wrapper for the Tenor search response.
|
||||
#[derive(Debug, serde::Serialize, serde::Deserialize)]
|
||||
pub struct TenorSearchResult {
|
||||
pub results: Vec<TenorGifResult>,
|
||||
#[serde(default)]
|
||||
pub next: String,
|
||||
}
|
||||
|
||||
/// Search for GIFs via the backend's Tenor proxy endpoint.
|
||||
/// Requires a valid session JWT (the backend charges against user budget).
|
||||
pub async fn tenor_search(
|
||||
config: &Config,
|
||||
query: &str,
|
||||
limit: Option<u32>,
|
||||
) -> Result<RpcOutcome<TenorSearchResult>, String> {
|
||||
tracing::debug!(
|
||||
query,
|
||||
limit = ?limit,
|
||||
"[local_ai:gif] searching tenor"
|
||||
);
|
||||
|
||||
if query.trim().is_empty() {
|
||||
return Err("query is required".to_string());
|
||||
}
|
||||
|
||||
let api_url = effective_api_url(&config.api_url);
|
||||
let jwt = get_session_token(config)?
|
||||
.ok_or_else(|| "session JWT required; complete login first".to_string())?;
|
||||
|
||||
let client = BackendOAuthClient::new(&api_url).map_err(|e| e.to_string())?;
|
||||
let raw = client
|
||||
.search_tenor_gifs(&jwt, query, limit)
|
||||
.await
|
||||
.map_err(|e| format!("tenor search failed: {e}"))?;
|
||||
|
||||
tracing::debug!(
|
||||
result_keys = ?raw.as_object().map(|o| o.keys().collect::<Vec<_>>()),
|
||||
"[local_ai:gif] tenor search response received"
|
||||
);
|
||||
|
||||
// The backend wraps results in { success, data: { results, next, costUsd } }.
|
||||
// Extract the inner data.
|
||||
let data = raw.get("data").cloned().unwrap_or_else(|| raw.clone());
|
||||
|
||||
let result: TenorSearchResult = serde_json::from_value(data).map_err(|e| {
|
||||
tracing::debug!(error = %e, "[local_ai:gif] failed to parse tenor response");
|
||||
format!("parse tenor response: {e}")
|
||||
})?;
|
||||
|
||||
tracing::debug!(
|
||||
count = result.results.len(),
|
||||
"[local_ai:gif] tenor returned {} results",
|
||||
result.results.len()
|
||||
);
|
||||
|
||||
Ok(RpcOutcome::single_log(result, "tenor search completed"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_none_response() {
|
||||
let d = parse_gif_response("NONE");
|
||||
assert!(!d.should_send_gif);
|
||||
assert!(d.search_query.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_none_case_insensitive() {
|
||||
let d = parse_gif_response("none");
|
||||
assert!(!d.should_send_gif);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_empty_response() {
|
||||
let d = parse_gif_response("");
|
||||
assert!(!d.should_send_gif);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_valid_query() {
|
||||
let d = parse_gif_response("happy dance celebration");
|
||||
assert!(d.should_send_gif);
|
||||
assert_eq!(d.search_query.as_deref(), Some("happy dance celebration"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_short_query() {
|
||||
let d = parse_gif_response("thumbs up");
|
||||
assert!(d.should_send_gif);
|
||||
assert_eq!(d.search_query.as_deref(), Some("thumbs up"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_too_long_response() {
|
||||
let long = "this is a very long response that the model should not have generated because it rambled on and on";
|
||||
let d = parse_gif_response(long);
|
||||
assert!(!d.should_send_gif);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_no_gif_variant() {
|
||||
let d = parse_gif_response("no gif");
|
||||
assert!(!d.should_send_gif);
|
||||
}
|
||||
}
|
||||
@@ -2,9 +2,11 @@
|
||||
|
||||
mod core;
|
||||
pub mod device;
|
||||
pub mod gif_decision;
|
||||
pub mod ops;
|
||||
pub mod presets;
|
||||
mod schemas;
|
||||
pub mod sentiment;
|
||||
|
||||
mod install;
|
||||
pub(crate) mod model_ids;
|
||||
@@ -16,6 +18,7 @@ mod types;
|
||||
|
||||
pub use core::*;
|
||||
pub use device::DeviceProfile;
|
||||
pub use gif_decision::{GifDecision, TenorGifResult, TenorSearchResult};
|
||||
pub use ops as rpc;
|
||||
pub use ops::*;
|
||||
pub use presets::{ModelPreset, ModelTier};
|
||||
@@ -23,6 +26,7 @@ pub use schemas::{
|
||||
all_controller_schemas as all_local_ai_controller_schemas,
|
||||
all_registered_controllers as all_local_ai_registered_controllers,
|
||||
};
|
||||
pub use sentiment::SentimentResult;
|
||||
pub(crate) use service::whisper_engine;
|
||||
pub use service::LocalAiService;
|
||||
pub use types::{
|
||||
|
||||
@@ -117,6 +117,23 @@ struct LocalAiShouldReactParams {
|
||||
channel_type: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct LocalAiAnalyzeSentimentParams {
|
||||
message: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct LocalAiShouldSendGifParams {
|
||||
message: String,
|
||||
channel_type: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct LocalAiTenorSearchParams {
|
||||
query: String,
|
||||
limit: Option<u32>,
|
||||
}
|
||||
|
||||
pub fn all_controller_schemas() -> Vec<ControllerSchema> {
|
||||
vec![
|
||||
schemas("agent_chat"),
|
||||
@@ -145,6 +162,9 @@ pub fn all_controller_schemas() -> Vec<ControllerSchema> {
|
||||
schemas("local_ai_diagnostics"),
|
||||
schemas("local_ai_chat"),
|
||||
schemas("local_ai_should_react"),
|
||||
schemas("local_ai_analyze_sentiment"),
|
||||
schemas("local_ai_should_send_gif"),
|
||||
schemas("local_ai_tenor_search"),
|
||||
]
|
||||
}
|
||||
|
||||
@@ -254,6 +274,18 @@ pub fn all_registered_controllers() -> Vec<RegisteredController> {
|
||||
schema: schemas("local_ai_should_react"),
|
||||
handler: handle_local_ai_should_react,
|
||||
},
|
||||
RegisteredController {
|
||||
schema: schemas("local_ai_analyze_sentiment"),
|
||||
handler: handle_local_ai_analyze_sentiment,
|
||||
},
|
||||
RegisteredController {
|
||||
schema: schemas("local_ai_should_send_gif"),
|
||||
handler: handle_local_ai_should_send_gif,
|
||||
},
|
||||
RegisteredController {
|
||||
schema: schemas("local_ai_tenor_search"),
|
||||
handler: handle_local_ai_tenor_search,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
@@ -509,6 +541,35 @@ pub fn schemas(function: &str) -> ControllerSchema {
|
||||
],
|
||||
outputs: vec![json_output("decision", "Reaction decision: {should_react, emoji}.")],
|
||||
},
|
||||
"local_ai_analyze_sentiment" => ControllerSchema {
|
||||
namespace: "local_ai",
|
||||
function: "analyze_sentiment",
|
||||
description: "Classify the emotion and sentiment of a user message. Returns emotion label, valence, and confidence.",
|
||||
inputs: vec![
|
||||
required_string("message", "User message content to analyze."),
|
||||
],
|
||||
outputs: vec![json_output("sentiment", "Sentiment result: {emotion, valence, confidence}.")],
|
||||
},
|
||||
"local_ai_should_send_gif" => ControllerSchema {
|
||||
namespace: "local_ai",
|
||||
function: "should_send_gif",
|
||||
description: "Ask the local model whether a GIF response is appropriate, and if so return a Tenor search query.",
|
||||
inputs: vec![
|
||||
required_string("message", "User message content to evaluate."),
|
||||
required_string("channel_type", "Channel type: web, telegram, discord, slack, etc."),
|
||||
],
|
||||
outputs: vec![json_output("decision", "GIF decision: {should_send_gif, search_query}.")],
|
||||
},
|
||||
"local_ai_tenor_search" => ControllerSchema {
|
||||
namespace: "local_ai",
|
||||
function: "tenor_search",
|
||||
description: "Search for GIFs via the backend Tenor proxy. Requires a valid session.",
|
||||
inputs: vec![
|
||||
required_string("query", "Tenor search query."),
|
||||
optional_u64("limit", "Max results to return (default 5, max 50)."),
|
||||
],
|
||||
outputs: vec![json_output("result", "Tenor search result: {results, next}.")],
|
||||
},
|
||||
_ => ControllerSchema {
|
||||
namespace: "local_ai",
|
||||
function: "unknown",
|
||||
@@ -896,6 +957,43 @@ fn handle_local_ai_should_react(params: Map<String, Value>) -> ControllerFuture
|
||||
})
|
||||
}
|
||||
|
||||
fn handle_local_ai_analyze_sentiment(params: Map<String, Value>) -> ControllerFuture {
|
||||
Box::pin(async move {
|
||||
let p = deserialize_params::<LocalAiAnalyzeSentimentParams>(params)?;
|
||||
let config = config_rpc::load_config_with_timeout().await?;
|
||||
to_json(
|
||||
crate::openhuman::local_ai::sentiment::local_ai_analyze_sentiment(&config, &p.message)
|
||||
.await?,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn handle_local_ai_should_send_gif(params: Map<String, Value>) -> ControllerFuture {
|
||||
Box::pin(async move {
|
||||
let p = deserialize_params::<LocalAiShouldSendGifParams>(params)?;
|
||||
let config = config_rpc::load_config_with_timeout().await?;
|
||||
to_json(
|
||||
crate::openhuman::local_ai::gif_decision::local_ai_should_send_gif(
|
||||
&config,
|
||||
&p.message,
|
||||
&p.channel_type,
|
||||
)
|
||||
.await?,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn handle_local_ai_tenor_search(params: Map<String, Value>) -> ControllerFuture {
|
||||
Box::pin(async move {
|
||||
let p = deserialize_params::<LocalAiTenorSearchParams>(params)?;
|
||||
let config = config_rpc::load_config_with_timeout().await?;
|
||||
to_json(
|
||||
crate::openhuman::local_ai::gif_decision::tenor_search(&config, &p.query, p.limit)
|
||||
.await?,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn handle_local_ai_chat(params: Map<String, Value>) -> ControllerFuture {
|
||||
Box::pin(async move {
|
||||
let p = deserialize_params::<LocalAiChatParams>(params)?;
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
//! Emotion / sentiment analysis via the bundled local AI model.
|
||||
|
||||
use crate::openhuman::config::Config;
|
||||
use crate::openhuman::local_ai;
|
||||
use crate::rpc::RpcOutcome;
|
||||
|
||||
/// Result of sentiment / emotion analysis on a user message.
|
||||
#[derive(Debug, serde::Serialize)]
|
||||
pub struct SentimentResult {
|
||||
/// Primary emotion label.
|
||||
/// One of: joy, sadness, anger, surprise, fear, disgust, neutral.
|
||||
pub emotion: String,
|
||||
/// Overall valence: positive, negative, or neutral.
|
||||
pub valence: String,
|
||||
/// Model's self-reported confidence (0.0–1.0).
|
||||
pub confidence: f32,
|
||||
}
|
||||
|
||||
impl SentimentResult {
|
||||
/// Safe default when analysis is skipped or parsing fails.
|
||||
fn neutral() -> Self {
|
||||
Self {
|
||||
emotion: "neutral".to_string(),
|
||||
valence: "neutral".to_string(),
|
||||
confidence: 1.0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Known emotion labels the model is expected to produce.
|
||||
const VALID_EMOTIONS: &[&str] = &[
|
||||
"joy", "sadness", "anger", "surprise", "fear", "disgust", "neutral",
|
||||
];
|
||||
|
||||
/// Known valence labels.
|
||||
const VALID_VALENCES: &[&str] = &["positive", "negative", "neutral"];
|
||||
|
||||
/// Ask the local model to classify the emotion and sentiment of a user
|
||||
/// message. Designed to be called periodically (e.g. every hour), not on
|
||||
/// every single message. Lightweight: ~8 output tokens, fire-and-forget safe.
|
||||
pub async fn local_ai_analyze_sentiment(
|
||||
config: &Config,
|
||||
message: &str,
|
||||
) -> Result<RpcOutcome<SentimentResult>, String> {
|
||||
tracing::debug!(
|
||||
msg_len = message.len(),
|
||||
"[local_ai:sentiment] evaluating sentiment"
|
||||
);
|
||||
|
||||
if message.trim().is_empty() {
|
||||
return Ok(RpcOutcome::single_log(
|
||||
SentimentResult::neutral(),
|
||||
"empty message — neutral sentiment",
|
||||
));
|
||||
}
|
||||
|
||||
let service = local_ai::global(config);
|
||||
let status = service.status();
|
||||
if !matches!(status.state.as_str(), "ready") {
|
||||
tracing::debug!("[local_ai:sentiment] local model not ready, returning neutral");
|
||||
return Ok(RpcOutcome::single_log(
|
||||
SentimentResult::neutral(),
|
||||
"local model not ready",
|
||||
));
|
||||
}
|
||||
|
||||
let prompt = format!(
|
||||
"Classify the emotion and sentiment of this user message.\n\
|
||||
Reply with EXACTLY three words separated by spaces:\n\
|
||||
EMOTION VALENCE CONFIDENCE\n\
|
||||
Where EMOTION is one of: joy, sadness, anger, surprise, fear, disgust, neutral\n\
|
||||
VALENCE is one of: positive, negative, neutral\n\
|
||||
CONFIDENCE is a number from 0.0 to 1.0\n\n\
|
||||
User message: {message}"
|
||||
);
|
||||
|
||||
let output = service.prompt(config, &prompt, Some(8), true).await;
|
||||
|
||||
let result = match output {
|
||||
Ok(raw) => {
|
||||
let trimmed = raw.trim().to_lowercase();
|
||||
tracing::debug!(
|
||||
raw = %trimmed,
|
||||
"[local_ai:sentiment] model response"
|
||||
);
|
||||
parse_sentiment_response(&trimmed)
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::debug!(error = %e, "[local_ai:sentiment] inference failed, returning neutral");
|
||||
SentimentResult::neutral()
|
||||
}
|
||||
};
|
||||
|
||||
tracing::debug!(
|
||||
emotion = %result.emotion,
|
||||
valence = %result.valence,
|
||||
confidence = result.confidence,
|
||||
"[local_ai:sentiment] analysis complete"
|
||||
);
|
||||
Ok(RpcOutcome::single_log(
|
||||
result,
|
||||
"sentiment analysis completed",
|
||||
))
|
||||
}
|
||||
|
||||
/// Parse the model's 3-word response into a `SentimentResult`.
|
||||
/// Falls back to neutral on any parsing error.
|
||||
fn parse_sentiment_response(text: &str) -> SentimentResult {
|
||||
let parts: Vec<&str> = text.split_whitespace().collect();
|
||||
if parts.len() < 3 {
|
||||
tracing::debug!(
|
||||
parts = parts.len(),
|
||||
"[local_ai:sentiment] unexpected token count, falling back to neutral"
|
||||
);
|
||||
return SentimentResult::neutral();
|
||||
}
|
||||
|
||||
let emotion = parts[0].to_string();
|
||||
let valence = parts[1].to_string();
|
||||
let confidence: f32 = parts[2].parse().unwrap_or(0.5);
|
||||
|
||||
// Validate labels, fall back to neutral for garbage
|
||||
let emotion = if VALID_EMOTIONS.contains(&emotion.as_str()) {
|
||||
emotion
|
||||
} else {
|
||||
tracing::debug!(raw = %emotion, "[local_ai:sentiment] unknown emotion label, defaulting to neutral");
|
||||
"neutral".to_string()
|
||||
};
|
||||
|
||||
let valence = if VALID_VALENCES.contains(&valence.as_str()) {
|
||||
valence
|
||||
} else {
|
||||
tracing::debug!(raw = %valence, "[local_ai:sentiment] unknown valence label, defaulting to neutral");
|
||||
"neutral".to_string()
|
||||
};
|
||||
|
||||
let confidence = confidence.clamp(0.0, 1.0);
|
||||
|
||||
SentimentResult {
|
||||
emotion,
|
||||
valence,
|
||||
confidence,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_valid_response() {
|
||||
let r = parse_sentiment_response("joy positive 0.9");
|
||||
assert_eq!(r.emotion, "joy");
|
||||
assert_eq!(r.valence, "positive");
|
||||
assert!((r.confidence - 0.9).abs() < 0.01);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_valid_negative() {
|
||||
let r = parse_sentiment_response("anger negative 0.75");
|
||||
assert_eq!(r.emotion, "anger");
|
||||
assert_eq!(r.valence, "negative");
|
||||
assert!((r.confidence - 0.75).abs() < 0.01);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_unknown_emotion_falls_back() {
|
||||
let r = parse_sentiment_response("excited positive 0.8");
|
||||
assert_eq!(r.emotion, "neutral");
|
||||
assert_eq!(r.valence, "positive");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_too_few_tokens() {
|
||||
let r = parse_sentiment_response("joy");
|
||||
assert_eq!(r.emotion, "neutral");
|
||||
assert_eq!(r.valence, "neutral");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_bad_confidence() {
|
||||
let r = parse_sentiment_response("sadness negative abc");
|
||||
assert_eq!(r.emotion, "sadness");
|
||||
assert_eq!(r.valence, "negative");
|
||||
assert!((r.confidence - 0.5).abs() < 0.01);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_clamps_confidence() {
|
||||
let r = parse_sentiment_response("joy positive 2.5");
|
||||
assert!((r.confidence - 1.0).abs() < 0.01);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_empty_returns_neutral() {
|
||||
let r = parse_sentiment_response("");
|
||||
assert_eq!(r.emotion, "neutral");
|
||||
assert_eq!(r.valence, "neutral");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user