mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-30 06:32:24 +00:00
fix: make Rust log previews UTF-8 safe (#1874)
Co-authored-by: honor2030 <19909783+honor2030@users.noreply.github.com>
This commit is contained in:
@@ -878,7 +878,7 @@ async fn run_tool_call_loop_applies_per_tool_max_result_size_cap() {
|
||||
assert!(
|
||||
tool_results.content.contains("[truncated by tool cap:"),
|
||||
"expected truncation marker, got body: {}",
|
||||
&tool_results.content[..tool_results.content.len().min(200)]
|
||||
crate::openhuman::util::utf8_safe_prefix_at_byte_boundary(&tool_results.content, 200)
|
||||
);
|
||||
assert!(
|
||||
tool_results.content.len() < 1_000,
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
//! integration, this calls the Seltz API directly — no backend proxy needed.
|
||||
|
||||
use crate::openhuman::tools::traits::{Tool, ToolCallOptions, ToolResult};
|
||||
use crate::openhuman::util::utf8_safe_prefix_at_byte_boundary;
|
||||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::json;
|
||||
@@ -290,11 +291,7 @@ impl Tool for SeltzSearchTool {
|
||||
let status = resp.status();
|
||||
if !status.is_success() {
|
||||
let body_text = resp.text().await.unwrap_or_default();
|
||||
let detail = if body_text.len() > 500 {
|
||||
&body_text[..500]
|
||||
} else {
|
||||
&body_text
|
||||
};
|
||||
let detail = utf8_safe_prefix_at_byte_boundary(&body_text, 500);
|
||||
tracing::warn!(
|
||||
status = %status,
|
||||
"[seltz] non-2xx response: {detail}"
|
||||
|
||||
@@ -12,6 +12,8 @@ use log::{debug, info, warn};
|
||||
use parking_lot::Mutex;
|
||||
use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters};
|
||||
|
||||
use crate::openhuman::util::utf8_safe_prefix_at_byte_boundary;
|
||||
|
||||
/// Per-segment confidence threshold: reject segments with avg log-probability below this.
|
||||
const SEGMENT_LOGPROB_REJECT: f32 = -0.7;
|
||||
|
||||
@@ -168,7 +170,7 @@ pub fn transcribe_pcm_f32(
|
||||
params.set_initial_prompt(prompt);
|
||||
debug!(
|
||||
"{LOG_PREFIX} set initial_prompt: '{}...'",
|
||||
&prompt[..prompt.len().min(80)]
|
||||
utf8_safe_prefix_at_byte_boundary(prompt, 80)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ use tokio_tungstenite::{
|
||||
};
|
||||
|
||||
use crate::api::models::socket::ConnectionStatus;
|
||||
use crate::openhuman::util::utf8_safe_prefix_at_byte_boundary;
|
||||
|
||||
use super::event_handlers::{handle_sio_event, parse_sio_event};
|
||||
use super::manager::{emit_state_change, SharedState};
|
||||
@@ -371,7 +372,7 @@ async fn read_eio_open(
|
||||
}
|
||||
log::debug!(
|
||||
"[socket] Skipping non-OPEN packet: {}",
|
||||
&s[..s.len().min(40)]
|
||||
utf8_safe_prefix_at_byte_boundary(s, 40)
|
||||
);
|
||||
}
|
||||
Some(Ok(_)) => continue,
|
||||
@@ -416,7 +417,7 @@ async fn read_sio_connect_ack(
|
||||
}
|
||||
log::debug!(
|
||||
"[socket] Skipping packet during SIO handshake: {}",
|
||||
&s[..s.len().min(40)]
|
||||
utf8_safe_prefix_at_byte_boundary(s, 40)
|
||||
);
|
||||
}
|
||||
Some(Ok(_)) => continue,
|
||||
@@ -463,7 +464,7 @@ fn handle_eio_message(
|
||||
_ => {
|
||||
log::debug!(
|
||||
"[socket] Unknown EIO packet: {}",
|
||||
&text[..text.len().min(30)]
|
||||
utf8_safe_prefix_at_byte_boundary(text, 30)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -487,7 +488,7 @@ fn handle_sio_packet(
|
||||
} else {
|
||||
log::warn!(
|
||||
"[socket] Failed to parse SIO EVENT: {}",
|
||||
&text[..text.len().min(80)]
|
||||
utf8_safe_prefix_at_byte_boundary(text, 80)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -522,7 +523,7 @@ fn handle_sio_packet(
|
||||
_ => {
|
||||
log::debug!(
|
||||
"[socket] Unknown SIO packet type: {}",
|
||||
&text[..text.len().min(30)]
|
||||
utf8_safe_prefix_at_byte_boundary(text, 30)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -168,6 +168,19 @@ fn handle_eio_message_close_and_noop_do_not_panic() {
|
||||
handle_eio_message("9", &tx, &shared); // unknown
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn handle_eio_message_unknown_packet_is_utf8_safe_at_preview_boundary() {
|
||||
let previous_max_level = log::max_level();
|
||||
log::set_max_level(log::LevelFilter::Trace);
|
||||
let shared = make_shared();
|
||||
let (tx, _rx) = mpsc::unbounded_channel::<String>();
|
||||
let packet = format!("9{}{}", "a".repeat(28), "魔");
|
||||
assert!(!packet.is_char_boundary(30));
|
||||
|
||||
handle_eio_message(&packet, &tx, &shared);
|
||||
log::set_max_level(previous_max_level);
|
||||
}
|
||||
|
||||
// ── handle_sio_packet ──────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
@@ -190,6 +203,32 @@ fn handle_sio_packet_event_with_unparseable_payload_is_logged_only() {
|
||||
assert_eq!(*shared.status.read(), ConnectionStatus::Disconnected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn handle_sio_packet_unparseable_event_is_utf8_safe_at_preview_boundary() {
|
||||
let previous_max_level = log::max_level();
|
||||
log::set_max_level(log::LevelFilter::Trace);
|
||||
let shared = make_shared();
|
||||
let (tx, _rx) = mpsc::unbounded_channel::<String>();
|
||||
let packet = format!("2{}{}", "a".repeat(78), "魔");
|
||||
assert!(!packet.is_char_boundary(80));
|
||||
|
||||
handle_sio_packet(&packet, &tx, &shared);
|
||||
log::set_max_level(previous_max_level);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn handle_sio_packet_unknown_type_is_utf8_safe_at_preview_boundary() {
|
||||
let previous_max_level = log::max_level();
|
||||
log::set_max_level(log::LevelFilter::Trace);
|
||||
let shared = make_shared();
|
||||
let (tx, _rx) = mpsc::unbounded_channel::<String>();
|
||||
let packet = format!("9{}{}", "a".repeat(28), "魔");
|
||||
assert!(!packet.is_char_boundary(30));
|
||||
|
||||
handle_sio_packet(&packet, &tx, &shared);
|
||||
log::set_max_level(previous_max_level);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn handle_sio_packet_connect_reack_updates_sid() {
|
||||
let shared = make_shared();
|
||||
|
||||
@@ -9,6 +9,7 @@ use std::os::unix::fs::PermissionsExt as _;
|
||||
|
||||
use crate::openhuman::config::UpdateRestartStrategy;
|
||||
use crate::openhuman::update::types::{GitHubAsset, GitHubRelease, UpdateApplyResult, UpdateInfo};
|
||||
use crate::openhuman::util::utf8_safe_prefix_at_byte_boundary;
|
||||
|
||||
/// GitHub owner/repo for the core binary releases.
|
||||
const GITHUB_OWNER: &str = "tinyhumansai";
|
||||
@@ -139,7 +140,7 @@ pub async fn check_available() -> Result<UpdateInfo, String> {
|
||||
log::warn!(
|
||||
"[update] GitHub API returned {}: {}",
|
||||
status,
|
||||
&body[..body.len().min(200)]
|
||||
utf8_safe_prefix_at_byte_boundary(&body, 200)
|
||||
);
|
||||
let msg = format!("GitHub API error: {status}");
|
||||
if crate::core::observability::is_updater_transient_http_status(status.as_u16()) {
|
||||
|
||||
@@ -79,6 +79,13 @@ pub fn floor_char_boundary(s: &str, index: usize) -> usize {
|
||||
end
|
||||
}
|
||||
|
||||
/// Return a prefix of `s` whose byte length is at most `max_bytes`, backing up
|
||||
/// to the nearest UTF-8 character boundary when `max_bytes` falls in the middle
|
||||
/// of a multi-byte character.
|
||||
pub fn utf8_safe_prefix_at_byte_boundary(s: &str, max_bytes: usize) -> &str {
|
||||
&s[..floor_char_boundary(s, max_bytes)]
|
||||
}
|
||||
|
||||
/// Round a byte index UP to the nearest UTF-8 character boundary.
|
||||
pub fn ceil_char_boundary(s: &str, index: usize) -> usize {
|
||||
if index >= s.len() {
|
||||
@@ -227,6 +234,25 @@ mod tests {
|
||||
assert_eq!(floor_char_boundary(s, 100), 6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_utf8_safe_prefix_at_byte_boundary() {
|
||||
let s = format!("{}{}tail", "a".repeat(79), "魔");
|
||||
assert_eq!(utf8_safe_prefix_at_byte_boundary(&s, 80), "a".repeat(79));
|
||||
assert_eq!(utf8_safe_prefix_at_byte_boundary(&s, s.len()), s);
|
||||
assert_eq!(
|
||||
utf8_safe_prefix_at_byte_boundary("ascii preview", 5),
|
||||
"ascii"
|
||||
);
|
||||
assert_eq!(utf8_safe_prefix_at_byte_boundary("short", 80), "short");
|
||||
|
||||
for cap in [30, 40, 80, 200, 500] {
|
||||
let preview = format!("{}{}tail", "a".repeat(cap - 1), "界");
|
||||
let truncated = utf8_safe_prefix_at_byte_boundary(&preview, cap);
|
||||
assert_eq!(truncated, "a".repeat(cap - 1));
|
||||
assert!(preview.is_char_boundary(truncated.len()));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ceil_char_boundary() {
|
||||
let s = "A🦀C";
|
||||
|
||||
@@ -24,6 +24,7 @@ use super::postprocess;
|
||||
use crate::openhuman::config::Config;
|
||||
use crate::openhuman::local_ai;
|
||||
use crate::openhuman::local_ai::whisper_engine;
|
||||
use crate::openhuman::util::utf8_safe_prefix_at_byte_boundary;
|
||||
|
||||
const LOG_PREFIX: &str = "[voice-stream]";
|
||||
const AUDIO_SAMPLE_RATE: usize = 16_000;
|
||||
@@ -116,7 +117,7 @@ pub async fn handle_dictation_ws(mut socket: WebSocket, config: Arc<Config>) {
|
||||
"{LOG_PREFIX} partial transcription ({} samples, avg_logprob={:.3}): {}",
|
||||
samples.len(),
|
||||
result.avg_logprob.unwrap_or(0.0),
|
||||
&result.text[..result.text.len().min(80)]
|
||||
utf8_safe_prefix_at_byte_boundary(&result.text, 80)
|
||||
);
|
||||
if partial_tx.send(result.text).await.is_err() {
|
||||
break; // receiver dropped
|
||||
|
||||
Reference in New Issue
Block a user