diff --git a/src/openhuman/tokenjuice/detect/kind.rs b/src/openhuman/tokenjuice/detect/kind.rs index 6593aee2b..6a3201556 100644 --- a/src/openhuman/tokenjuice/detect/kind.rs +++ b/src/openhuman/tokenjuice/detect/kind.rs @@ -143,7 +143,9 @@ pub fn looks_like_diff(content: &str) -> bool { /// True if `content` looks like an HTML document: a doctype / ` bool { - let head = &content[..content.len().min(8192)]; + // Snap to a UTF-8 char boundary: a raw `&content[..8192]` panics when byte + // 8192 lands inside a multi-byte codepoint (CJK/emoji in large tool output). + let head = crate::openhuman::util::utf8_safe_prefix_at_byte_boundary(content, 8192); let lower = head.to_ascii_lowercase(); if lower.contains(" 8192); + // Plain text with no tags: just assert it returns without panicking. + assert!(!looks_like_html(&content)); + // And the full detector stays reachable on the same input. + assert_eq!( + detect_content_kind(&content, &hint()), + ContentKind::PlainText + ); + } + #[test] fn detect_code() { let c = "use std::fmt;\n\npub fn add(a: i32, b: i32) -> i32 {\n let c = a + b;\n return c;\n}\n\nstruct Foo {\n x: i32,\n}";