diff --git a/.env.example b/.env.example index 64fde524c..826cc2e4c 100644 --- a/.env.example +++ b/.env.example @@ -318,6 +318,37 @@ OPENHUMAN_SKILLS_WORKING_MEMORY_ENABLED=true # [optional] Reserved cache directory for future managed CPython installs. # OPENHUMAN_RUNTIME_PYTHON_CACHE_DIR= +# --------------------------------------------------------------------------- +# TokenJuice — content-aware tool-output compaction (the content router) +# --------------------------------------------------------------------------- +# [optional] Master switch for the content router (default: true). +# OPENHUMAN_TOKENJUICE_ENABLED=true +# [optional] Offload originals to the CCR cache + emit ⟦tj:…⟧ retrieval markers +# (default: true). Disabling makes compaction one-way. +# OPENHUMAN_TOKENJUICE_CCR_ENABLED=true +# [optional] Persist CCR originals to /.tokenjuice/ccr (default: false). +# OPENHUMAN_TOKENJUICE_CCR_DISK_ENABLED=false +# [optional] Per-compressor toggles (all default: true). +# OPENHUMAN_TOKENJUICE_SEARCH_ENABLED=true +# OPENHUMAN_TOKENJUICE_CODE_ENABLED=true +# OPENHUMAN_TOKENJUICE_HTML_ENABLED=true +# [optional] CCR cache limits. +# OPENHUMAN_TOKENJUICE_MAX_CACHE_ENTRIES=256 +# OPENHUMAN_TOKENJUICE_MAX_CACHE_BYTES=67108864 +# OPENHUMAN_TOKENJUICE_CCR_TTL_SECS= +# [optional] CCR only fires (offload + lossy compaction) for results estimated +# at >= this many tokens; smaller results pass through. Default 500. +# OPENHUMAN_TOKENJUICE_CCR_MIN_TOKENS=500 +# [optional] ML plain-text compressor ("Kompress", ModernBERT). Default OFF. +# Runs as a `kompress` backend of the shared runtime_python_server (torch is +# pip-installed at runtime), so it also requires OPENHUMAN_RUNTIME_PYTHON_ENABLED. +# OPENHUMAN_TOKENJUICE_ML_COMPRESSION_ENABLED=false +# OPENHUMAN_TOKENJUICE_ML_MODEL_ID=answerdotai/ModernBERT-base +# OPENHUMAN_TOKENJUICE_ML_DEVICE=cpu +# OPENHUMAN_TOKENJUICE_ML_TARGET_RATIO=0.5 +# OPENHUMAN_TOKENJUICE_ML_MAX_INPUT_CHARS=200000 +# OPENHUMAN_TOKENJUICE_ML_SIDECAR_IDLE_TIMEOUT_SECS=900 + # --------------------------------------------------------------------------- # Error Reporting (Sentry) # --------------------------------------------------------------------------- diff --git a/Cargo.lock b/Cargo.lock index 4f779bf44..7f562e105 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5275,6 +5275,10 @@ dependencies = [ "tracing-appender", "tracing-log", "tracing-subscriber", + "tree-sitter", + "tree-sitter-python", + "tree-sitter-rust", + "tree-sitter-typescript", "uiautomation", "unicode-normalization", "unicode-segmentation", @@ -7642,6 +7646,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + [[package]] name = "string_cache" version = "0.8.9" @@ -8367,6 +8377,55 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "tree-sitter" +version = "0.24.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5387dffa7ffc7d2dae12b50c6f7aab8ff79d6210147c6613561fc3d474c6f75" +dependencies = [ + "cc", + "regex", + "regex-syntax", + "streaming-iterator", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-language" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782" + +[[package]] +name = "tree-sitter-python" +version = "0.23.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d065aaa27f3aaceaf60c1f0e0ac09e1cb9eb8ed28e7bcdaa52129cffc7f4b04" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-rust" +version = "0.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca8ccb3e3a3495c8a943f6c3fd24c3804c471fd7f4f16087623c7fa4c0068e8a" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-typescript" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5f76ed8d947a75cc446d5fccd8b602ebf0cde64ccf2ffa434d873d7a575eff" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "try-lock" version = "0.2.5" diff --git a/Cargo.toml b/Cargo.toml index acbe63871..2a7a1da70 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,13 @@ crate-type = ["rlib"] [dependencies] # tiny.place A2A social network SDK — published on crates.io (tinyhumansai/tiny.place) tinyplace = "1.0.1" +# TokenJuice code compressor — AST-aware signature extraction. Optional (C build) +# behind the default `tokenjuice-treesitter` feature; disabling it falls back to +# the language-agnostic brace-depth heuristic. See src/openhuman/tokenjuice/compressors/code.rs. +tree-sitter = { version = "0.24", optional = true } +tree-sitter-rust = { version = "0.23", optional = true } +tree-sitter-typescript = { version = "0.23", optional = true } +tree-sitter-python = { version = "0.23", optional = true } serde = { version = "1", features = ["derive"] } serde_json = "1" serde_repr = "0.1" @@ -274,6 +281,15 @@ wiremock = "0.6" filetime = "0.2" [features] +default = ["tokenjuice-treesitter"] +# AST-aware code compression (tree-sitter Rust/TS/Python grammars; C build). +# On by default; disable to fall back to the brace-depth heuristic. +tokenjuice-treesitter = [ + "dep:tree-sitter", + "dep:tree-sitter-rust", + "dep:tree-sitter-typescript", + "dep:tree-sitter-python", +] sandbox-landlock = ["dep:landlock"] sandbox-bubblewrap = [] channel-matrix = ["dep:matrix-sdk"] diff --git a/app/src/components/settings/panels/TokenUsagePanel.tsx b/app/src/components/settings/panels/TokenUsagePanel.tsx new file mode 100644 index 000000000..c8fc3cfc1 --- /dev/null +++ b/app/src/components/settings/panels/TokenUsagePanel.tsx @@ -0,0 +1,329 @@ +import { useCallback, useEffect, useRef, useState } from 'react'; + +import { useT } from '../../../lib/i18n/I18nContext'; +import { + getTokenjuiceSavings, + getTokenjuiceSettings, + resetTokenjuiceSavings, + type SavingsStats, + type TokenjuiceSettings, + type TokenjuiceSettingsPatch, + updateTokenjuiceSettings, +} from '../../../utils/tauriCommands/tokenjuice'; +import Button from '../../ui/Button'; +import { + SettingsNumberField, + SettingsRow, + SettingsSection, + SettingsStatusLine, + SettingsSwitch, +} from '../controls'; +import SettingsPanel from '../layout/SettingsPanel'; + +function formatInt(n: number): string { + return Math.round(n).toLocaleString(); +} + +function formatUsd(n: number): string { + if (n > 0 && n < 0.01) return '<$0.01'; + return `$${n.toFixed(2)}`; +} + +function formatBytes(n: number): string { + if (n < 1024) return `${n} B`; + if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`; + return `${(n / (1024 * 1024)).toFixed(1)} MB`; +} + +interface StatTileProps { + label: string; + value: string; + hint?: string; +} + +const StatTile = ({ label, value, hint }: StatTileProps) => ( +
+
{label}
+
+ {value} +
+ {hint &&
{hint}
} +
+); + +const TokenUsagePanel = () => { + const { t } = useT(); + + const [settings, setSettings] = useState(null); + const [savings, setSavings] = useState(null); + const [error, setError] = useState(null); + const [saving, setSaving] = useState(false); + const [savedNote, setSavedNote] = useState(null); + + // Local editable string for the token-threshold field. + const [minTokensInput, setMinTokensInput] = useState(''); + const savedMinTokensRef = useRef(null); + + const loadSavings = useCallback(async () => { + try { + setSavings(await getTokenjuiceSavings()); + } catch (e) { + setError(e instanceof Error ? e.message : String(e)); + } + }, []); + + useEffect(() => { + let cancelled = false; + const load = async () => { + try { + const [s, v] = await Promise.all([getTokenjuiceSettings(), getTokenjuiceSavings()]); + if (cancelled) return; + setSettings(s); + setSavings(v); + setMinTokensInput(String(s.ccr_min_tokens)); + savedMinTokensRef.current = s.ccr_min_tokens; + } catch (e) { + if (!cancelled) setError(e instanceof Error ? e.message : String(e)); + } + }; + void load(); + return () => { + cancelled = true; + }; + }, []); + + const patch = useCallback( + async (p: TokenjuiceSettingsPatch) => { + setSaving(true); + setError(null); + setSavedNote(null); + try { + const next = await updateTokenjuiceSettings(p); + setSettings(next); + setMinTokensInput(String(next.ccr_min_tokens)); + savedMinTokensRef.current = next.ccr_min_tokens; + setSavedNote(t('settings.tokenUsage.saved')); + } catch (e) { + setError(e instanceof Error ? e.message : String(e)); + } finally { + setSaving(false); + } + }, + [t] + ); + + const commitMinTokens = useCallback(() => { + const parsed = Number.parseInt(minTokensInput, 10); + if (!Number.isFinite(parsed) || parsed < 0) { + setMinTokensInput(String(savedMinTokensRef.current ?? 0)); + return; + } + if (parsed === savedMinTokensRef.current) return; + void patch({ ccr_min_tokens: parsed }); + }, [minTokensInput, patch]); + + const onReset = useCallback(async () => { + try { + await resetTokenjuiceSavings(); + await loadSavings(); + } catch (e) { + setError(e instanceof Error ? e.message : String(e)); + } + }, [loadSavings]); + + const total = savings?.total; + + return ( + + {/* ── Savings statistics ─────────────────────────────────────────── */} + +
+ + + + +
+ + {savings && Object.keys(savings.byCompressor).length > 0 && ( +
+
+ {t('settings.tokenUsage.byCompressor')} +
+
+ {Object.entries(savings.byCompressor) + .sort((a, b) => b[1].tokensSaved - a[1].tokensSaved) + .map(([name, b]) => ( +
+ {name} + + {formatInt(b.tokensSaved)} tok · {formatUsd(b.costSavedUsd)} + +
+ ))} +
+
+ )} + +
+ + +
+
+ + {/* ── Compression toggles ────────────────────────────────────────── */} + +
+ void patch({ router_enabled: v })} + aria-label={t('settings.tokenUsage.routerEnabled')} + /> + } + /> + void patch({ search_enabled: v })} + aria-label={t('settings.tokenUsage.search')} + /> + } + /> + void patch({ code_enabled: v })} + aria-label={t('settings.tokenUsage.code')} + /> + } + /> + void patch({ html_enabled: v })} + aria-label={t('settings.tokenUsage.html')} + /> + } + /> + void patch({ ml_compression_enabled: v })} + aria-label={t('settings.tokenUsage.ml')} + /> + } + /> +
+
+ + {/* ── CCR cache ──────────────────────────────────────────────────── */} + +
+ void patch({ ccr_enabled: v })} + aria-label={t('settings.tokenUsage.ccrEnabled')} + /> + } + /> + + } + /> + void patch({ ccr_disk_enabled: v })} + aria-label={t('settings.tokenUsage.ccrDisk')} + /> + } + /> +
+
+ +
+
+
+ ); +}; + +export default TokenUsagePanel; diff --git a/app/src/components/settings/settingsRouteElements.tsx b/app/src/components/settings/settingsRouteElements.tsx index c9da5773d..38fcf573a 100644 --- a/app/src/components/settings/settingsRouteElements.tsx +++ b/app/src/components/settings/settingsRouteElements.tsx @@ -48,6 +48,7 @@ import TeamInvitesPanel from './panels/TeamInvitesPanel'; import TeamManagementPanel from './panels/TeamManagementPanel'; import TeamMembersPanel from './panels/TeamMembersPanel'; import TeamPanel from './panels/TeamPanel'; +import TokenUsagePanel from './panels/TokenUsagePanel'; import ToolPolicyDiagnosticsPanel from './panels/ToolPolicyDiagnosticsPanel'; import ToolsPanel from './panels/ToolsPanel'; import UsagePanel from './panels/UsagePanel'; @@ -153,6 +154,7 @@ export function settingsRouteElements(): ReactNode { {/* ── System ──────────────────────────────────────────────── */} )} /> )} /> + )} /> )} /> {/* ── Developer & Diagnostics leaf panels ─────────────────── */} diff --git a/app/src/components/settings/settingsRouteRegistry.ts b/app/src/components/settings/settingsRouteRegistry.ts index 3c06afcd6..6ee756653 100644 --- a/app/src/components/settings/settingsRouteRegistry.ts +++ b/app/src/components/settings/settingsRouteRegistry.ts @@ -737,6 +737,27 @@ export const SETTINGS_ROUTE_REGISTRY: SettingsRegistryEntry[] = [ navGroup: 'diagnosticsLogs', }, + // Token & Cost — TokenJuice compression settings + savings statistics. + { + id: 'token-usage', + titleKey: 'settings.tokenUsage.title', + descriptionKey: 'settings.tokenUsage.menuDesc', + section: 'ai', + navGroup: 'modelsInference', + navOrder: 5, + searchKeywords: [ + 'token', + 'tokens', + 'cost', + 'compression', + 'compaction', + 'tokenjuice', + 'cache', + 'ccr', + 'savings', + ], + }, + // ========================================================================= // INTENTIONALLY HIDDEN / DEEP-LINK ONLY (not surfaced in any menu) // ========================================================================= diff --git a/app/src/lib/i18n/ar.ts b/app/src/lib/i18n/ar.ts index 25cea2a8c..42d34231b 100644 --- a/app/src/lib/i18n/ar.ts +++ b/app/src/lib/i18n/ar.ts @@ -281,6 +281,47 @@ const messages: TranslationMap = { 'تحليل رسم الذاكرة البياني — المخطط، والمركزية، والتماسك، والارتباطات، والحداثة، والجدول الزمني، والمسارات، والمساحات', 'settings.buildInfo.title': 'معلومات الإصدار / البناء', 'settings.buildInfo.menuDesc': 'تفاصيل بناء التطبيق وإصداره واتصال النواة', + 'settings.tokenUsage.title': 'الرموز والتكلفة', + 'settings.tokenUsage.menuDesc': 'إعدادات الضغط ومقدار ما وفّرته من رموز ودولارات', + 'settings.tokenUsage.saving': 'جارٍ الحفظ…', + 'settings.tokenUsage.saved': 'تم الحفظ', + 'settings.tokenUsage.savingsTitle': 'التوفير', + 'settings.tokenUsage.attributedTo': 'تم تسعير التكلفة لـ {model}', + 'settings.tokenUsage.tokensSaved': 'الرموز الموفّرة', + 'settings.tokenUsage.costSaved': 'التكلفة الموفّرة', + 'settings.tokenUsage.cacheOccupancy': 'النسخ الأصلية المخزّنة مؤقتًا', + 'settings.tokenUsage.compactions': 'عمليات الضغط', + 'settings.tokenUsage.overEvents': 'عبر {count} عملية ضغط', + 'settings.tokenUsage.byCompressor': 'حسب الضاغط', + 'settings.tokenUsage.refresh': 'تحديث', + 'settings.tokenUsage.reset': 'إعادة تعيين الإحصائيات', + 'settings.tokenUsage.compressionTitle': 'الضغط', + 'settings.tokenUsage.compressionDesc': + 'ضغط واعٍ بالمحتوى لمخرجات الأدوات الكبيرة قبل دخولها سياق النموذج.', + 'settings.tokenUsage.routerEnabled': 'تمكين الضغط', + 'settings.tokenUsage.routerEnabledDesc': 'المفتاح الرئيسي لموجّه المحتوى.', + 'settings.tokenUsage.search': 'نتائج البحث', + 'settings.tokenUsage.searchDesc': + 'ترتيب مخرجات grep/البحث حسب الصلة، مع الاحتفاظ بأفضل التطابقات.', + 'settings.tokenUsage.code': 'الشيفرة المصدرية', + 'settings.tokenUsage.codeDesc': 'الاحتفاظ بالتواقيع وطيّ أجسام الدوال.', + 'settings.tokenUsage.html': 'HTML', + 'settings.tokenUsage.htmlDesc': 'إزالة الترميز لإظهار نص قابل للقراءة.', + 'settings.tokenUsage.ml': 'ضاغط النص بالتعلّم الآلي', + 'settings.tokenUsage.mlDesc': 'نموذج ModernBERT محلي للنص العادي (يتطلب بيئة تشغيل Python).', + 'settings.tokenUsage.ccrTitle': 'التخزين المؤقت والاسترجاع (CCR)', + 'settings.tokenUsage.ccrDesc': + 'يتم تخزين النسخ الأصلية المضغوطة مؤقتًا حتى يتمكّن الوكيل من جلب النص الكامل عند الطلب.', + 'settings.tokenUsage.ccrEnabled': 'الاحتفاظ بالنسخ الأصلية للاسترجاع', + 'settings.tokenUsage.ccrEnabledDesc': + 'تفريغ النسخة الأصلية وإضافة علامة استرجاع حتى لا يضيع أي شيء.', + 'settings.tokenUsage.ccrMinTokens': 'الحد الأدنى للحجم المراد تخزينه', + 'settings.tokenUsage.ccrMinTokensDesc': + 'لا يتم التخزين والضغط مع الفقد إلا للنتائج المقدّرة بهذا العدد من الرموز أو أكثر.', + 'settings.tokenUsage.tokensUnit': 'رمز', + 'settings.tokenUsage.ccrDisk': 'حفظ التخزين المؤقت على القرص', + 'settings.tokenUsage.ccrDiskDesc': + 'الاحتفاظ بالنسخ الأصلية القابلة للاسترجاع عبر عمليات إعادة التشغيل.', 'settings.dataSync.title': 'مزامنة البيانات', 'settings.dataSync.menuDesc': 'ما يقوم مساعدك بمزامنته — المصادر والحداثة والحالة', 'settings.dataSync.description': diff --git a/app/src/lib/i18n/bn.ts b/app/src/lib/i18n/bn.ts index 1edc134d9..a06f882db 100644 --- a/app/src/lib/i18n/bn.ts +++ b/app/src/lib/i18n/bn.ts @@ -286,6 +286,47 @@ const messages: TranslationMap = { 'মেমরি গ্রাফ বিশ্লেষণ — ডায়াগ্রাম, কেন্দ্রীয়তা, সংহতি, সংযোগ, সতেজতা, টাইমলাইন, পাথ এবং নেমস্পেস', 'settings.buildInfo.title': 'বিল্ড / সংস্করণ তথ্য', 'settings.buildInfo.menuDesc': 'অ্যাপ বিল্ড, সংস্করণ এবং কোর সংযোগের বিবরণ', + 'settings.tokenUsage.title': 'টোকেন ও খরচ', + 'settings.tokenUsage.menuDesc': 'সংকোচন সেটিংস এবং সেগুলি কত টোকেন ও ডলার সাশ্রয় করেছে', + 'settings.tokenUsage.saving': 'সংরক্ষণ করা হচ্ছে…', + 'settings.tokenUsage.saved': 'সংরক্ষিত হয়েছে', + 'settings.tokenUsage.savingsTitle': 'সাশ্রয়', + 'settings.tokenUsage.attributedTo': '{model}-এর জন্য খরচ নির্ধারিত', + 'settings.tokenUsage.tokensSaved': 'সাশ্রয় হওয়া টোকেন', + 'settings.tokenUsage.costSaved': 'সাশ্রয় হওয়া খরচ', + 'settings.tokenUsage.cacheOccupancy': 'ক্যাশ করা মূল কপি', + 'settings.tokenUsage.compactions': 'সংকোচন', + 'settings.tokenUsage.overEvents': '{count}টি সংকোচন জুড়ে', + 'settings.tokenUsage.byCompressor': 'সংকোচকারী অনুযায়ী', + 'settings.tokenUsage.refresh': 'রিফ্রেশ', + 'settings.tokenUsage.reset': 'পরিসংখ্যান রিসেট করুন', + 'settings.tokenUsage.compressionTitle': 'সংকোচন', + 'settings.tokenUsage.compressionDesc': + 'বড় টুল আউটপুট মডেল প্রসঙ্গে প্রবেশের আগে বিষয়বস্তু-সচেতন সংকোচন।', + 'settings.tokenUsage.routerEnabled': 'সংকোচন সক্ষম করুন', + 'settings.tokenUsage.routerEnabledDesc': 'বিষয়বস্তু রাউটারের প্রধান সুইচ।', + 'settings.tokenUsage.search': 'অনুসন্ধানের ফলাফল', + 'settings.tokenUsage.searchDesc': + 'প্রাসঙ্গিকতা অনুযায়ী grep/অনুসন্ধান আউটপুট সাজান, সেরা মিলগুলি রেখে দিন।', + 'settings.tokenUsage.code': 'সোর্স কোড', + 'settings.tokenUsage.codeDesc': 'স্বাক্ষর রাখুন, ফাংশন বডি সংকুচিত করুন।', + 'settings.tokenUsage.html': 'HTML', + 'settings.tokenUsage.htmlDesc': 'পঠনযোগ্য টেক্সটে মার্কআপ সরিয়ে ফেলুন।', + 'settings.tokenUsage.ml': 'ML টেক্সট সংকোচক', + 'settings.tokenUsage.mlDesc': + 'সাধারণ টেক্সটের জন্য স্থানীয় ModernBERT মডেল (Python রানটাইম প্রয়োজন)।', + 'settings.tokenUsage.ccrTitle': 'ক্যাশ ও পুনরুদ্ধার (CCR)', + 'settings.tokenUsage.ccrDesc': + 'সংকুচিত মূল কপিগুলি ক্যাশ করা হয় যাতে এজেন্ট চাহিদামতো সম্পূর্ণ টেক্সট আনতে পারে।', + 'settings.tokenUsage.ccrEnabled': 'পুনরুদ্ধারের জন্য মূল কপি রাখুন', + 'settings.tokenUsage.ccrEnabledDesc': + 'মূল কপি অফলোড করুন এবং একটি পুনরুদ্ধার চিহ্ন যোগ করুন যাতে কিছুই হারিয়ে না যায়।', + 'settings.tokenUsage.ccrMinTokens': 'ক্যাশ করার ন্যূনতম আকার', + 'settings.tokenUsage.ccrMinTokensDesc': + 'কেবল এই সংখ্যক টোকেন বা তার বেশি অনুমানিত ফলাফলই ক্যাশ ও ক্ষতিসহ সংকুচিত করুন।', + 'settings.tokenUsage.tokensUnit': 'টোকেন', + 'settings.tokenUsage.ccrDisk': 'ক্যাশ ডিস্কে সংরক্ষণ করুন', + 'settings.tokenUsage.ccrDiskDesc': 'রিস্টার্ট জুড়ে পুনরুদ্ধারযোগ্য মূল কপি রাখুন।', 'settings.dataSync.title': 'ডেটা সিঙ্ক', 'settings.dataSync.menuDesc': 'আপনার সহকারী যা সিঙ্ক করে — উৎস, সতেজতা এবং স্থিতি', 'settings.dataSync.description': diff --git a/app/src/lib/i18n/de.ts b/app/src/lib/i18n/de.ts index 4c9a6a88b..c1e2376c8 100644 --- a/app/src/lib/i18n/de.ts +++ b/app/src/lib/i18n/de.ts @@ -289,6 +289,48 @@ const messages: TranslationMap = { 'Speichergraph-Analyse — Diagramm, Zentralität, Kohäsion, Verknüpfungen, Aktualität, Zeitachse, Pfade und Namensräume', 'settings.buildInfo.title': 'Build-/Versionsinfo', 'settings.buildInfo.menuDesc': 'App-Build, Version und Details zur Core-Verbindung', + 'settings.tokenUsage.title': 'Tokens & Kosten', + 'settings.tokenUsage.menuDesc': + 'Komprimierungseinstellungen und wie viele Tokens und Dollar sie eingespart haben', + 'settings.tokenUsage.saving': 'Wird gespeichert…', + 'settings.tokenUsage.saved': 'Gespeichert', + 'settings.tokenUsage.savingsTitle': 'Einsparungen', + 'settings.tokenUsage.attributedTo': 'Kosten kalkuliert für {model}', + 'settings.tokenUsage.tokensSaved': 'Eingesparte Tokens', + 'settings.tokenUsage.costSaved': 'Eingesparte Kosten', + 'settings.tokenUsage.cacheOccupancy': 'Zwischengespeicherte Originale', + 'settings.tokenUsage.compactions': 'Komprimierungen', + 'settings.tokenUsage.overEvents': 'über {count} Komprimierungen', + 'settings.tokenUsage.byCompressor': 'Nach Kompressor', + 'settings.tokenUsage.refresh': 'Aktualisieren', + 'settings.tokenUsage.reset': 'Statistiken zurücksetzen', + 'settings.tokenUsage.compressionTitle': 'Komprimierung', + 'settings.tokenUsage.compressionDesc': + 'Inhaltsbewusste Komprimierung großer Tool-Ausgaben, bevor sie in den Modellkontext gelangen.', + 'settings.tokenUsage.routerEnabled': 'Komprimierung aktivieren', + 'settings.tokenUsage.routerEnabledDesc': 'Hauptschalter für den Inhalts-Router.', + 'settings.tokenUsage.search': 'Suchergebnisse', + 'settings.tokenUsage.searchDesc': + 'grep-/Suchausgabe nach Relevanz sortieren und die besten Treffer behalten.', + 'settings.tokenUsage.code': 'Quellcode', + 'settings.tokenUsage.codeDesc': 'Signaturen behalten, Funktionskörper einklappen.', + 'settings.tokenUsage.html': 'HTML', + 'settings.tokenUsage.htmlDesc': 'Markup zu lesbarem Text entfernen.', + 'settings.tokenUsage.ml': 'ML-Textkompressor', + 'settings.tokenUsage.mlDesc': + 'Lokales ModernBERT-Modell für Klartext (erfordert Python-Laufzeitumgebung).', + 'settings.tokenUsage.ccrTitle': 'Cache & Wiederherstellung (CCR)', + 'settings.tokenUsage.ccrDesc': + 'Komprimierte Originale werden zwischengespeichert, damit der Agent den vollständigen Text bei Bedarf abrufen kann.', + 'settings.tokenUsage.ccrEnabled': 'Originale zur Wiederherstellung behalten', + 'settings.tokenUsage.ccrEnabledDesc': + 'Das Original auslagern und eine Abrufmarkierung hinzufügen, damit nichts verloren geht.', + 'settings.tokenUsage.ccrMinTokens': 'Mindestgröße zum Zwischenspeichern', + 'settings.tokenUsage.ccrMinTokensDesc': + 'Nur Ergebnisse zwischenspeichern und verlustbehaftet komprimieren, die auf diese Anzahl an Tokens oder mehr geschätzt werden.', + 'settings.tokenUsage.tokensUnit': 'Tokens', + 'settings.tokenUsage.ccrDisk': 'Cache auf Datenträger speichern', + 'settings.tokenUsage.ccrDiskDesc': 'Wiederherstellbare Originale über Neustarts hinweg behalten.', 'settings.dataSync.title': 'Datensynchronisierung', 'settings.dataSync.menuDesc': 'Was dein Assistent synchronisiert — Quellen, Aktualität und Status', diff --git a/app/src/lib/i18n/en.ts b/app/src/lib/i18n/en.ts index 5546520e0..d8fdedaa9 100644 --- a/app/src/lib/i18n/en.ts +++ b/app/src/lib/i18n/en.ts @@ -298,6 +298,46 @@ const en: TranslationMap = { 'Memory graph analysis — diagram, centrality, cohesion, associations, freshness, timeline, paths, and namespaces', 'settings.buildInfo.title': 'Build / version info', 'settings.buildInfo.menuDesc': 'App build, version, and core connection details', + 'settings.tokenUsage.title': 'Token & Cost', + 'settings.tokenUsage.menuDesc': + 'Compression settings and how many tokens and dollars they have saved', + 'settings.tokenUsage.saving': 'Saving…', + 'settings.tokenUsage.saved': 'Saved', + 'settings.tokenUsage.savingsTitle': 'Savings', + 'settings.tokenUsage.attributedTo': 'Cost priced for {model}', + 'settings.tokenUsage.tokensSaved': 'Tokens saved', + 'settings.tokenUsage.costSaved': 'Cost saved', + 'settings.tokenUsage.cacheOccupancy': 'Cached originals', + 'settings.tokenUsage.compactions': 'Compactions', + 'settings.tokenUsage.overEvents': 'over {count} compactions', + 'settings.tokenUsage.byCompressor': 'By compressor', + 'settings.tokenUsage.refresh': 'Refresh', + 'settings.tokenUsage.reset': 'Reset stats', + 'settings.tokenUsage.compressionTitle': 'Compression', + 'settings.tokenUsage.compressionDesc': + 'Content-aware compaction of large tool output before it enters the model context.', + 'settings.tokenUsage.routerEnabled': 'Enable compression', + 'settings.tokenUsage.routerEnabledDesc': 'Master switch for the content router.', + 'settings.tokenUsage.search': 'Search results', + 'settings.tokenUsage.searchDesc': 'Relevance-rank grep/search output, keeping top matches.', + 'settings.tokenUsage.code': 'Source code', + 'settings.tokenUsage.codeDesc': 'Keep signatures, collapse function bodies.', + 'settings.tokenUsage.html': 'HTML', + 'settings.tokenUsage.htmlDesc': 'Strip markup to readable text.', + 'settings.tokenUsage.ml': 'ML text compressor', + 'settings.tokenUsage.mlDesc': 'Local ModernBERT model for plain text (requires Python runtime).', + 'settings.tokenUsage.ccrTitle': 'Cache & recovery (CCR)', + 'settings.tokenUsage.ccrDesc': + 'Compacted originals are cached so the agent can fetch the full text on demand.', + 'settings.tokenUsage.ccrEnabled': 'Keep originals for recovery', + 'settings.tokenUsage.ccrEnabledDesc': + 'Offload the original and add a retrieval marker so nothing is lost.', + 'settings.tokenUsage.ccrMinTokens': 'Minimum size to cache', + 'settings.tokenUsage.ccrMinTokensDesc': + 'Only cache and lossily compact results estimated at or above this many tokens.', + 'settings.tokenUsage.tokensUnit': 'tokens', + 'settings.tokenUsage.ccrDisk': 'Persist cache to disk', + 'settings.tokenUsage.ccrDiskDesc': 'Keep recoverable originals across restarts.', 'settings.dataSync.title': 'Data Sync', 'settings.dataSync.menuDesc': 'What your assistant syncs — sources, freshness, and status', 'settings.dataSync.description': diff --git a/app/src/lib/i18n/es.ts b/app/src/lib/i18n/es.ts index 21d7b6366..f7816a75e 100644 --- a/app/src/lib/i18n/es.ts +++ b/app/src/lib/i18n/es.ts @@ -288,6 +288,47 @@ const messages: TranslationMap = { 'Análisis del grafo de memoria: diagrama, centralidad, cohesión, asociaciones, frescura, línea de tiempo, rutas y espacios de nombres', 'settings.buildInfo.title': 'Información de compilación/versión', 'settings.buildInfo.menuDesc': 'Compilación de la app, versión y detalles de conexión del núcleo', + 'settings.tokenUsage.title': 'Tokens y coste', + 'settings.tokenUsage.menuDesc': 'Ajustes de compresión y cuántos tokens y dólares han ahorrado', + 'settings.tokenUsage.saving': 'Guardando…', + 'settings.tokenUsage.saved': 'Guardado', + 'settings.tokenUsage.savingsTitle': 'Ahorros', + 'settings.tokenUsage.attributedTo': 'Coste calculado para {model}', + 'settings.tokenUsage.tokensSaved': 'Tokens ahorrados', + 'settings.tokenUsage.costSaved': 'Coste ahorrado', + 'settings.tokenUsage.cacheOccupancy': 'Originales en caché', + 'settings.tokenUsage.compactions': 'Compactaciones', + 'settings.tokenUsage.overEvents': 'a lo largo de {count} compactaciones', + 'settings.tokenUsage.byCompressor': 'Por compresor', + 'settings.tokenUsage.refresh': 'Actualizar', + 'settings.tokenUsage.reset': 'Restablecer estadísticas', + 'settings.tokenUsage.compressionTitle': 'Compresión', + 'settings.tokenUsage.compressionDesc': + 'Compactación según el contenido de las salidas grandes de herramientas antes de que entren en el contexto del modelo.', + 'settings.tokenUsage.routerEnabled': 'Activar compresión', + 'settings.tokenUsage.routerEnabledDesc': 'Interruptor principal del enrutador de contenido.', + 'settings.tokenUsage.search': 'Resultados de búsqueda', + 'settings.tokenUsage.searchDesc': + 'Ordenar la salida de grep/búsqueda por relevancia, conservando las mejores coincidencias.', + 'settings.tokenUsage.code': 'Código fuente', + 'settings.tokenUsage.codeDesc': 'Conservar las firmas, plegar los cuerpos de las funciones.', + 'settings.tokenUsage.html': 'HTML', + 'settings.tokenUsage.htmlDesc': 'Eliminar el marcado para obtener texto legible.', + 'settings.tokenUsage.ml': 'Compresor de texto con ML', + 'settings.tokenUsage.mlDesc': + 'Modelo ModernBERT local para texto sin formato (requiere entorno de ejecución de Python).', + 'settings.tokenUsage.ccrTitle': 'Caché y recuperación (CCR)', + 'settings.tokenUsage.ccrDesc': + 'Los originales compactados se almacenan en caché para que el agente pueda recuperar el texto completo cuando lo necesite.', + 'settings.tokenUsage.ccrEnabled': 'Conservar originales para recuperación', + 'settings.tokenUsage.ccrEnabledDesc': + 'Descargar el original y añadir un marcador de recuperación para que no se pierda nada.', + 'settings.tokenUsage.ccrMinTokens': 'Tamaño mínimo para almacenar en caché', + 'settings.tokenUsage.ccrMinTokensDesc': + 'Almacenar en caché y compactar con pérdida solo los resultados estimados en esta cantidad de tokens o más.', + 'settings.tokenUsage.tokensUnit': 'tokens', + 'settings.tokenUsage.ccrDisk': 'Conservar la caché en disco', + 'settings.tokenUsage.ccrDiskDesc': 'Mantener los originales recuperables entre reinicios.', 'settings.dataSync.title': 'Sincronización de datos', 'settings.dataSync.menuDesc': 'Lo que sincroniza tu asistente: fuentes, actualidad y estado', 'settings.dataSync.description': diff --git a/app/src/lib/i18n/fr.ts b/app/src/lib/i18n/fr.ts index d4fec3212..a2bda42d7 100644 --- a/app/src/lib/i18n/fr.ts +++ b/app/src/lib/i18n/fr.ts @@ -290,6 +290,49 @@ const messages: TranslationMap = { 'Analyse du graphe mémoire — diagramme, centralité, cohésion, associations, fraîcheur, chronologie, chemins et espaces de noms', 'settings.buildInfo.title': 'Infos de build/version', 'settings.buildInfo.menuDesc': 'Build de l’application, version et détails de connexion du cœur', + 'settings.tokenUsage.title': 'Tokens et coût', + 'settings.tokenUsage.menuDesc': + 'Paramètres de compression et combien de tokens et de dollars ils ont économisés', + 'settings.tokenUsage.saving': 'Enregistrement…', + 'settings.tokenUsage.saved': 'Enregistré', + 'settings.tokenUsage.savingsTitle': 'Économies', + 'settings.tokenUsage.attributedTo': 'Coût calculé pour {model}', + 'settings.tokenUsage.tokensSaved': 'Tokens économisés', + 'settings.tokenUsage.costSaved': 'Coût économisé', + 'settings.tokenUsage.cacheOccupancy': 'Originaux mis en cache', + 'settings.tokenUsage.compactions': 'Compactages', + 'settings.tokenUsage.overEvents': 'sur {count} compactages', + 'settings.tokenUsage.byCompressor': 'Par compresseur', + 'settings.tokenUsage.refresh': 'Actualiser', + 'settings.tokenUsage.reset': 'Réinitialiser les statistiques', + 'settings.tokenUsage.compressionTitle': 'Compression', + 'settings.tokenUsage.compressionDesc': + 'Compactage adapté au contenu des grandes sorties d’outils avant leur entrée dans le contexte du modèle.', + 'settings.tokenUsage.routerEnabled': 'Activer la compression', + 'settings.tokenUsage.routerEnabledDesc': 'Interrupteur principal du routeur de contenu.', + 'settings.tokenUsage.search': 'Résultats de recherche', + 'settings.tokenUsage.searchDesc': + 'Classer la sortie grep/recherche par pertinence, en conservant les meilleures correspondances.', + 'settings.tokenUsage.code': 'Code source', + 'settings.tokenUsage.codeDesc': 'Conserver les signatures, replier le corps des fonctions.', + 'settings.tokenUsage.html': 'HTML', + 'settings.tokenUsage.htmlDesc': 'Supprimer le balisage pour obtenir un texte lisible.', + 'settings.tokenUsage.ml': 'Compresseur de texte ML', + 'settings.tokenUsage.mlDesc': + 'Modèle ModernBERT local pour le texte brut (nécessite l’environnement d’exécution Python).', + 'settings.tokenUsage.ccrTitle': 'Cache et récupération (CCR)', + 'settings.tokenUsage.ccrDesc': + 'Les originaux compactés sont mis en cache afin que l’agent puisse récupérer le texte complet à la demande.', + 'settings.tokenUsage.ccrEnabled': 'Conserver les originaux pour récupération', + 'settings.tokenUsage.ccrEnabledDesc': + 'Décharger l’original et ajouter un marqueur de récupération afin que rien ne soit perdu.', + 'settings.tokenUsage.ccrMinTokens': 'Taille minimale à mettre en cache', + 'settings.tokenUsage.ccrMinTokensDesc': + 'Ne mettre en cache et compacter avec perte que les résultats estimés à ce nombre de tokens ou plus.', + 'settings.tokenUsage.tokensUnit': 'tokens', + 'settings.tokenUsage.ccrDisk': 'Conserver le cache sur le disque', + 'settings.tokenUsage.ccrDiskDesc': + 'Conserver les originaux récupérables d’un redémarrage à l’autre.', 'settings.dataSync.title': 'Synchronisation des données', 'settings.dataSync.menuDesc': 'Ce que votre assistant synchronise — sources, fraîcheur et état', 'settings.dataSync.description': diff --git a/app/src/lib/i18n/hi.ts b/app/src/lib/i18n/hi.ts index 32ae90a22..d9e7dfb83 100644 --- a/app/src/lib/i18n/hi.ts +++ b/app/src/lib/i18n/hi.ts @@ -285,6 +285,47 @@ const messages: TranslationMap = { 'मेमोरी ग्राफ़ विश्लेषण — डायग्राम, केंद्रीयता, संसक्ति, संबंध, ताज़गी, टाइमलाइन, पथ और नेमस्पेस', 'settings.buildInfo.title': 'बिल्ड / संस्करण जानकारी', 'settings.buildInfo.menuDesc': 'ऐप बिल्ड, संस्करण और कोर कनेक्शन विवरण', + 'settings.tokenUsage.title': 'टोकन और लागत', + 'settings.tokenUsage.menuDesc': 'संपीड़न सेटिंग्स और उन्होंने कितने टोकन और डॉलर बचाए हैं', + 'settings.tokenUsage.saving': 'सहेजा जा रहा है…', + 'settings.tokenUsage.saved': 'सहेजा गया', + 'settings.tokenUsage.savingsTitle': 'बचत', + 'settings.tokenUsage.attributedTo': '{model} के लिए लागत निर्धारित', + 'settings.tokenUsage.tokensSaved': 'बचाए गए टोकन', + 'settings.tokenUsage.costSaved': 'बचाई गई लागत', + 'settings.tokenUsage.cacheOccupancy': 'कैश की गई मूल प्रतियाँ', + 'settings.tokenUsage.compactions': 'संपीड़न', + 'settings.tokenUsage.overEvents': '{count} संपीड़न में', + 'settings.tokenUsage.byCompressor': 'कंप्रेसर के अनुसार', + 'settings.tokenUsage.refresh': 'रीफ़्रेश करें', + 'settings.tokenUsage.reset': 'आँकड़े रीसेट करें', + 'settings.tokenUsage.compressionTitle': 'संपीड़न', + 'settings.tokenUsage.compressionDesc': + 'बड़े टूल आउटपुट को मॉडल संदर्भ में प्रवेश करने से पहले सामग्री-जागरूक संपीड़न।', + 'settings.tokenUsage.routerEnabled': 'संपीड़न सक्षम करें', + 'settings.tokenUsage.routerEnabledDesc': 'सामग्री राउटर का मुख्य स्विच।', + 'settings.tokenUsage.search': 'खोज परिणाम', + 'settings.tokenUsage.searchDesc': + 'grep/खोज आउटपुट को प्रासंगिकता के अनुसार क्रमबद्ध करें, शीर्ष मिलानों को रखते हुए।', + 'settings.tokenUsage.code': 'स्रोत कोड', + 'settings.tokenUsage.codeDesc': 'हस्ताक्षर रखें, फ़ंक्शन बॉडी को संक्षिप्त करें।', + 'settings.tokenUsage.html': 'HTML', + 'settings.tokenUsage.htmlDesc': 'पठनीय टेक्स्ट के लिए मार्कअप हटाएँ।', + 'settings.tokenUsage.ml': 'ML टेक्स्ट कंप्रेसर', + 'settings.tokenUsage.mlDesc': + 'सादे टेक्स्ट के लिए स्थानीय ModernBERT मॉडल (Python रनटाइम आवश्यक)।', + 'settings.tokenUsage.ccrTitle': 'कैश और पुनर्प्राप्ति (CCR)', + 'settings.tokenUsage.ccrDesc': + 'संपीड़ित मूल प्रतियाँ कैश की जाती हैं ताकि एजेंट माँग पर पूरा टेक्स्ट प्राप्त कर सके।', + 'settings.tokenUsage.ccrEnabled': 'पुनर्प्राप्ति के लिए मूल प्रतियाँ रखें', + 'settings.tokenUsage.ccrEnabledDesc': + 'मूल प्रति को ऑफ़लोड करें और एक पुनर्प्राप्ति चिह्न जोड़ें ताकि कुछ भी न खोए।', + 'settings.tokenUsage.ccrMinTokens': 'कैश करने का न्यूनतम आकार', + 'settings.tokenUsage.ccrMinTokensDesc': + 'केवल इतने या इससे अधिक टोकन अनुमानित परिणामों को ही कैश और हानिपूर्ण रूप से संपीड़ित करें।', + 'settings.tokenUsage.tokensUnit': 'टोकन', + 'settings.tokenUsage.ccrDisk': 'कैश को डिस्क पर सहेजें', + 'settings.tokenUsage.ccrDiskDesc': 'पुनरारंभ के बीच पुनर्प्राप्त करने योग्य मूल प्रतियाँ रखें।', 'settings.dataSync.title': 'डेटा सिंक', 'settings.dataSync.menuDesc': 'आपका सहायक क्या सिंक करता है — स्रोत, ताज़गी और स्थिति', 'settings.dataSync.description': diff --git a/app/src/lib/i18n/id.ts b/app/src/lib/i18n/id.ts index f6d7d5404..ca38f3870 100644 --- a/app/src/lib/i18n/id.ts +++ b/app/src/lib/i18n/id.ts @@ -286,6 +286,49 @@ const messages: TranslationMap = { 'Analisis grafik memori — diagram, sentralitas, kohesi, asosiasi, kesegaran, lini masa, jalur, dan namespace', 'settings.buildInfo.title': 'Info build / versi', 'settings.buildInfo.menuDesc': 'Build aplikasi, versi, dan detail koneksi core', + 'settings.tokenUsage.title': 'Token & Biaya', + 'settings.tokenUsage.menuDesc': + 'Pengaturan kompresi dan berapa banyak token dan dolar yang telah dihemat', + 'settings.tokenUsage.saving': 'Menyimpan…', + 'settings.tokenUsage.saved': 'Tersimpan', + 'settings.tokenUsage.savingsTitle': 'Penghematan', + 'settings.tokenUsage.attributedTo': 'Biaya dihitung untuk {model}', + 'settings.tokenUsage.tokensSaved': 'Token yang dihemat', + 'settings.tokenUsage.costSaved': 'Biaya yang dihemat', + 'settings.tokenUsage.cacheOccupancy': 'Naskah asli yang di-cache', + 'settings.tokenUsage.compactions': 'Pemadatan', + 'settings.tokenUsage.overEvents': 'dari {count} pemadatan', + 'settings.tokenUsage.byCompressor': 'Menurut kompresor', + 'settings.tokenUsage.refresh': 'Segarkan', + 'settings.tokenUsage.reset': 'Setel ulang statistik', + 'settings.tokenUsage.compressionTitle': 'Kompresi', + 'settings.tokenUsage.compressionDesc': + 'Pemadatan yang sadar konten untuk keluaran alat besar sebelum masuk ke konteks model.', + 'settings.tokenUsage.routerEnabled': 'Aktifkan kompresi', + 'settings.tokenUsage.routerEnabledDesc': 'Sakelar utama untuk router konten.', + 'settings.tokenUsage.search': 'Hasil pencarian', + 'settings.tokenUsage.searchDesc': + 'Mengurutkan keluaran grep/pencarian menurut relevansi, mempertahankan kecocokan teratas.', + 'settings.tokenUsage.code': 'Kode sumber', + 'settings.tokenUsage.codeDesc': 'Pertahankan tanda tangan, lipat badan fungsi.', + 'settings.tokenUsage.html': 'HTML', + 'settings.tokenUsage.htmlDesc': 'Hapus markup menjadi teks yang dapat dibaca.', + 'settings.tokenUsage.ml': 'Kompresor teks ML', + 'settings.tokenUsage.mlDesc': + 'Model ModernBERT lokal untuk teks biasa (memerlukan runtime Python).', + 'settings.tokenUsage.ccrTitle': 'Cache & pemulihan (CCR)', + 'settings.tokenUsage.ccrDesc': + 'Naskah asli yang dipadatkan di-cache sehingga agen dapat mengambil teks lengkap sesuai permintaan.', + 'settings.tokenUsage.ccrEnabled': 'Simpan naskah asli untuk pemulihan', + 'settings.tokenUsage.ccrEnabledDesc': + 'Pindahkan naskah asli dan tambahkan penanda pengambilan sehingga tidak ada yang hilang.', + 'settings.tokenUsage.ccrMinTokens': 'Ukuran minimum untuk di-cache', + 'settings.tokenUsage.ccrMinTokensDesc': + 'Hanya cache dan padatkan secara lossy hasil yang diperkirakan sebanyak ini token atau lebih.', + 'settings.tokenUsage.tokensUnit': 'token', + 'settings.tokenUsage.ccrDisk': 'Simpan cache ke disk', + 'settings.tokenUsage.ccrDiskDesc': + 'Pertahankan naskah asli yang dapat dipulihkan di seluruh mulai ulang.', 'settings.dataSync.title': 'Sinkronisasi Data', 'settings.dataSync.menuDesc': 'Yang disinkronkan asisten Anda — sumber, kesegaran, dan status', 'settings.dataSync.description': diff --git a/app/src/lib/i18n/it.ts b/app/src/lib/i18n/it.ts index b62072076..ba884275f 100644 --- a/app/src/lib/i18n/it.ts +++ b/app/src/lib/i18n/it.ts @@ -289,6 +289,49 @@ const messages: TranslationMap = { 'Analisi del grafo di memoria — diagramma, centralità, coesione, associazioni, freschezza, cronologia, percorsi e namespace', 'settings.buildInfo.title': 'Info build/versione', 'settings.buildInfo.menuDesc': 'Build dell’app, versione e dettagli di connessione del core', + 'settings.tokenUsage.title': 'Token e costo', + 'settings.tokenUsage.menuDesc': + 'Impostazioni di compressione e quanti token e dollari hanno fatto risparmiare', + 'settings.tokenUsage.saving': 'Salvataggio…', + 'settings.tokenUsage.saved': 'Salvato', + 'settings.tokenUsage.savingsTitle': 'Risparmi', + 'settings.tokenUsage.attributedTo': 'Costo calcolato per {model}', + 'settings.tokenUsage.tokensSaved': 'Token risparmiati', + 'settings.tokenUsage.costSaved': 'Costo risparmiato', + 'settings.tokenUsage.cacheOccupancy': 'Originali memorizzati nella cache', + 'settings.tokenUsage.compactions': 'Compattazioni', + 'settings.tokenUsage.overEvents': 'su {count} compattazioni', + 'settings.tokenUsage.byCompressor': 'Per compressore', + 'settings.tokenUsage.refresh': 'Aggiorna', + 'settings.tokenUsage.reset': 'Reimposta statistiche', + 'settings.tokenUsage.compressionTitle': 'Compressione', + 'settings.tokenUsage.compressionDesc': + 'Compattazione consapevole del contenuto dei grandi output degli strumenti prima che entrino nel contesto del modello.', + 'settings.tokenUsage.routerEnabled': 'Abilita compressione', + 'settings.tokenUsage.routerEnabledDesc': 'Interruttore principale del router dei contenuti.', + 'settings.tokenUsage.search': 'Risultati di ricerca', + 'settings.tokenUsage.searchDesc': + 'Ordina l’output di grep/ricerca per pertinenza, mantenendo le corrispondenze migliori.', + 'settings.tokenUsage.code': 'Codice sorgente', + 'settings.tokenUsage.codeDesc': 'Mantieni le firme, comprimi i corpi delle funzioni.', + 'settings.tokenUsage.html': 'HTML', + 'settings.tokenUsage.htmlDesc': 'Rimuovi il markup per ottenere testo leggibile.', + 'settings.tokenUsage.ml': 'Compressore di testo ML', + 'settings.tokenUsage.mlDesc': + 'Modello ModernBERT locale per testo semplice (richiede l’ambiente di esecuzione Python).', + 'settings.tokenUsage.ccrTitle': 'Cache e recupero (CCR)', + 'settings.tokenUsage.ccrDesc': + 'Gli originali compattati vengono memorizzati nella cache così l’agente può recuperare il testo completo su richiesta.', + 'settings.tokenUsage.ccrEnabled': 'Conserva gli originali per il recupero', + 'settings.tokenUsage.ccrEnabledDesc': + 'Sposta l’originale e aggiungi un indicatore di recupero così non si perde nulla.', + 'settings.tokenUsage.ccrMinTokens': 'Dimensione minima da memorizzare nella cache', + 'settings.tokenUsage.ccrMinTokensDesc': + 'Memorizza nella cache e compatta con perdita solo i risultati stimati a questo numero di token o più.', + 'settings.tokenUsage.tokensUnit': 'token', + 'settings.tokenUsage.ccrDisk': 'Mantieni la cache su disco', + 'settings.tokenUsage.ccrDiskDesc': + 'Conserva gli originali recuperabili tra un riavvio e l’altro.', 'settings.dataSync.title': 'Sincronizzazione dati', 'settings.dataSync.menuDesc': 'Ciò che il tuo assistente sincronizza — fonti, freschezza e stato', 'settings.dataSync.description': diff --git a/app/src/lib/i18n/ko.ts b/app/src/lib/i18n/ko.ts index 0ba079a95..f7cb7f661 100644 --- a/app/src/lib/i18n/ko.ts +++ b/app/src/lib/i18n/ko.ts @@ -287,6 +287,46 @@ const messages: TranslationMap = { '메모리 그래프 분석 — 다이어그램, 중심성, 응집도, 연관, 최신성, 타임라인, 경로 및 네임스페이스', 'settings.buildInfo.title': '빌드 / 버전 정보', 'settings.buildInfo.menuDesc': '앱 빌드, 버전 및 코어 연결 세부 정보', + 'settings.tokenUsage.title': '토큰 및 비용', + 'settings.tokenUsage.menuDesc': '압축 설정과 그것이 절약한 토큰 및 달러의 양', + 'settings.tokenUsage.saving': '저장 중…', + 'settings.tokenUsage.saved': '저장됨', + 'settings.tokenUsage.savingsTitle': '절감', + 'settings.tokenUsage.attributedTo': '{model} 기준으로 비용 책정됨', + 'settings.tokenUsage.tokensSaved': '절약된 토큰', + 'settings.tokenUsage.costSaved': '절약된 비용', + 'settings.tokenUsage.cacheOccupancy': '캐시된 원본', + 'settings.tokenUsage.compactions': '압축 횟수', + 'settings.tokenUsage.overEvents': '{count}회 압축에 걸쳐', + 'settings.tokenUsage.byCompressor': '압축기별', + 'settings.tokenUsage.refresh': '새로고침', + 'settings.tokenUsage.reset': '통계 재설정', + 'settings.tokenUsage.compressionTitle': '압축', + 'settings.tokenUsage.compressionDesc': + '대용량 도구 출력이 모델 컨텍스트에 들어가기 전에 콘텐츠를 인식하여 압축합니다.', + 'settings.tokenUsage.routerEnabled': '압축 활성화', + 'settings.tokenUsage.routerEnabledDesc': '콘텐츠 라우터의 마스터 스위치입니다.', + 'settings.tokenUsage.search': '검색 결과', + 'settings.tokenUsage.searchDesc': + 'grep/검색 출력을 관련성 순으로 정렬하고 상위 일치 항목을 유지합니다.', + 'settings.tokenUsage.code': '소스 코드', + 'settings.tokenUsage.codeDesc': '시그니처는 유지하고 함수 본문은 접습니다.', + 'settings.tokenUsage.html': 'HTML', + 'settings.tokenUsage.htmlDesc': '마크업을 제거하여 읽기 쉬운 텍스트로 만듭니다.', + 'settings.tokenUsage.ml': 'ML 텍스트 압축기', + 'settings.tokenUsage.mlDesc': '일반 텍스트용 로컬 ModernBERT 모델(Python 런타임 필요).', + 'settings.tokenUsage.ccrTitle': '캐시 및 복구(CCR)', + 'settings.tokenUsage.ccrDesc': + '압축된 원본이 캐시되어 에이전트가 필요할 때 전체 텍스트를 가져올 수 있습니다.', + 'settings.tokenUsage.ccrEnabled': '복구를 위해 원본 유지', + 'settings.tokenUsage.ccrEnabledDesc': + '원본을 오프로드하고 검색 표식을 추가하여 아무것도 손실되지 않도록 합니다.', + 'settings.tokenUsage.ccrMinTokens': '캐시할 최소 크기', + 'settings.tokenUsage.ccrMinTokensDesc': + '이 토큰 수 이상으로 추정되는 결과만 캐시하고 손실 압축합니다.', + 'settings.tokenUsage.tokensUnit': '토큰', + 'settings.tokenUsage.ccrDisk': '캐시를 디스크에 유지', + 'settings.tokenUsage.ccrDiskDesc': '재시작 후에도 복구 가능한 원본을 유지합니다.', 'settings.dataSync.title': '데이터 동기화', 'settings.dataSync.menuDesc': '어시스턴트가 동기화하는 항목 — 소스, 최신성 및 상태', 'settings.dataSync.description': diff --git a/app/src/lib/i18n/pl.ts b/app/src/lib/i18n/pl.ts index 44403dafd..0faf451e2 100644 --- a/app/src/lib/i18n/pl.ts +++ b/app/src/lib/i18n/pl.ts @@ -286,6 +286,48 @@ const messages: TranslationMap = { 'Analiza grafu pamięci — diagram, centralność, spójność, powiązania, świeżość, oś czasu, ścieżki i przestrzenie nazw', 'settings.buildInfo.title': 'Informacje o kompilacji/wersji', 'settings.buildInfo.menuDesc': 'Kompilacja aplikacji, wersja i szczegóły połączenia z rdzeniem', + 'settings.tokenUsage.title': 'Tokeny i koszt', + 'settings.tokenUsage.menuDesc': 'Ustawienia kompresji oraz ile tokenów i dolarów zaoszczędziły', + 'settings.tokenUsage.saving': 'Zapisywanie…', + 'settings.tokenUsage.saved': 'Zapisano', + 'settings.tokenUsage.savingsTitle': 'Oszczędności', + 'settings.tokenUsage.attributedTo': 'Koszt wyceniony dla {model}', + 'settings.tokenUsage.tokensSaved': 'Zaoszczędzone tokeny', + 'settings.tokenUsage.costSaved': 'Zaoszczędzony koszt', + 'settings.tokenUsage.cacheOccupancy': 'Oryginały w pamięci podręcznej', + 'settings.tokenUsage.compactions': 'Kompaktowania', + 'settings.tokenUsage.overEvents': 'w ciągu {count} kompaktowań', + 'settings.tokenUsage.byCompressor': 'Według kompresora', + 'settings.tokenUsage.refresh': 'Odśwież', + 'settings.tokenUsage.reset': 'Resetuj statystyki', + 'settings.tokenUsage.compressionTitle': 'Kompresja', + 'settings.tokenUsage.compressionDesc': + 'Kompaktowanie dużych wyników narzędzi z uwzględnieniem treści, zanim trafią do kontekstu modelu.', + 'settings.tokenUsage.routerEnabled': 'Włącz kompresję', + 'settings.tokenUsage.routerEnabledDesc': 'Główny przełącznik routera treści.', + 'settings.tokenUsage.search': 'Wyniki wyszukiwania', + 'settings.tokenUsage.searchDesc': + 'Sortuj wynik grep/wyszukiwania według trafności, zachowując najlepsze dopasowania.', + 'settings.tokenUsage.code': 'Kod źródłowy', + 'settings.tokenUsage.codeDesc': 'Zachowaj sygnatury, zwiń ciała funkcji.', + 'settings.tokenUsage.html': 'HTML', + 'settings.tokenUsage.htmlDesc': 'Usuń znaczniki, aby uzyskać czytelny tekst.', + 'settings.tokenUsage.ml': 'Kompresor tekstu ML', + 'settings.tokenUsage.mlDesc': + 'Lokalny model ModernBERT dla zwykłego tekstu (wymaga środowiska uruchomieniowego Python).', + 'settings.tokenUsage.ccrTitle': 'Pamięć podręczna i odzyskiwanie (CCR)', + 'settings.tokenUsage.ccrDesc': + 'Skompaktowane oryginały są przechowywane w pamięci podręcznej, aby agent mógł pobrać pełny tekst na żądanie.', + 'settings.tokenUsage.ccrEnabled': 'Zachowaj oryginały do odzyskania', + 'settings.tokenUsage.ccrEnabledDesc': + 'Przenieś oryginał i dodaj znacznik pobierania, aby nic nie zostało utracone.', + 'settings.tokenUsage.ccrMinTokens': 'Minimalny rozmiar do zapisania w pamięci podręcznej', + 'settings.tokenUsage.ccrMinTokensDesc': + 'Zapisuj w pamięci podręcznej i stratnie kompaktuj tylko wyniki szacowane na tę liczbę tokenów lub więcej.', + 'settings.tokenUsage.tokensUnit': 'tokeny', + 'settings.tokenUsage.ccrDisk': 'Zachowaj pamięć podręczną na dysku', + 'settings.tokenUsage.ccrDiskDesc': + 'Zachowuj możliwe do odzyskania oryginały między ponownymi uruchomieniami.', 'settings.dataSync.title': 'Synchronizacja danych', 'settings.dataSync.menuDesc': 'Co synchronizuje Twój asystent — źródła, świeżość i status', 'settings.dataSync.description': diff --git a/app/src/lib/i18n/pt.ts b/app/src/lib/i18n/pt.ts index 9996cbd5d..5ff33dd42 100644 --- a/app/src/lib/i18n/pt.ts +++ b/app/src/lib/i18n/pt.ts @@ -289,6 +289,48 @@ const messages: TranslationMap = { 'Análise do grafo de memória — diagrama, centralidade, coesão, associações, atualidade, linha do tempo, caminhos e namespaces', 'settings.buildInfo.title': 'Informações de build/versão', 'settings.buildInfo.menuDesc': 'Build do app, versão e detalhes de conexão do núcleo', + 'settings.tokenUsage.title': 'Tokens e custo', + 'settings.tokenUsage.menuDesc': + 'Configurações de compressão e quantos tokens e dólares elas economizaram', + 'settings.tokenUsage.saving': 'Salvando…', + 'settings.tokenUsage.saved': 'Salvo', + 'settings.tokenUsage.savingsTitle': 'Economias', + 'settings.tokenUsage.attributedTo': 'Custo calculado para {model}', + 'settings.tokenUsage.tokensSaved': 'Tokens economizados', + 'settings.tokenUsage.costSaved': 'Custo economizado', + 'settings.tokenUsage.cacheOccupancy': 'Originais em cache', + 'settings.tokenUsage.compactions': 'Compactações', + 'settings.tokenUsage.overEvents': 'ao longo de {count} compactações', + 'settings.tokenUsage.byCompressor': 'Por compressor', + 'settings.tokenUsage.refresh': 'Atualizar', + 'settings.tokenUsage.reset': 'Redefinir estatísticas', + 'settings.tokenUsage.compressionTitle': 'Compressão', + 'settings.tokenUsage.compressionDesc': + 'Compactação ciente do conteúdo de grandes saídas de ferramentas antes de entrarem no contexto do modelo.', + 'settings.tokenUsage.routerEnabled': 'Ativar compressão', + 'settings.tokenUsage.routerEnabledDesc': 'Interruptor principal do roteador de conteúdo.', + 'settings.tokenUsage.search': 'Resultados da busca', + 'settings.tokenUsage.searchDesc': + 'Ordenar a saída de grep/busca por relevância, mantendo as melhores correspondências.', + 'settings.tokenUsage.code': 'Código-fonte', + 'settings.tokenUsage.codeDesc': 'Manter as assinaturas, recolher os corpos das funções.', + 'settings.tokenUsage.html': 'HTML', + 'settings.tokenUsage.htmlDesc': 'Remover a marcação para obter texto legível.', + 'settings.tokenUsage.ml': 'Compressor de texto por ML', + 'settings.tokenUsage.mlDesc': + 'Modelo ModernBERT local para texto simples (requer ambiente de execução Python).', + 'settings.tokenUsage.ccrTitle': 'Cache e recuperação (CCR)', + 'settings.tokenUsage.ccrDesc': + 'Os originais compactados ficam em cache para que o agente possa buscar o texto completo sob demanda.', + 'settings.tokenUsage.ccrEnabled': 'Manter originais para recuperação', + 'settings.tokenUsage.ccrEnabledDesc': + 'Descarregar o original e adicionar um marcador de recuperação para que nada se perca.', + 'settings.tokenUsage.ccrMinTokens': 'Tamanho mínimo para armazenar em cache', + 'settings.tokenUsage.ccrMinTokensDesc': + 'Armazenar em cache e compactar com perdas apenas os resultados estimados nesse número de tokens ou mais.', + 'settings.tokenUsage.tokensUnit': 'tokens', + 'settings.tokenUsage.ccrDisk': 'Persistir cache no disco', + 'settings.tokenUsage.ccrDiskDesc': 'Manter originais recuperáveis entre reinicializações.', 'settings.dataSync.title': 'Sincronização de dados', 'settings.dataSync.menuDesc': 'O que o seu assistente sincroniza — fontes, atualidade e status', 'settings.dataSync.description': diff --git a/app/src/lib/i18n/ru.ts b/app/src/lib/i18n/ru.ts index cc900831b..30081a0ee 100644 --- a/app/src/lib/i18n/ru.ts +++ b/app/src/lib/i18n/ru.ts @@ -288,6 +288,47 @@ const messages: TranslationMap = { 'Анализ графа памяти — диаграмма, центральность, связность, ассоциации, свежесть, временная шкала, пути и пространства имён', 'settings.buildInfo.title': 'Сведения о сборке/версии', 'settings.buildInfo.menuDesc': 'Сборка приложения, версия и сведения о подключении ядра', + 'settings.tokenUsage.title': 'Токены и стоимость', + 'settings.tokenUsage.menuDesc': 'Настройки сжатия и сколько токенов и долларов они сэкономили', + 'settings.tokenUsage.saving': 'Сохранение…', + 'settings.tokenUsage.saved': 'Сохранено', + 'settings.tokenUsage.savingsTitle': 'Экономия', + 'settings.tokenUsage.attributedTo': 'Стоимость рассчитана для {model}', + 'settings.tokenUsage.tokensSaved': 'Сэкономлено токенов', + 'settings.tokenUsage.costSaved': 'Сэкономлено средств', + 'settings.tokenUsage.cacheOccupancy': 'Кэшированные оригиналы', + 'settings.tokenUsage.compactions': 'Сжатия', + 'settings.tokenUsage.overEvents': 'за {count} сжатий', + 'settings.tokenUsage.byCompressor': 'По компрессору', + 'settings.tokenUsage.refresh': 'Обновить', + 'settings.tokenUsage.reset': 'Сбросить статистику', + 'settings.tokenUsage.compressionTitle': 'Сжатие', + 'settings.tokenUsage.compressionDesc': + 'Сжатие больших выводов инструментов с учётом содержимого до их попадания в контекст модели.', + 'settings.tokenUsage.routerEnabled': 'Включить сжатие', + 'settings.tokenUsage.routerEnabledDesc': 'Главный переключатель маршрутизатора содержимого.', + 'settings.tokenUsage.search': 'Результаты поиска', + 'settings.tokenUsage.searchDesc': + 'Ранжировать вывод grep/поиска по релевантности, сохраняя лучшие совпадения.', + 'settings.tokenUsage.code': 'Исходный код', + 'settings.tokenUsage.codeDesc': 'Сохранять сигнатуры, сворачивать тела функций.', + 'settings.tokenUsage.html': 'HTML', + 'settings.tokenUsage.htmlDesc': 'Удалить разметку до читаемого текста.', + 'settings.tokenUsage.ml': 'ML-компрессор текста', + 'settings.tokenUsage.mlDesc': + 'Локальная модель ModernBERT для обычного текста (требуется среда выполнения Python).', + 'settings.tokenUsage.ccrTitle': 'Кэш и восстановление (CCR)', + 'settings.tokenUsage.ccrDesc': + 'Сжатые оригиналы кэшируются, чтобы агент мог получить полный текст по запросу.', + 'settings.tokenUsage.ccrEnabled': 'Сохранять оригиналы для восстановления', + 'settings.tokenUsage.ccrEnabledDesc': + 'Выгрузить оригинал и добавить маркер извлечения, чтобы ничего не потерялось.', + 'settings.tokenUsage.ccrMinTokens': 'Минимальный размер для кэширования', + 'settings.tokenUsage.ccrMinTokensDesc': + 'Кэшировать и сжимать с потерями только результаты, оценённые в это количество токенов или больше.', + 'settings.tokenUsage.tokensUnit': 'токенов', + 'settings.tokenUsage.ccrDisk': 'Сохранять кэш на диск', + 'settings.tokenUsage.ccrDiskDesc': 'Сохранять восстанавливаемые оригиналы между перезапусками.', 'settings.dataSync.title': 'Синхронизация данных', 'settings.dataSync.menuDesc': 'Что синхронизирует ваш ассистент — источники, свежесть и статус', 'settings.dataSync.description': diff --git a/app/src/lib/i18n/zh-CN.ts b/app/src/lib/i18n/zh-CN.ts index 2cf2a52a0..cd6a1ea4d 100644 --- a/app/src/lib/i18n/zh-CN.ts +++ b/app/src/lib/i18n/zh-CN.ts @@ -280,6 +280,41 @@ const messages: TranslationMap = { '内存图分析 — 关系图、中心性、内聚性、关联、新鲜度、时间线、路径和命名空间', 'settings.buildInfo.title': '构建/版本信息', 'settings.buildInfo.menuDesc': '应用构建、版本和核心连接详情', + 'settings.tokenUsage.title': '令牌与成本', + 'settings.tokenUsage.menuDesc': '压缩设置以及它们已节省的令牌和美元数量', + 'settings.tokenUsage.saving': '正在保存…', + 'settings.tokenUsage.saved': '已保存', + 'settings.tokenUsage.savingsTitle': '节省', + 'settings.tokenUsage.attributedTo': '按 {model} 计价的成本', + 'settings.tokenUsage.tokensSaved': '节省的令牌', + 'settings.tokenUsage.costSaved': '节省的成本', + 'settings.tokenUsage.cacheOccupancy': '已缓存的原文', + 'settings.tokenUsage.compactions': '压缩次数', + 'settings.tokenUsage.overEvents': '在 {count} 次压缩中', + 'settings.tokenUsage.byCompressor': '按压缩器', + 'settings.tokenUsage.refresh': '刷新', + 'settings.tokenUsage.reset': '重置统计', + 'settings.tokenUsage.compressionTitle': '压缩', + 'settings.tokenUsage.compressionDesc': '在大型工具输出进入模型上下文之前,对其进行内容感知压缩。', + 'settings.tokenUsage.routerEnabled': '启用压缩', + 'settings.tokenUsage.routerEnabledDesc': '内容路由器的总开关。', + 'settings.tokenUsage.search': '搜索结果', + 'settings.tokenUsage.searchDesc': '按相关性对 grep/搜索输出进行排序,保留最佳匹配项。', + 'settings.tokenUsage.code': '源代码', + 'settings.tokenUsage.codeDesc': '保留签名,折叠函数主体。', + 'settings.tokenUsage.html': 'HTML', + 'settings.tokenUsage.htmlDesc': '去除标记,转为可读文本。', + 'settings.tokenUsage.ml': 'ML 文本压缩器', + 'settings.tokenUsage.mlDesc': '用于纯文本的本地 ModernBERT 模型(需要 Python 运行时)。', + 'settings.tokenUsage.ccrTitle': '缓存与恢复(CCR)', + 'settings.tokenUsage.ccrDesc': '压缩后的原文会被缓存,以便代理可按需获取完整文本。', + 'settings.tokenUsage.ccrEnabled': '保留原文以便恢复', + 'settings.tokenUsage.ccrEnabledDesc': '卸载原文并添加检索标记,确保不丢失任何内容。', + 'settings.tokenUsage.ccrMinTokens': '缓存的最小大小', + 'settings.tokenUsage.ccrMinTokensDesc': '仅缓存并有损压缩估算达到或超过此令牌数量的结果。', + 'settings.tokenUsage.tokensUnit': '令牌', + 'settings.tokenUsage.ccrDisk': '将缓存持久化到磁盘', + 'settings.tokenUsage.ccrDiskDesc': '在重启之间保留可恢复的原文。', 'settings.dataSync.title': '数据同步', 'settings.dataSync.menuDesc': '助手同步的内容 — 来源、新鲜度和状态', 'settings.dataSync.description': diff --git a/app/src/utils/tauriCommands/tokenjuice.ts b/app/src/utils/tauriCommands/tokenjuice.ts new file mode 100644 index 000000000..3da4dbad7 --- /dev/null +++ b/app/src/utils/tauriCommands/tokenjuice.ts @@ -0,0 +1,77 @@ +/** + * TokenJuice content-router client. + * + * Thin wrapper around the `openhuman.tokenjuice_*` JSON-RPC methods exposed by + * the Rust core (`src/openhuman/tokenjuice/schemas.rs`): read/update the + * `[tokenjuice]` settings block and read/reset compaction savings statistics. + */ +import { callCoreRpc } from '../../services/coreRpcClient'; + +/** The `[tokenjuice]` config block (snake_case, matching the Rust config keys). */ +export interface TokenjuiceSettings { + router_enabled: boolean; + ccr_enabled: boolean; + ccr_disk_enabled: boolean; + max_cache_entries: number; + max_cache_bytes: number; + ccr_ttl_secs: number | null; + min_bytes_to_compress: number; + ccr_min_tokens: number; + search_enabled: boolean; + code_enabled: boolean; + html_enabled: boolean; + ml_compression_enabled: boolean; + ml_model_id: string; + ml_target_ratio: number; + ml_sidecar_idle_timeout_secs: number; + ml_max_input_chars: number; + ml_device: string; +} + +/** Partial update — only present fields are changed. */ +export type TokenjuiceSettingsPatch = Partial; + +export interface SavingsBucket { + events: number; + originalTokens: number; + compactedTokens: number; + tokensSaved: number; + costSavedUsd: number; +} + +export interface SavingsStats { + attributionModel: string; + total: SavingsBucket; + byModel: Record; + byCompressor: Record; + cache: { entries: number; bytes: number }; +} + +export async function getTokenjuiceSettings(): Promise { + const res = await callCoreRpc<{ settings: TokenjuiceSettings }>({ + method: 'openhuman.tokenjuice_settings_get', + params: {}, + }); + return res.settings; +} + +export async function updateTokenjuiceSettings( + patch: TokenjuiceSettingsPatch +): Promise { + const res = await callCoreRpc<{ settings: TokenjuiceSettings }>({ + method: 'openhuman.tokenjuice_settings_update', + params: { patch }, + }); + return res.settings; +} + +export async function getTokenjuiceSavings(): Promise { + return await callCoreRpc({ + method: 'openhuman.tokenjuice_savings_stats', + params: {}, + }); +} + +export async function resetTokenjuiceSavings(): Promise { + await callCoreRpc<{ ok: boolean }>({ method: 'openhuman.tokenjuice_savings_reset', params: {} }); +} diff --git a/gitbooks/developing/architecture/agent-harness.md b/gitbooks/developing/architecture/agent-harness.md index 7e68466f7..30ebdfc2c 100644 --- a/gitbooks/developing/architecture/agent-harness.md +++ b/gitbooks/developing/architecture/agent-harness.md @@ -117,6 +117,20 @@ Some tool calls return enormous payloads - a Composio action dumping 200 KB of J When a tool result exceeds the summarizer's threshold, it gets routed through a dedicated `summarizer` sub-agent before entering the parent's history. The summarizer compresses the payload per an extraction contract that preserves identifiers and key facts, and the parent agent only sees the compressed summary. Hard truncation remains the backstop downstream when summarization fails or the payload is so absurdly large that paying for an LLM call on it makes no economic sense. +### TokenJuice - content-aware tool-output compaction (Stage 1a) + +Before a fresh tool result enters history (and ahead of the byte-budget backstop), it passes through the **TokenJuice content router** (`src/openhuman/tokenjuice/`). Inspired by Headroom, the router *detects the content kind* (JSON, code, log, search, diff, HTML, plain text) from the bytes and/or a hint derived from the tool name and arguments, then dispatches to a specialised compressor: + +* **JSON** → SmartCrusher: array-of-objects → table (each key once), preserving rows that carry errors or numeric outliers. +* **Code** → tree-sitter (Rust/TS/JS/Python) signature keeper that collapses function bodies; brace-depth heuristic fallback. +* **Log** → the 100-rule engine for *command* output (git/cargo/npm/…), signal-based keep-failures otherwise. +* **Search** → relevance-ranked top-K matches per file with a `+N more` tally. +* **Diff** → keep changed hunks, collapse unchanged context, summarise lockfile hunks. +* **HTML** → strip markup to readable text. +* **Plain text** → the opt-in Python/ML "Kompress" compressor (ModernBERT), or pass-through. + +Every lossy compression offloads the original to the **CCR (Compress-Cache-Retrieve)** store behind a `⟦tj:⟧` marker, so compaction is effectively lossless: the agent calls `tokenjuice_retrieve` (token + optional byte/line range) to fetch the full original on demand. The same engine is exposed as a universal `compress_content(content, hint, opts)` for any large payload (file reads, web fetches), and as read-only `tokenjuice.*` debug RPCs. Configured via the `[tokenjuice]` block / `OPENHUMAN_TOKENJUICE_*` env. The ML (Kompress) path runs as a `kompress` backend of the shared [`runtime_python_server`](../../../src/openhuman/runtime_python_server/) (torch + ModernBERT pip-installed at runtime), gated by the `ml_compression_enabled` flag and degrading gracefully to a native compressor when the Python runtime is unavailable. + ### Self-healing for missing commands When the code-executor sub-agent runs a shell command and the runtime answers "command not found", a self-healing interceptor catches the error, spawns a `ToolMaker` sub-agent to write a polyfill script for the missing command, and retries the original call. There's a per-command attempt cap so a genuinely impossible command can't loop forever. diff --git a/src/core/all.rs b/src/core/all.rs index 75dfc4552..6eff82fd5 100644 --- a/src/core/all.rs +++ b/src/core/all.rs @@ -284,6 +284,8 @@ fn build_registered_controllers() -> Vec { controllers.extend(crate::openhuman::learning::all_learning_registered_controllers()); // Conversation thread and message management controllers.extend(crate::openhuman::threads::all_threads_registered_controllers()); + // TokenJuice content-router debug controllers (detect / compress / cache_stats / retrieve) + controllers.extend(crate::openhuman::tokenjuice::all_tokenjuice_registered_controllers()); // Per-thread todo list (agent task board CRUD over RPC) controllers.extend(crate::openhuman::todos::all_todos_registered_controllers()); // Embedded webview native notifications @@ -443,6 +445,8 @@ fn build_declared_controller_schemas() -> Vec { schemas.extend(crate::openhuman::learning::all_learning_controller_schemas()); // Conversation thread and message management schemas.extend(crate::openhuman::threads::all_threads_controller_schemas()); + // TokenJuice content-router debug controllers + schemas.extend(crate::openhuman::tokenjuice::all_tokenjuice_controller_schemas()); // Per-thread todo list (agent task board CRUD over RPC) schemas.extend(crate::openhuman::todos::all_todos_controller_schemas()); // Embedded webview native notifications diff --git a/src/core/jsonrpc.rs b/src/core/jsonrpc.rs index e72637840..2239fafca 100644 --- a/src/core/jsonrpc.rs +++ b/src/core/jsonrpc.rs @@ -2246,6 +2246,12 @@ fn register_domain_subscribers( // initial throttle decision on battery-powered hosts). crate::openhuman::scheduler_gate::init_global(&config); + // Install the TokenJuice content-router runtime config (compressor + // toggles + CCR cache limits + optional on-disk tier). Compaction runs + // on every agent's tool output, so this must be set before any agent + // loop executes a tool. + crate::openhuman::tokenjuice::install_from_config(&config); + // Seed the scheduler-gate signed-out override from the on-disk // session. Without this, a sidecar that boots with no stored JWT // would happily spin up cron / channel loops and fire LLM requests diff --git a/src/openhuman/agent/harness/compaction/demo.rs b/src/openhuman/agent/harness/compaction/demo.rs deleted file mode 100644 index b00b9098a..000000000 --- a/src/openhuman/agent/harness/compaction/demo.rs +++ /dev/null @@ -1,179 +0,0 @@ -//! Human-eyeball demo of compaction on dummy data. -//! -//! Unlike [`super::measure`] (which asserts token deltas), this prints the -//! actual BEFORE → AFTER text for each content type so you can see exactly -//! what the model would receive. Run it with output: -//! -//! ```text -//! cargo test -p openhuman --lib compaction::demo -- --nocapture -//! ``` - -#![cfg(test)] - -use super::super::token_budget::estimate_tokens; -use super::{compact_tool_output, store}; -use std::fmt::Write as _; - -/// Print a labelled before/after block. Input is shown head+tail so the -/// terminal stays readable; the compacted output is shown in full. -fn show(title: &str, tool: &str, raw: &str) -> String { - let before = estimate_tokens(raw); - let out = compact_tool_output(raw.to_string(), tool, true); - let after = estimate_tokens(&out); - let pct = if before == 0 { - 0.0 - } else { - 100.0 * (1.0 - after as f64 / before as f64) - }; - - println!("\n══════════════════════════════════════════════════════════════"); - println!("▶ {title} (tool={tool})"); - println!( - " {} bytes / ~{} tok → {} bytes / ~{} tok ({:.0}% saved)", - raw.len(), - before, - out.len(), - after, - pct - ); - println!("─── INPUT (first 8 + last 2 lines) ──────────────────────────"); - print_head_tail(raw, 8, 2); - println!("─── COMPACTED (what the model sees) ─────────────────────────"); - println!("{out}"); - out -} - -fn print_head_tail(text: &str, head: usize, tail: usize) { - let lines: Vec<&str> = text.lines().collect(); - if lines.len() <= head + tail { - for l in &lines { - println!("{l}"); - } - return; - } - for l in &lines[..head] { - println!("{l}"); - } - println!(" … {} lines …", lines.len() - head - tail); - for l in &lines[lines.len() - tail..] { - println!("{l}"); - } -} - -#[test] -fn demo_all_content_types() { - // 1 ── grep / code search is INTENTIONALLY NOT compacted (completeness - // tool). Demonstrate that it passes through untouched even when large. - let mut grep = String::from("48 match(es); scanned 4 file(s)\n"); - for f in 0..4 { - for i in 1..=12 { - let _ = writeln!( - grep, - "src/feature_{f}/service.rs:{i}: let outcome = dispatch_request(&context, request_payload_{i})?;" - ); - } - } - let grep_out = compact_tool_output(grep.clone(), "grep", true); - println!("\n══════════════════════════════════════════════════════════════"); - println!("▶ Code search (grep) — intentionally NOT compacted"); - println!( - " {} bytes → {} bytes (unchanged: {})", - grep.len(), - grep_out.len(), - grep_out == grep - ); - - // 2 ── build/test log: noise + warnings + a real error + summary. - let mut log = String::new(); - for i in 0..120 { - let _ = writeln!(log, " Compiling some_dependency_{i} v1.{i}.0"); - } - for i in 0..30 { - let _ = writeln!( - log, - "warning: unused variable `scratch` (occurrence {i}) at src/util.rs" - ); - } - let _ = writeln!(log, "error[E0382]: borrow of moved value `session`"); - let _ = writeln!(log, " --> src/agent/loop.rs:88:21"); - let _ = writeln!(log, " |"); - let _ = writeln!( - log, - " = note: move occurs because `session` has type `Session`" - ); - let _ = writeln!(log, "error: aborting due to previous error"); - let _ = writeln!(log, "test result: FAILED. 142 passed; 1 failed; 3 ignored"); - show("Build/test log (run_tests)", "run_tests", &log); - - // 3 ── git diff: small change wrapped in lots of unchanged context. - // (Fixtures are sized above MIN_BYTES_TO_COMPRESS = 2048 so the - // compressors actually engage — smaller outputs pass through untouched.) - let mut diff = - String::from("diff --git a/src/router.rs b/src/router.rs\n@@ -10,84 +10,85 @@\n"); - for i in 0..40 { - let _ = writeln!( - diff, - " // unchanged surrounding context line {i} carried along by the diff" - ); - } - let _ = writeln!(diff, "- route.register(\"/old\", old_handler);"); - let _ = writeln!(diff, "+ route.register(\"/new\", new_handler);"); - let _ = writeln!(diff, "+ route.register(\"/extra\", extra_handler);"); - for i in 0..40 { - let _ = writeln!( - diff, - " // trailing unchanged context line {i} after the change" - ); - } - show("Git diff (read_diff)", "read_diff", &diff); - - // 4 ── JSON list, small enough to stay lossless (under the row-drop - // threshold) but big enough to clear the byte floor → lossless table. - let mut small = Vec::new(); - for i in 0..30 { - small.push(format!( - r#"{{"id":{i},"name":"widget_{i}","status":"in_stock","warehouse":"east-1","sku":"WH-{i:04}"}}"# - )); - } - show( - "JSON list — 30 rows (lossless table)", - "list_inventory", - &format!("[{}]", small.join(",")), - ); - - // 5 ── JSON list, large (row-drop + CCR retrieval marker). - let mut big = Vec::new(); - for i in 0..80 { - big.push(format!( - r#"{{"id":{i},"name":"account_{i}","email":"a{i}@example.com","tier":"gold"}}"# - )); - } - let big_input = format!("[{}]", big.join(",")); - let out = show( - "JSON list — 80 rows (row-drop + CCR)", - "list_accounts", - &big_input, - ); - - // ── Demonstrate the reversibility: pull the hash from the marker and - // retrieve the full original via the CCR store (what the - // `retrieve_tool_output` tool does). - if let Some(marker) = out.lines().find(|l| l.contains("retrieve_tool_output(")) { - let hash = marker - .split("retrieve_tool_output(\"") - .nth(1) - .and_then(|s| s.split('"').next()) - .unwrap_or(""); - println!("\n─── CCR RETRIEVE round-trip ─────────────────────────────────"); - println!(" marker hash = {hash}"); - match store::retrieve(hash) { - Some(original) => println!( - " retrieve_tool_output(\"{hash}\") → {} bytes restored (matches original: {})", - original.len(), - original == big_input - ), - None => println!(" (evicted)"), - } - } - println!("\n══════════════════════════════════════════════════════════════\n"); -} diff --git a/src/openhuman/agent/harness/compaction/detect.rs b/src/openhuman/agent/harness/compaction/detect.rs deleted file mode 100644 index bb9e63882..000000000 --- a/src/openhuman/agent/harness/compaction/detect.rs +++ /dev/null @@ -1,301 +0,0 @@ -//! Content-type detection and tool-name routing for compaction. -//! -//! Clean-room port of the routing behavior in headroom's `content_detector` -//! (Apache-2.0): cheap structural heuristics that classify a tool-output blob -//! so [`super::compact_tool_output`] can pick the right compressor. The tool -//! name gives a strong prior ([`ContentHint`]); detection validates/overrides -//! it for the `Auto` case. - -/// The kind of content a tool produced, after detection. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ContentType { - /// grep / ripgrep style `path:line:content` matches. - Search, - /// Build / test / lint output (compiler logs, test runners). - Log, - /// Unified git diff / patch. - Diff, - /// JSON array of objects (handled by the Phase-2 crusher). - JsonArray, - /// Nothing we compress — pass through unchanged. - PlainText, -} - -/// A prior derived from the tool name at the call site, so we don't have to -/// detect from scratch for the common case. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ContentHint { - Search, - Log, - Diff, - Json, - Auto, -} - -/// Map a tool name to its content prior. Mirrors the target-tool table in the -/// compaction plan. Unknown tools fall through to [`ContentHint::Auto`]. -pub fn hint_for_tool(tool_name: &str) -> ContentHint { - match tool_name { - "grep" | "glob_search" => ContentHint::Search, - // Tools whose output is *reliably* a build/test/lint log. - "run_tests" | "run_linter" | "npm_exec" | "node_exec" | "install_tool" | "lsp" => { - ContentHint::Log - } - "read_diff" | "git_operations" => ContentHint::Diff, - // `shell` is generic — its output is often NOT a log (find, seq, cat, - // generated CSV, a script printing a list). Route it through detection - // so only output that actually looks like a log gets log-compressed; - // anything else passes through. See the log compressor's no-signal guard. - _ => ContentHint::Auto, - } -} - -/// Resolve the content type to compress as. A non-`Auto` hint is trusted -/// unless detection strongly disagrees (e.g. a `shell` call that printed a -/// diff). For `Auto`, run full detection. -pub fn resolve(hint: ContentHint, content: &str) -> ContentType { - match hint { - ContentHint::Search => { - // Trust the hint, but if the body is clearly a diff/log prefer that. - if looks_like_diff(content) { - ContentType::Diff - } else { - ContentType::Search - } - } - ContentHint::Log => { - if looks_like_diff(content) { - ContentType::Diff - } else if search_line_ratio(content) >= 0.6 { - ContentType::Search - } else { - ContentType::Log - } - } - ContentHint::Diff => { - if looks_like_diff(content) { - ContentType::Diff - } else { - detect(content) - } - } - ContentHint::Json => { - if looks_like_json_array(content) { - ContentType::JsonArray - } else { - detect(content) - } - } - ContentHint::Auto => detect(content), - } -} - -/// Full structural detection, in priority order: JSON → diff → search → log → -/// plain text. Thresholds mirror headroom's detector. -pub fn detect(content: &str) -> ContentType { - let trimmed = content.trim_start(); - if trimmed.is_empty() { - return ContentType::PlainText; - } - if looks_like_json_array(content) { - return ContentType::JsonArray; - } - if looks_like_diff(content) { - return ContentType::Diff; - } - if search_line_ratio(content) >= 0.6 { - return ContentType::Search; - } - if log_line_ratio(content) >= 0.5 { - return ContentType::Log; - } - ContentType::PlainText -} - -/// True if `content` parses as a JSON array of objects (the crusher's input). -pub fn looks_like_json_array(content: &str) -> bool { - let trimmed = content.trim_start(); - if !trimmed.starts_with('[') { - return false; - } - match serde_json::from_str::(trimmed) { - Ok(serde_json::Value::Array(items)) => { - items.len() >= 2 && items.iter().any(|v| v.is_object()) - } - _ => false, - } -} - -/// True if `content` looks like a unified diff: a `diff --git` header or at -/// least one hunk header (`@@ ... @@`). -pub fn looks_like_diff(content: &str) -> bool { - let mut hunks = 0usize; - for line in content.lines().take(400) { - if line.starts_with("diff --git ") || line.starts_with("Index: ") { - return true; - } - if line.starts_with("@@ ") && line[3..].contains("@@") { - hunks += 1; - if hunks >= 1 { - return true; - } - } - } - false -} - -/// Fraction of non-empty lines that look like `path:line:...` search hits. -/// Handles a leading Windows drive letter (`C:\...`) so those paths aren't -/// mistaken for the line-number separator. -fn search_line_ratio(content: &str) -> f32 { - let mut total = 0usize; - let mut hits = 0usize; - for line in content.lines().take(2000) { - if line.trim().is_empty() { - continue; - } - total += 1; - if parse_search_line(line).is_some() { - hits += 1; - } - } - if total == 0 { - 0.0 - } else { - hits as f32 / total as f32 - } -} - -/// Fraction of lines carrying an error/warning indicator — the log signal. -fn log_line_ratio(content: &str) -> f32 { - use super::signals::{severity, Severity}; - let mut total = 0usize; - let mut hits = 0usize; - for line in content.lines().take(2000) { - if line.trim().is_empty() { - continue; - } - total += 1; - if severity(line) != Severity::Other { - hits += 1; - } - } - if total == 0 { - 0.0 - } else { - hits as f32 / total as f32 - } -} - -/// Parse a single grep/ripgrep line into `(path, line_number, content)`. -/// -/// Anchors on the earliest `::` marker, skipping a leading Windows -/// drive prefix (`C:`), so paths may contain `:` (drive), `-`, and spaces. -/// Returns `None` for context lines (`rg` `-` separators) and non-matches. -pub fn parse_search_line(line: &str) -> Option<(&str, u64, &str)> { - // Skip an optional Windows drive prefix like `C:` before scanning for the - // line-number marker. - let scan_from = if line.len() >= 2 { - let bytes = line.as_bytes(); - if bytes[0].is_ascii_alphabetic() && bytes[1] == b':' { - 2 - } else { - 0 - } - } else { - 0 - }; - - let rest = &line[scan_from..]; - // Find the first ':' that is followed by digits and then another ':'. - let mut search_start = 0usize; - while let Some(rel) = rest[search_start..].find(':') { - let colon = search_start + rel; - let after = &rest[colon + 1..]; - let digits_len = after.chars().take_while(|c| c.is_ascii_digit()).count(); - if digits_len > 0 && after.as_bytes().get(digits_len) == Some(&b':') { - let path = &line[..scan_from + colon]; - let num: u64 = after[..digits_len].parse().ok()?; - let body = &after[digits_len + 1..]; - if path.is_empty() { - return None; - } - return Some((path, num, body)); - } - search_start = colon + 1; - } - None -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn hints_map_known_tools() { - assert_eq!(hint_for_tool("grep"), ContentHint::Search); - assert_eq!(hint_for_tool("run_tests"), ContentHint::Log); - assert_eq!(hint_for_tool("read_diff"), ContentHint::Diff); - assert_eq!(hint_for_tool("file_read"), ContentHint::Auto); - // `shell` is generic → Auto (detection decides), not forced to Log. - assert_eq!(hint_for_tool("shell"), ContentHint::Auto); - } - - #[test] - fn detect_search_results() { - let c = - "src/main.rs:42:fn process() {\nsrc/lib.rs:7:pub use foo;\nsrc/x.rs:99: let y = 1;"; - assert_eq!(detect(c), ContentType::Search); - } - - #[test] - fn parse_unix_and_windows_paths() { - assert_eq!( - parse_search_line("src/main.rs:42:fn process() {"), - Some(("src/main.rs", 42, "fn process() {")) - ); - assert_eq!( - parse_search_line(r"C:\Users\me\a.rs:10:let x = 1;"), - Some((r"C:\Users\me\a.rs", 10, "let x = 1;")) - ); - // dashes in filename must survive - assert_eq!( - parse_search_line("pre-commit-config.yaml:3:foo"), - Some(("pre-commit-config.yaml", 3, "foo")) - ); - assert_eq!(parse_search_line("just a sentence"), None); - } - - #[test] - fn detect_diff() { - let c = "diff --git a/x.rs b/x.rs\n@@ -1,3 +1,4 @@\n+added\n-removed"; - assert_eq!(detect(c), ContentType::Diff); - } - - #[test] - fn detect_log() { - let c = - "Compiling foo\nwarning: unused\nerror[E0382]: borrow of moved value\nerror: aborting"; - assert_eq!(detect(c), ContentType::Log); - } - - #[test] - fn detect_json_array() { - let c = r#"[{"id":1,"name":"a"},{"id":2,"name":"b"}]"#; - assert_eq!(detect(c), ContentType::JsonArray); - } - - #[test] - fn plain_text_passes_through() { - assert_eq!( - detect("just some prose about a topic"), - ContentType::PlainText - ); - } - - #[test] - fn log_hint_with_search_body_routes_search() { - let body = "a.rs:1:x\nb.rs:2:y\nc.rs:3:z\nd.rs:4:w"; - assert_eq!(resolve(ContentHint::Log, body), ContentType::Search); - } -} diff --git a/src/openhuman/agent/harness/compaction/json_crusher.rs b/src/openhuman/agent/harness/compaction/json_crusher.rs deleted file mode 100644 index 6ecdfd88f..000000000 --- a/src/openhuman/agent/harness/compaction/json_crusher.rs +++ /dev/null @@ -1,264 +0,0 @@ -//! JSON-array crusher. -//! -//! Clean-room port of the *lossless* core of headroom's `SmartCrusher` -//! (Apache-2.0): an array of objects that repeat the same keys is the single -//! most common bloated tool output (API list responses, DB rows, search -//! manifests). Re-rendering it as a table emits each key **once** instead of -//! per row, dropping the repeated key names and JSON punctuation. -//! -//! Up to [`ROW_DROP_THRESHOLD`] rows every value is preserved (nested values -//! render as compact JSON in their cell); the array→table reformat changes only -//! layout, so it returns [`Compacted::reformatted`]. Above the threshold the -//! table is additionally **row-dropped** (head + tail kept) and returns -//! [`Compacted::lossy`]. Either way the caller (`compact_tool_output`) offloads -//! the full original to CCR behind a `retrieve_tool_output("")` footer, so -//! the agent can always fetch the exact original JSON back on demand. - -use super::Compacted; -use serde_json::Value; -use std::fmt::Write as _; - -/// Minimum rows before tabular rendering is worth the header overhead. -pub const MIN_ROWS: usize = 3; -/// Above this many rows the table is *also* row-dropped: head + tail rows are -/// kept and the full original is offloaded to CCR behind a retrieval marker. -pub const ROW_DROP_THRESHOLD: usize = 40; -/// Rows kept from the head when row-dropping. -pub const HEAD_ROWS: usize = 20; -/// Rows kept from the tail when row-dropping. -pub const TAIL_ROWS: usize = 10; - -/// Compress a JSON array-of-objects into a compact table. Returns `None` when -/// the content isn't a uniform-enough array of objects or wouldn't shrink. -/// -/// Lossless (only reformats) up to [`ROW_DROP_THRESHOLD`] rows; above it the -/// middle rows are dropped and the result is marked `lossy` so the caller -/// offloads the original to CCR behind the retrieval footer. -pub fn compress(content: &str) -> Option { - let value: Value = serde_json::from_str(content.trim()).ok()?; - let array = value.as_array()?; - if array.len() < MIN_ROWS { - return None; - } - // Every element must be an object for a clean table; mixed arrays bail. - if !array.iter().all(Value::is_object) { - return None; - } - - // Column order = first-seen key order across all rows (union, stable). - let mut columns: Vec = Vec::new(); - for item in array { - if let Some(obj) = item.as_object() { - for key in obj.keys() { - if !columns.iter().any(|c| c == key) { - columns.push(key.clone()); - } - } - } - } - if columns.len() < 2 { - return None; - } - - // Render every row's cells up front so we can choose full vs. row-dropped. - let mut rows: Vec = Vec::with_capacity(array.len()); - for item in array { - let obj = item.as_object()?; - let cells: Vec = columns - .iter() - .map(|col| match obj.get(col) { - // Distinguish a truly absent key (blank) from an explicit null - // (rendered as `null` by render_cell) so the view stays faithful. - None => String::new(), - Some(v) => render_cell(v), - }) - .collect(); - rows.push(cells.join(" | ")); - } - - let lossy = rows.len() > ROW_DROP_THRESHOLD; - let mut out = String::with_capacity(content.len()); - let _ = writeln!( - out, - "[json table: {} rows × {} cols · blank=absent key · exact original via retrieve footer]", - rows.len(), - columns.len() - ); - let _ = writeln!(out, "{}", columns.join(" | ")); - - if lossy { - // Keep head + tail; the caller offloads the full original to CCR and - // appends the retrieve_tool_output footer, so the dropped middle stays - // recoverable. The inline marker here just shows where rows were cut. - let dropped = rows.len() - HEAD_ROWS - TAIL_ROWS; - for row in rows.iter().take(HEAD_ROWS) { - let _ = writeln!(out, "{row}"); - } - let _ = writeln!(out, "[... {dropped} middle rows omitted ...]"); - for row in rows.iter().skip(rows.len() - TAIL_ROWS) { - let _ = writeln!(out, "{row}"); - } - } else { - for row in &rows { - let _ = writeln!(out, "{row}"); - } - } - - let out = out.trim_end().to_string(); - if out.len() >= content.len() { - return None; - } - log::debug!( - "[compaction][json] {} rows × {} cols, lossy={} ({} -> {} bytes)", - rows.len(), - columns.len(), - lossy, - content.len(), - out.len(), - ); - if lossy { - Some(Compacted::lossy(out)) - } else { - // All values preserved, but the array→table reformat changes layout - // (key order, quoting). Reported as `reformatted` so the caller still - // offloads the original — the agent can fetch exact JSON bytes back. - Some(Compacted::reformatted(out)) - } -} - -/// Render a single cell. Scalars print bare-ish (strings unquoted unless they -/// contain the column separator); nested values stay as compact JSON so the -/// table remains lossless. -fn render_cell(v: &Value) -> String { - match v { - Value::String(s) if !s.contains('|') && !s.contains('\n') => s.clone(), - Value::Bool(b) => b.to_string(), - Value::Number(n) => n.to_string(), - other => serde_json::to_string(other).unwrap_or_default(), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn crushes_uniform_array() { - let mut rows = Vec::new(); - for i in 0..20 { - rows.push(format!( - r#"{{"id":{i},"name":"item number {i}","status":"active","owner":"team-alpha"}}"# - )); - } - let input = format!("[{}]", rows.join(",")); - let out = compress(&input).expect("compresses").text; - // Header + key names appear once, not 20× (column order is whatever - // serde_json yields — don't assume insertion order here). - assert_eq!(out.matches("status").count(), 1, "{out}"); - for col in ["id", "name", "status", "owner"] { - assert!(out.lines().nth(1).unwrap().contains(col), "missing {col}"); - } - // Data preserved. - assert!(out.contains("item number 7")); - assert!(out.contains("19")); - assert!(out.len() < input.len(), "expected shrink"); - } - - #[test] - fn preserves_nested_values_losslessly() { - // Enough rows that the table beats the input even with the header. - let mut rows = Vec::new(); - for i in 0..8 { - rows.push(format!( - r#"{{"id":{i},"tags":["alpha","beta"],"meta":{{"k":{i},"label":"row{i}"}}}}"# - )); - } - let input = format!("[{}]", rows.join(",")); - let out = compress(&input).expect("compresses").text; - assert!(out.contains(r#"["alpha","beta"]"#), "{out}"); - assert!(out.contains(r#""label":"row3""#), "{out}"); - } - - #[test] - fn handles_missing_keys() { - // Enough rows with longish values that dropping repeated keys shrinks. - let mut rows = Vec::new(); - for i in 0..12 { - rows.push(format!( - r#"{{"alpha":{i},"bravo":"value string {i}","charlie":"another value {i}"}}"# - )); - } - rows.push(r#"{"alpha":99}"#.to_string()); // missing bravo/charlie - let input = format!("[{}]", rows.join(",")); - let out = compress(&input).expect("compresses").text; - let header = out.lines().nth(1).unwrap(); - for col in ["alpha", "bravo", "charlie"] { - assert!(header.contains(col), "header missing {col}: {header}"); - } - assert!(out.len() < input.len()); - } - - #[test] - fn large_array_row_drops_and_is_marked_lossy() { - let mut rows = Vec::new(); - for i in 0..200 { - rows.push(format!( - r#"{{"id":{i},"name":"record number {i}","status":"active","note":"some detail {i}"}}"# - )); - } - let input = format!("[{}]", rows.join(",")); - let c = compress(&input).expect("compresses"); - - // Marked lossy → the caller (compact_tool_output) offloads to CCR and - // appends the retrieve footer. The CCR round-trip is covered at the - // mod.rs level (lossy_outputs_are_recoverable). - assert!(c.lossy, "row-dropped output must be lossy"); - assert!(c.text.contains("middle rows omitted"), "{}", c.text); - - // Head + tail rows survive; the middle is dropped. - assert!(c.text.contains("record number 0"), "{}", c.text); - assert!(c.text.contains("record number 199"), "{}", c.text); - assert!( - !c.text.contains("record number 100"), - "middle should be dropped" - ); - assert!(c.text.len() < input.len()); - } - - #[test] - fn distinguishes_explicit_null_from_absent_key() { - // explicit null → "null"; absent key → blank. (Faithfulness: the two - // must not be conflated.) - let mut rows = Vec::new(); - for i in 0..10 { - rows.push(format!( - r#"{{"id":{i},"note":null,"tag":"long enough value to ensure shrink {i}"}}"# - )); - } - let input = format!("[{}]", rows.join(",")); - let c = compress(&input).expect("compresses"); - // A row with explicit null renders the literal "null" (not blank). - assert!(c.text.contains("null"), "{}", c.text); - } - - #[test] - fn small_table_is_lossless() { - let mut rows = Vec::new(); - for i in 0..10 { - rows.push(format!( - r#"{{"id":{i},"name":"row {i} with a reasonably long value","kind":"sample"}}"# - )); - } - let input = format!("[{}]", rows.join(",")); - let c = compress(&input).expect("compresses"); - assert!(!c.lossy, "a full table drops no data"); - assert!(!c.text.contains("omitted")); - } - - #[test] - fn non_array_returns_none() { - assert!(compress(r#"{"a":1}"#).is_none()); - assert!(compress("[1,2,3]").is_none()); - assert!(compress(r#"[{"a":1}]"#).is_none()); // too few rows - } -} diff --git a/src/openhuman/agent/harness/compaction/measure.rs b/src/openhuman/agent/harness/compaction/measure.rs deleted file mode 100644 index 9b81aa89b..000000000 --- a/src/openhuman/agent/harness/compaction/measure.rs +++ /dev/null @@ -1,150 +0,0 @@ -//! Deterministic token-savings harness for compaction (the `[agent_cost]` -//! A/B, measured at the compaction boundary). -//! -//! A live A/B runs the agent against the backend with `OPENHUMAN_COMPACTION=0` -//! vs on and diffs the `[agent_cost] … tokens_in=…` lines. That needs LLM -//! credentials + a real workspace, so it's the operator's job (commands in -//! `compaction-plan.md`). What we *can* pin down reproducibly is the input-side -//! reduction those `tokens_in` deltas come from: run representative tool -//! outputs through [`super::compact_tool_output`] and measure the token delta -//! with the **same estimator the budget/cost path uses** -//! ([`super::super::token_budget::estimate_tokens`]). -//! -//! These tests double as a regression guard: if a future change regresses the -//! savings on a fixture, they fail. Run with output: -//! -//! ```text -//! cargo test -p openhuman --lib compaction::measure -- --nocapture -//! ``` - -#![cfg(test)] - -use super::super::token_budget::estimate_tokens; -use super::compact_tool_output; -use std::fmt::Write as _; - -/// A single A/B sample: token counts before/after compaction for one fixture. -struct Sample { - label: &'static str, - tool: &'static str, - tokens_before: usize, - tokens_after: usize, -} - -impl Sample { - fn run(label: &'static str, tool: &'static str, raw: String) -> Self { - let before = estimate_tokens(&raw); - let compacted = compact_tool_output(raw, tool, true); - let after = estimate_tokens(&compacted); - Sample { - label, - tool, - tokens_before: before, - tokens_after: after, - } - } - - fn saved_pct(&self) -> f64 { - if self.tokens_before == 0 { - 0.0 - } else { - 100.0 * (1.0 - self.tokens_after as f64 / self.tokens_before as f64) - } - } -} - -// ── Representative fixtures (the loud tool families from the plan) ────────── -// Note: grep/search is intentionally not compacted, so it is not measured here. - -fn cargo_test_log_fixture() -> String { - let mut s = String::new(); - for i in 0..300 { - let _ = writeln!(s, " Compiling dependency_crate_{i} v0.{i}.0"); - } - for i in 0..40 { - let _ = writeln!( - s, - "warning: unused variable `tmp` at src/x.rs (occurrence {i})" - ); - } - let _ = writeln!(s, "error[E0382]: borrow of moved value `config`"); - let _ = writeln!(s, " --> src/server/boot.rs:142:18"); - let _ = writeln!(s, "error: aborting due to previous error"); - let _ = writeln!(s, "test result: FAILED. 87 passed; 1 failed; 0 ignored"); - s -} - -fn json_list_fixture() -> String { - let mut rows = Vec::new(); - for i in 0..150 { - rows.push(format!( - r#"{{"id":{i},"name":"user_{i}","email":"user{i}@example.com","status":"active","role":"member"}}"# - )); - } - format!("[{}]", rows.join(",")) -} - -fn diff_fixture() -> String { - let mut s = String::from("diff --git a/src/big.rs b/src/big.rs\n@@ -1,80 +1,82 @@\n"); - for i in 0..40 { - let _ = writeln!(s, " unchanged context line {i} that the diff carries along"); - } - let _ = writeln!(s, "- let x = old_implementation();"); - let _ = writeln!(s, "+ let x = new_implementation(with_args);"); - for i in 0..40 { - let _ = writeln!(s, " more unchanged context line {i} after the change"); - } - s -} - -#[test] -fn ab_token_savings_report() { - let samples = vec![ - Sample::run("cargo test failure", "run_tests", cargo_test_log_fixture()), - Sample::run("JSON list (150 rows)", "list_records", json_list_fixture()), - Sample::run("git diff (large context)", "read_diff", diff_fixture()), - ]; - - let mut total_before = 0usize; - let mut total_after = 0usize; - println!("\n[compaction A/B] token_in savings at the compaction boundary (estimate_tokens):"); - println!( - " {:<26} {:>10} {:>10} {:>9}", - "workload", "before", "after", "saved" - ); - for s in &samples { - total_before += s.tokens_before; - total_after += s.tokens_after; - println!( - " {:<26} {:>10} {:>10} {:>8.0}% (tool={})", - s.label, - s.tokens_before, - s.tokens_after, - s.saved_pct(), - s.tool - ); - // Each loud-family fixture must save meaningfully — regression guard. - assert!( - s.saved_pct() >= 30.0, - "{} only saved {:.0}%", - s.label, - s.saved_pct() - ); - } - let overall = 100.0 * (1.0 - total_after as f64 / total_before as f64); - println!( - " {:<26} {:>10} {:>10} {:>8.0}%", - "TOTAL", total_before, total_after, overall - ); - assert!(overall >= 50.0, "overall savings only {overall:.0}%"); -} - -#[test] -fn disabled_yields_zero_savings() { - // Sanity: with the kill-switch off, the harness sees no reduction — this is - // the control arm of the A/B. (Uses a fixture that *would* compact when on.) - let raw = cargo_test_log_fixture(); - let before = estimate_tokens(&raw); - let after = estimate_tokens(&compact_tool_output(raw, "run_tests", false)); - assert_eq!(before, after); -} diff --git a/src/openhuman/agent/harness/compaction/mod.rs b/src/openhuman/agent/harness/compaction/mod.rs index 96c171633..993ac7516 100644 --- a/src/openhuman/agent/harness/compaction/mod.rs +++ b/src/openhuman/agent/harness/compaction/mod.rs @@ -1,350 +1,14 @@ -//! Native tool-output compaction (Stage 1a). +//! System-prompt cache alignment (the content-compaction engine has moved). //! -//! Content-aware compression of large tool outputs, applied in -//! `Agent::execute_tool_call` **before** the byte-cap truncation in -//! [`crate::openhuman::context::tool_result_budget`] (Stage 1) and before the -//! result enters conversation history. Operates on fresh bytes that have not -//! been sent to the backend, so — like Stage 1 — it never mutates -//! previously-sent history and cannot bust the provider KV-cache prefix. +//! The content-aware tool-output compaction that used to live here — content +//! routing, the JSON/diff/log compressors, and the CCR store — has been +//! consolidated into the unified [`crate::openhuman::tokenjuice`] content +//! router (TokenJuice 2.0). The single entry points are +//! [`crate::openhuman::tokenjuice::compact_tool_output`] / +//! [`crate::openhuman::tokenjuice::compact_output`], and recovery is via the +//! `tokenjuice_retrieve` tool (with `retrieve_tool_output` kept as an alias). //! -//! This is a clean-room Rust port of the deterministic (non-ML) compressors -//! from headroom (, Apache-2.0): -//! content routing + grep/log/diff compaction. The ML text/image compressors -//! are intentionally out of scope (no Python, no ONNX, no model download). -//! -//! See `compaction-plan.md` for the full design. The downstream byte cap lives -//! in [`crate::openhuman::context::tool_result_budget`]; this stage runs just -//! ahead of it in `agent_tool_exec::run_agent_tool_call`. -//! -//! Compressors: build/test logs, unified diffs, and JSON arrays (tabular; -//! large arrays additionally row-dropped with a reversible CCR offload — see -//! [`store`]). The system-prompt cache-aligner ([`cache_align`]) runs warn-only -//! from `ContextManager::build_system_prompt`. Every lossy path is recoverable -//! via the `retrieve_tool_output` tool, so it is safe under the always-on -//! default. -//! -//! **Search/grep output is intentionally not compacted** — see the router in -//! [`compact_tool_output`]. It's a completeness tool; structured match-dropping -//! does more harm than the tokens it saves. +//! Only the system-prompt cache-aligner remains here, as it is concerned with +//! KV-cache prefix stability rather than content compression. pub mod cache_align; -pub mod detect; -pub mod diff; -pub mod json_crusher; -pub mod logs; -pub mod signals; -pub mod store; - -#[cfg(test)] -mod demo; -#[cfg(test)] -mod measure; - -use detect::{hint_for_tool, resolve, ContentHint, ContentType}; -use std::fmt::Write as _; - -/// Outputs below this many bytes are never compressed — they're already cheap -/// and the structural compressors add overhead (markers) that can outweigh the -/// saving. Matches the spirit of the plan's `min_bytes_to_compress`. -pub const MIN_BYTES_TO_COMPRESS: usize = 2048; - -/// The CCR recovery tool's name (its `name()` in -/// `tools/impl/system/retrieve_tool_output.rs`). It has two cross-cutting -/// requirements, both keyed off this constant: -/// -/// 1. Its own output is **never compacted** ([`NEVER_COMPACT_TOOLS`]) — it -/// exists to return a previously-compacted original *in full*. -/// 2. It is **always advertised** to every agent regardless of `ToolScope`, -/// because compaction applies to every agent's tool output — so any agent -/// that sees a `retrieve_tool_output("…")` footer must actually be able to -/// call it (enforced in the tool-visibility filters). -pub const RECOVERY_TOOL_NAME: &str = "retrieve_tool_output"; - -/// Tools whose output must never be re-compacted. See [`RECOVERY_TOOL_NAME`]. -pub const NEVER_COMPACT_TOOLS: &[&str] = &[RECOVERY_TOOL_NAME]; - -/// Result of a single compressor: the compacted body plus whether any data was -/// actually dropped. Both kinds offload the original to CCR and carry a recovery -/// footer (see [`compact_tool_output`]); `lossy` only changes the wording — -/// "partial view" vs "faithful reformat" — so the model knows whether it's -/// missing data or just exact formatting. -pub struct Compacted { - pub text: String, - pub lossy: bool, -} - -impl Compacted { - /// All values preserved — only structure/formatting changed (e.g. the JSON - /// table). The exact original is still offered for recovery. - pub fn reformatted(text: String) -> Self { - Self { text, lossy: false } - } - /// Data was dropped — the original is offloaded so it stays recoverable. - pub fn lossy(text: String) -> Self { - Self { text, lossy: true } - } -} - -/// Compress a tool's output for the model context, routed by the tool name. -/// -/// Returns the (possibly) compacted string. Always falls back to the original -/// when: compaction is disabled, the output is small, the content type isn't -/// one we compress, or compression wouldn't shrink it. The result still flows -/// through the downstream byte budget, so this can only ever *help*. -/// -/// **Reversibility / honesty:** whenever a compressor drops data (`lossy`), the -/// full original is stashed in the [`store`] (CCR) and the returned text ends -/// with an explicit footer telling the model this is a partial view and how to -/// fetch the original via `retrieve_tool_output("")`. So the model is -/// never silently handed a truncated result, and nothing is unrecoverable. -pub fn compact_tool_output(content: String, tool_name: &str, enabled: bool) -> String { - if !enabled || content.len() < MIN_BYTES_TO_COMPRESS || NEVER_COMPACT_TOOLS.contains(&tool_name) - { - return content; - } - - let hint = hint_for_tool(tool_name); - // A Search hint is absolute: grep output is never compacted, even if its - // body happens to look diff-like — don't let `resolve` remap it to Diff. - let content_type = if matches!(hint, ContentHint::Search) { - ContentType::Search - } else { - resolve(hint, &content) - }; - - let compressed = match content_type { - // Search/grep output is deliberately NOT compacted. grep is a - // completeness tool — the agent runs it to find *every* call site — - // and dropping matches (even with a recovery footer) risks it acting - // on a partial set, with a relevance heuristic that doesn't apply to - // search results anyway. Large grep output is left to the downstream - // byte budget, which persists the full result to a `file_read`-able - // artifact rather than dropping it. See the design note in the PR. - ContentType::Search => None, - ContentType::Log => logs::compress(&content), - ContentType::Diff => diff::compress(&content), - ContentType::JsonArray => json_crusher::compress(&content), - // Plain text has no structural compressor (the ML text compressor is - // intentionally out of scope) — pass through to the byte budget. - ContentType::PlainText => None, - }; - - match compressed { - Some(c) if c.text.len() < content.len() => { - let mut out = c.text; - // Always offload the original and tell the model how to get it back. - // Lossy outputs are a partial view (data dropped); reformatted ones - // (the JSON table) keep every value but change layout — either way - // the exact original is one retrieve_tool_output call away. - let hash = store::offload(&content); - if c.lossy { - let _ = write!( - out, - "\n\n[compacted tool output — this is a PARTIAL view; the \ - full original ({} bytes) is available by calling \ - retrieve_tool_output(\"{hash}\")]", - content.len() - ); - } else { - let _ = write!( - out, - "\n\n[reformatted tool output — no data lost, but layout \ - changed; the exact original ({} bytes, e.g. raw JSON) is \ - available by calling retrieve_tool_output(\"{hash}\")]", - content.len() - ); - } - // The shrink check above ran on the compressed body; the recovery - // footer adds bytes, so re-check the final size and fall back to the - // original if the footer tipped it over (marginal inputs only). - if out.len() >= content.len() { - return content; - } - let ratio = 1.0 - (out.len() as f64 / content.len() as f64); - // `::log` is the logging crate (the sibling `logs` module shadows - // the bare `log` path inside this module). - ::log::debug!( - "[compaction] tool={tool_name} type={content_type:?} lossy={} in_bytes={} out_bytes={} ratio={ratio:.2}", - c.lossy, - content.len(), - out.len(), - ); - out - } - _ => content, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::fmt::Write as _; - - #[test] - fn disabled_is_passthrough() { - let big = "x".repeat(MIN_BYTES_TO_COMPRESS + 10); - assert_eq!(compact_tool_output(big.clone(), "grep", false), big); - } - - #[test] - fn small_output_passthrough() { - let small = "a.rs:1:hit\nb.rs:2:hit".to_string(); - assert_eq!(compact_tool_output(small.clone(), "grep", true), small); - } - - #[test] - fn search_output_is_not_compacted() { - // grep is a completeness tool — its output must pass through untouched - // even when large, so the agent never acts on a silently-dropped subset. - let mut s = String::from("80 match(es); scanned 2 file(s)\n"); - for i in 1..=40 { - let _ = writeln!( - s, - "src/a.rs:{i}:let value_{i} = compute_something_long_{i}();" - ); - } - for i in 1..=40 { - let _ = writeln!( - s, - "src/b.rs:{i}:fn helper_function_number_{i}() {{ /* body */ }}" - ); - } - assert!(s.len() >= MIN_BYTES_TO_COMPRESS); - let out = compact_tool_output(s.clone(), "grep", true); - assert_eq!(out, s, "search output must pass through unchanged"); - } - - #[test] - fn unknown_tool_plain_text_passthrough() { - let prose = "lorem ipsum ".repeat(400); // > MIN, but plain text - let out = compact_tool_output(prose.clone(), "some_tool", true); - assert_eq!(out, prose); - } - - /// Pull the CCR hash out of the retrieval footer the model is shown. - fn footer_hash(out: &str) -> Option<&str> { - out.split("retrieve_tool_output(\"") - .nth(1) - .and_then(|s| s.split('"').next()) - } - - #[test] - fn retrieval_returns_the_full_original_uncompacted() { - // End-to-end recovery: a large JSON result is compacted (lossy) and its - // original offloaded; the model reads the footer hash and calls - // retrieve_tool_output. That tool's output flows back through Stage 1a — - // and must NOT be re-compacted, or the agent could never see the full - // data it asked for. - let mut rows = Vec::new(); - for i in 0..120 { - rows.push(format!( - r#"{{"id":{i},"name":"account_{i}","email":"a{i}@ex.com","tier":"gold"}}"# - )); - } - let original = format!("[{}]", rows.join(",")); - - // 1. First pass compacts + offloads. - let compacted = compact_tool_output(original.clone(), "list_accounts", true); - assert!(compacted.contains("retrieve_tool_output(")); - let hash = footer_hash(&compacted).expect("footer hash"); - - // 2. The retrieve tool fetches the original from CCR. - let fetched = store::retrieve(hash).expect("CCR has it"); - assert_eq!(fetched, original); - - // 3. That fetched output passes through Stage 1a under the retrieve - // tool's name — and must come back byte-for-byte, NOT re-compacted. - let delivered = compact_tool_output(fetched, "retrieve_tool_output", true); - assert_eq!( - delivered, original, - "recovery must deliver the full original" - ); - assert!(!delivered.contains("partial view")); - } - - #[test] - fn every_lossy_output_tells_the_model_and_is_recoverable() { - // One representative input per lossy compressor. Each must (a) carry the - // explicit "partial view / retrieve_tool_output" footer, and (b) have - // its full original recoverable byte-for-byte from the CCR store — i.e. - // no information is actually lost, only deferred. (grep is excluded — - // search output is intentionally never compacted.) - let mut log = String::new(); - for i in 0..200 { - let _ = writeln!(log, " Compiling crate_{i} v0.{i}.0"); - } - let _ = writeln!(log, "error: aborting"); - let mut diff = String::from("diff --git a/x.rs b/x.rs\n@@ -1,60 +1,61 @@\n"); - for i in 0..50 { - let _ = writeln!( - diff, - " unchanged context line {i} carried along by the diff" - ); - } - let _ = writeln!(diff, "+changed"); - let mut jrows = Vec::new(); - for i in 0..120 { - jrows.push(format!( - r#"{{"id":{i},"name":"account_{i}","email":"a{i}@ex.com","tier":"gold"}}"# - )); - } - let json = format!("[{}]", jrows.join(",")); - - for (tool, input) in [ - ("run_tests", log), - ("read_diff", diff), - ("list_accounts", json), - ] { - let out = compact_tool_output(input.clone(), tool, true); - assert!(out.len() < input.len(), "{tool}: not compacted"); - // (a) the model is explicitly told this is a partial view. - assert!( - out.contains("PARTIAL view") && out.contains("retrieve_tool_output("), - "{tool}: missing retrieval footer:\n{out}" - ); - // (b) the full original is recoverable byte-for-byte. - let hash = footer_hash(&out).expect("footer has a hash"); - assert_eq!( - store::retrieve(hash).as_deref(), - Some(input.as_str()), - "{tool}: CCR did not round-trip" - ); - } - } - - #[test] - fn reformatted_table_preserves_values_and_offers_exact_recovery() { - // A JSON list under the row-drop threshold is reformatted, not dropped: - // every value is present (no "omitted"), it's framed as a faithful - // reformat (not a "partial view"), and the EXACT original JSON is - // recoverable via the retrieve footer — that's how the agent asks for - // exact bytes. - // 38 rows: above the 2048-byte floor, below the 40-row drop threshold. - let mut rows = Vec::new(); - for i in 0..38 { - rows.push(format!( - r#"{{"id":{i},"sku":"SKU-{i:05}","name":"widget number {i} in the catalog","warehouse":"east-region-1"}}"# - )); - } - let input = format!("[{}]", rows.join(",")); - assert!(input.len() >= MIN_BYTES_TO_COMPRESS); - let out = compact_tool_output(input.clone(), "list_inventory", true); - - assert!(out.len() < input.len(), "table should shrink"); - assert!(!out.contains("omitted"), "reformat ⇒ nothing dropped"); - // Framed as a faithful reformat, not a lossy partial view. - assert!(out.contains("no data lost"), "{out}"); - assert!(!out.contains("PARTIAL view"), "{out}"); - // Every row's identifying values survive the reformat. - for i in 0..38 { - assert!(out.contains(&format!("SKU-{i:05}")), "lost SKU {i}"); - assert!( - out.contains(&format!("widget number {i} in the catalog")), - "lost name {i}" - ); - } - // The agent can fetch the exact original JSON back, byte-for-byte. - let hash = footer_hash(&out).expect("reformat footer has a hash"); - assert_eq!(store::retrieve(hash).as_deref(), Some(input.as_str())); - } -} diff --git a/src/openhuman/agent/harness/compaction/store.rs b/src/openhuman/agent/harness/compaction/store.rs deleted file mode 100644 index 4a3c07c4c..000000000 --- a/src/openhuman/agent/harness/compaction/store.rs +++ /dev/null @@ -1,135 +0,0 @@ -//! CCR — Compress-Cache-Retrieve store. -//! -//! When a compressor drops data (lossy paths), it stows the original here -//! keyed by a short content hash and emits a `retrieve_tool_output("")` -//! sentinel in the compacted text. The agent can call the -//! `retrieve_tool_output` tool to get the original back on demand — so even -//! aggressive compaction stays reversible and is safe under the always-on -//! default. -//! -//! Process-global and bounded: a fixed-capacity FIFO so a long session can't -//! grow it without bound. Keyed by content hash, so re-offloading identical -//! content is idempotent (the model sees a stable hash). Originals are not -//! persisted to disk — retrieval is best-effort within the session; an evicted -//! entry simply reports "no longer available", which is strictly better than -//! the pre-CCR behaviour (the data was gone the moment it was truncated). - -use sha2::{Digest, Sha256}; -use std::collections::{HashMap, VecDeque}; -use std::sync::{Mutex, OnceLock}; - -/// Max originals retained. ~256 large tool outputs is plenty for a session's -/// recent history while bounding worst-case memory. -const MAX_ENTRIES: usize = 256; -/// Bytes of the SHA-256 digest used for the key (→ 32 hex chars). Wide enough -/// that (a) collisions are infeasible and (b) the hash doubles as an -/// unguessable capability token — a session can only retrieve content whose -/// hash it was shown, so the process-global store can't be brute-force probed -/// across sessions. (Full per-session key namespacing is a tracked follow-up.) -const HASH_BYTES: usize = 16; - -#[derive(Default)] -struct Inner { - map: HashMap, - order: VecDeque, -} - -impl Inner { - /// Insert `content` under `hash` (idempotent) and FIFO-evict down to - /// [`MAX_ENTRIES`]. Pulled out of [`offload`] so the eviction policy can be - /// unit-tested on a local instance without touching the process-global - /// store (which would otherwise race other tests sharing it). - fn insert(&mut self, hash: String, content: String) { - if self.map.contains_key(&hash) { - return; - } - self.map.insert(hash.clone(), content); - self.order.push_back(hash); - while self.order.len() > MAX_ENTRIES { - if let Some(evicted) = self.order.pop_front() { - self.map.remove(&evicted); - } - } - } -} - -fn global() -> &'static Mutex { - static STORE: OnceLock> = OnceLock::new(); - STORE.get_or_init(|| Mutex::new(Inner::default())) -} - -/// Stash `content` and return its short hash. Idempotent for identical content. -pub fn offload(content: &str) -> String { - let hash = short_hash(content); - global() - .lock() - .unwrap_or_else(|p| p.into_inner()) - .insert(hash.clone(), content.to_string()); - hash -} - -/// Retrieve a previously-offloaded original by hash, if still cached. -pub fn retrieve(hash: &str) -> Option { - global() - .lock() - .unwrap_or_else(|p| p.into_inner()) - .map - .get(hash) - .cloned() -} - -/// Short hex content hash used as the CCR key. -pub fn short_hash(content: &str) -> String { - let mut hasher = Sha256::new(); - hasher.update(content.as_bytes()); - let digest = hasher.finalize(); - hex::encode(&digest[..HASH_BYTES]) -} - -#[cfg(test)] -mod tests { - use super::*; - - // Round-trip tests use globally-unique content and collectively stay well - // under MAX_ENTRIES, so they never evict each other even under parallel - // execution. The eviction policy is exercised on a *local* `Inner` below so - // it can't clobber the shared store other tests depend on. - - #[test] - fn round_trips() { - let original = "ccr round-trip unique payload ".repeat(50); - let hash = offload(&original); - assert_eq!(hash.len(), HASH_BYTES * 2); - assert_eq!(retrieve(&hash).as_deref(), Some(original.as_str())); - } - - #[test] - fn idempotent_hash() { - let a = offload("ccr idempotent unique payload content here"); - let b = offload("ccr idempotent unique payload content here"); - assert_eq!(a, b); - } - - #[test] - fn missing_hash_is_none() { - // A 32-hex hash that no test content maps to. - assert!(retrieve("ffffffffffffffffffffffffffffffff").is_none()); - } - - #[test] - fn eviction_bounds_size() { - // Exercise the FIFO eviction on a local instance — no shared state. - let mut inner = Inner::default(); - for i in 0..(MAX_ENTRIES + 50) { - inner.insert(format!("hash-{i}"), format!("content-{i}")); - } - assert!(inner.map.len() <= MAX_ENTRIES, "size bounded"); - assert!(!inner.map.contains_key("hash-0"), "oldest entry evicted"); - assert!( - inner - .map - .contains_key(&format!("hash-{}", MAX_ENTRIES + 49)), - "newest entry retained" - ); - } -} diff --git a/src/openhuman/agent/harness/engine/tools.rs b/src/openhuman/agent/harness/engine/tools.rs index 0e016d9fd..5cabbe57a 100644 --- a/src/openhuman/agent/harness/engine/tools.rs +++ b/src/openhuman/agent/harness/engine/tools.rs @@ -294,7 +294,8 @@ pub(crate) async fn run_one_tool( Some(&call.arguments), &scrubbed, Some(0), - ); + ) + .await; if tj_stats.applied { log::debug!( "[agent_loop] tokenjuice applied tool={} rule={} {}->{} bytes", @@ -383,7 +384,8 @@ pub(crate) async fn run_one_tool( Some(&call.arguments), &scrubbed, Some(1), - ); + ) + .await; (format!("Error: {compacted}"), false) } } diff --git a/src/openhuman/agent/harness/session/agent_tool_exec.rs b/src/openhuman/agent/harness/session/agent_tool_exec.rs index 80d8bc2fa..cbe7dc2fc 100644 --- a/src/openhuman/agent/harness/session/agent_tool_exec.rs +++ b/src/openhuman/agent/harness/session/agent_tool_exec.rs @@ -241,15 +241,18 @@ pub(super) async fn run_agent_tool_call( (format!("Unknown tool: {}", call.name), false) }; - // Stage 1a — content-aware compaction. Runs before the byte budget on the - // fresh tool output (never sent to the backend yet, so it's cache-safe like - // the budget below). Routes by tool name; only ever shrinks, otherwise - // passes the original through. See `agent::harness::compaction`. - let raw_result = crate::openhuman::agent::harness::compaction::compact_tool_output( + // Stage 1a — content-aware compaction via the TokenJuice content router. + // Runs before the byte budget on the fresh tool output (never sent to the + // backend yet, so it's cache-safe like the budget below). Detects the + // content kind, routes to the matching compressor, and offloads the + // original to CCR (recoverable via `tokenjuice_retrieve`); only ever + // shrinks, otherwise passes the original through. + let raw_result = crate::openhuman::tokenjuice::compact_output( raw_result, &call.name, ctx.compaction_enabled, - ); + ) + .await; // Per-result byte budget — the only cache-safe reduction stage (the full // body has never been sent to the backend). Oversized outputs are persisted diff --git a/src/openhuman/agent/harness/session/builder/builder_tests.rs b/src/openhuman/agent/harness/session/builder/builder_tests.rs index 5e4bdd967..b375d4297 100644 --- a/src/openhuman/agent/harness/session/builder/builder_tests.rs +++ b/src/openhuman/agent/harness/session/builder/builder_tests.rs @@ -16,7 +16,7 @@ fn spec(name: &str) -> ToolSpec { #[test] fn recovery_tool_joins_a_named_allowlist() { - use crate::openhuman::agent::harness::compaction::RECOVERY_TOOL_NAME; + use crate::openhuman::tokenjuice::RETRIEVE_TOOL_NAME as RECOVERY_TOOL_NAME; use std::collections::HashSet; // A curated Named-scope allowlist gains retrieve_tool_output as a *real* diff --git a/src/openhuman/agent/harness/session/builder/mod.rs b/src/openhuman/agent/harness/session/builder/mod.rs index 0a3175e6c..fe94895dc 100644 --- a/src/openhuman/agent/harness/session/builder/mod.rs +++ b/src/openhuman/agent/harness/session/builder/mod.rs @@ -74,8 +74,9 @@ pub(super) fn visible_tool_specs_for_policy( /// the deliberately tool-less `Named([])` case, which must stay tool-less. pub(super) fn ensure_recovery_tool_visible(visible: &mut std::collections::HashSet) { if !visible.is_empty() { - visible - .insert(crate::openhuman::agent::harness::compaction::RECOVERY_TOOL_NAME.to_string()); + for name in crate::openhuman::tokenjuice::RECOVERY_TOOL_NAMES { + visible.insert((*name).to_string()); + } } } diff --git a/src/openhuman/agent/harness/subagent_runner/tool_prep.rs b/src/openhuman/agent/harness/subagent_runner/tool_prep.rs index 6122ce7ce..34d6c609d 100644 --- a/src/openhuman/agent/harness/subagent_runner/tool_prep.rs +++ b/src/openhuman/agent/harness/subagent_runner/tool_prep.rs @@ -158,7 +158,7 @@ pub(crate) fn filter_tool_indices( // explicit `disallow` above still wins). A deliberately tool-less // agent (`Named([])`, e.g. the payload summarizer) runs no tools, // produces no compacted output, and so stays tool-less. - if name == crate::openhuman::agent::harness::compaction::RECOVERY_TOOL_NAME { + if crate::openhuman::tokenjuice::is_recovery_tool(name) { return !matches!(scope, ToolScope::Named(allowed) if allowed.is_empty()); } if let Some(prefix) = skill_prefix.as_deref() { @@ -205,7 +205,7 @@ mod tests { #[cfg(test)] mod recovery_visibility_tests { use super::*; - use crate::openhuman::agent::harness::compaction::RECOVERY_TOOL_NAME; + use crate::openhuman::tokenjuice::LEGACY_RETRIEVE_TOOL_NAME as RECOVERY_TOOL_NAME; use crate::openhuman::tools::{CurrentTimeTool, RetrieveToolOutputTool}; fn tools() -> Vec> { diff --git a/src/openhuman/config/mod.rs b/src/openhuman/config/mod.rs index d3f17e861..966aaae9c 100644 --- a/src/openhuman/config/mod.rs +++ b/src/openhuman/config/mod.rs @@ -45,9 +45,9 @@ pub use schema::{ SandboxConfig, SchedulerConfig, SchedulerGateConfig, SchedulerGateMode, ScreenIntelligenceConfig, SearchConfig, SearchEngine, SearchEngineCredentials, SearxngConfig, SecretsConfig, SecurityConfig, ShellConfig, SlackConfig, StorageConfig, StorageProviderConfig, - StorageProviderSection, StreamMode, TeamModelConfig, TelegramConfig, UpdateConfig, - UpdateRestartStrategy, VoiceActivationMode, VoiceServerConfig, WebSearchConfig, WebhookConfig, - DEFAULT_CLOUD_LLM_MODEL, DEFAULT_MEMORY_SYNC_INTERVAL_SECS, DEFAULT_MODEL, + StorageProviderSection, StreamMode, TeamModelConfig, TelegramConfig, TokenjuiceConfig, + UpdateConfig, UpdateRestartStrategy, VoiceActivationMode, VoiceServerConfig, WebSearchConfig, + WebhookConfig, DEFAULT_CLOUD_LLM_MODEL, DEFAULT_MEMORY_SYNC_INTERVAL_SECS, DEFAULT_MODEL, MEMORY_SYNC_INTERVAL_PRESETS_SECS, MODEL_AGENTIC_V1, MODEL_CHAT_V1, MODEL_CODING_V1, MODEL_REASONING_QUICK_V1, MODEL_REASONING_V1, MODEL_SUMMARIZATION_V1, MODEL_VISION_V1, SEARCH_ENGINE_BRAVE, SEARCH_ENGINE_DISABLED, SEARCH_ENGINE_MANAGED, SEARCH_ENGINE_PARALLEL, diff --git a/src/openhuman/config/schema/load/env_overlay.rs b/src/openhuman/config/schema/load/env_overlay.rs index 03464e017..6df6fd690 100644 --- a/src/openhuman/config/schema/load/env_overlay.rs +++ b/src/openhuman/config/schema/load/env_overlay.rs @@ -421,6 +421,93 @@ impl Config { if let Some(command) = env.get("OPENHUMAN_RUNTIME_PYTHON_PREFERRED_COMMAND") { self.runtime_python.preferred_command = command.trim().to_string(); } + + // --- TokenJuice content router ------------------------------------- + if let Some(flag) = env.get("OPENHUMAN_TOKENJUICE_ENABLED") { + if let Some(v) = parse_env_bool("OPENHUMAN_TOKENJUICE_ENABLED", &flag) { + self.tokenjuice.router_enabled = v; + } + } + if let Some(flag) = env.get("OPENHUMAN_TOKENJUICE_CCR_ENABLED") { + if let Some(v) = parse_env_bool("OPENHUMAN_TOKENJUICE_CCR_ENABLED", &flag) { + self.tokenjuice.ccr_enabled = v; + } + } + if let Some(flag) = env.get("OPENHUMAN_TOKENJUICE_CCR_DISK_ENABLED") { + if let Some(v) = parse_env_bool("OPENHUMAN_TOKENJUICE_CCR_DISK_ENABLED", &flag) { + self.tokenjuice.ccr_disk_enabled = v; + } + } + if let Some(flag) = env.get("OPENHUMAN_TOKENJUICE_SEARCH_ENABLED") { + if let Some(v) = parse_env_bool("OPENHUMAN_TOKENJUICE_SEARCH_ENABLED", &flag) { + self.tokenjuice.search_enabled = v; + } + } + if let Some(flag) = env.get("OPENHUMAN_TOKENJUICE_CODE_ENABLED") { + if let Some(v) = parse_env_bool("OPENHUMAN_TOKENJUICE_CODE_ENABLED", &flag) { + self.tokenjuice.code_enabled = v; + } + } + if let Some(flag) = env.get("OPENHUMAN_TOKENJUICE_HTML_ENABLED") { + if let Some(v) = parse_env_bool("OPENHUMAN_TOKENJUICE_HTML_ENABLED", &flag) { + self.tokenjuice.html_enabled = v; + } + } + if let Some(s) = env.get("OPENHUMAN_TOKENJUICE_MAX_CACHE_ENTRIES") { + if let Ok(v) = s.trim().parse::() { + self.tokenjuice.max_cache_entries = v; + } + } + if let Some(s) = env.get("OPENHUMAN_TOKENJUICE_MAX_CACHE_BYTES") { + if let Ok(v) = s.trim().parse::() { + self.tokenjuice.max_cache_bytes = v; + } + } + if let Some(s) = env.get("OPENHUMAN_TOKENJUICE_CCR_TTL_SECS") { + if let Ok(v) = s.trim().parse::() { + self.tokenjuice.ccr_ttl_secs = Some(v); + } + } + if let Some(s) = env.get("OPENHUMAN_TOKENJUICE_CCR_MIN_TOKENS") { + if let Ok(v) = s.trim().parse::() { + self.tokenjuice.ccr_min_tokens = v; + } + } + // ML plain-text compressor (Kompress). + if let Some(flag) = env.get("OPENHUMAN_TOKENJUICE_ML_COMPRESSION_ENABLED") { + if let Some(v) = parse_env_bool("OPENHUMAN_TOKENJUICE_ML_COMPRESSION_ENABLED", &flag) { + self.tokenjuice.ml_compression_enabled = v; + } + } + if let Some(m) = env.get("OPENHUMAN_TOKENJUICE_ML_MODEL_ID") { + let t = m.trim(); + if !t.is_empty() { + self.tokenjuice.ml_model_id = t.to_string(); + } + } + if let Some(d) = env.get("OPENHUMAN_TOKENJUICE_ML_DEVICE") { + let t = d.trim(); + if !t.is_empty() { + self.tokenjuice.ml_device = t.to_string(); + } + } + if let Some(r) = env.get("OPENHUMAN_TOKENJUICE_ML_TARGET_RATIO") { + if let Ok(v) = r.trim().parse::() { + if (0.0..=1.0).contains(&v) { + self.tokenjuice.ml_target_ratio = v; + } + } + } + if let Some(s) = env.get("OPENHUMAN_TOKENJUICE_ML_SIDECAR_IDLE_TIMEOUT_SECS") { + if let Ok(v) = s.trim().parse::() { + self.tokenjuice.ml_sidecar_idle_timeout_secs = v; + } + } + if let Some(s) = env.get("OPENHUMAN_TOKENJUICE_ML_MAX_INPUT_CHARS") { + if let Ok(v) = s.trim().parse::() { + self.tokenjuice.ml_max_input_chars = v; + } + } } fn apply_observability_env(&mut self, env: &E) { diff --git a/src/openhuman/config/schema/mod.rs b/src/openhuman/config/schema/mod.rs index fbcafbd74..bc3b0ce7e 100644 --- a/src/openhuman/config/schema/mod.rs +++ b/src/openhuman/config/schema/mod.rs @@ -41,6 +41,7 @@ mod runtime_python; mod scheduler_gate; mod storage_memory; mod task_sources; +mod tokenjuice; mod tools; mod update; @@ -84,6 +85,7 @@ pub use storage_memory::{ StorageProviderSection, DEFAULT_CLOUD_LLM_MODEL, }; pub use task_sources::TaskSourcesConfig; +pub use tokenjuice::TokenjuiceConfig; pub use tools::{ BrowserComputerUseConfig, BrowserConfig, ComposioConfig, ComputerControlConfig, CurlConfig, GitbooksConfig, HttpHeader, HttpRequestConfig, IntegrationToggle, IntegrationsConfig, diff --git a/src/openhuman/config/schema/tokenjuice.rs b/src/openhuman/config/schema/tokenjuice.rs new file mode 100644 index 000000000..da62fb217 --- /dev/null +++ b/src/openhuman/config/schema/tokenjuice.rs @@ -0,0 +1,163 @@ +//! TokenJuice content-router configuration (`[tokenjuice]`). +//! +//! Controls the content-aware tool-output compaction engine: which compressors +//! are enabled, the Compress-Cache-Retrieve (CCR) store limits, and the opt-in +//! Python/ML plain-text compressor. Installed into the runtime at startup via +//! [`crate::openhuman::tokenjuice::configure`] + the CCR cache `configure`. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +pub struct TokenjuiceConfig { + /// Master switch for the content router. When `false`, tool output passes + /// through uncompacted. + #[serde(default = "default_true")] + pub router_enabled: bool, + /// Whether lossy compressions offload the original to the CCR store and emit + /// a `⟦tj:⟧` retrieval footer. Disabling makes compaction one-way. + #[serde(default = "default_true")] + pub ccr_enabled: bool, + /// Persist CCR originals to disk under `/.tokenjuice/ccr` so + /// retrieval survives memory eviction (written by the core only). + #[serde(default)] + pub ccr_disk_enabled: bool, + /// Max number of originals retained in the in-memory CCR store. + #[serde(default = "default_max_cache_entries")] + pub max_cache_entries: usize, + /// Max total bytes retained in the in-memory CCR store. + #[serde(default = "default_max_cache_bytes")] + pub max_cache_bytes: usize, + /// Optional TTL (seconds) for CCR entries; `None` ⇒ no expiry. + #[serde(default)] + pub ccr_ttl_secs: Option, + /// Minimum output size (bytes) before compaction is attempted. + #[serde(default = "default_min_bytes")] + pub min_bytes_to_compress: usize, + /// CCR only fires (original offloaded + lossy compaction) when the tool + /// result is estimated at ≥ this many tokens. Smaller results pass through. + #[serde(default = "default_ccr_min_tokens")] + pub ccr_min_tokens: usize, + /// Enable the search-results (grep) relevance compressor. + #[serde(default = "default_true")] + pub search_enabled: bool, + /// Enable the AST/heuristic code compressor. + #[serde(default = "default_true")] + pub code_enabled: bool, + /// Enable the HTML→text extractor. + #[serde(default = "default_true")] + pub html_enabled: bool, + + // --- ML plain-text compressor (Kompress) — opt-in, default OFF --------- + /// Enable the Python/ML plain-text compressor ("Kompress"). Runs as a + /// `kompress` backend of the runtime_python_server (requires + /// `runtime_python.enabled`); degrades gracefully when unavailable. + #[serde(default)] + pub ml_compression_enabled: bool, + /// HuggingFace model id for the ML compressor. + #[serde(default = "default_ml_model_id")] + pub ml_model_id: String, + /// Target compression ratio (0–1) hint for the ML compressor. + #[serde(default = "default_ml_target_ratio")] + pub ml_target_ratio: f64, + /// Idle seconds before the ML sidecar process is reaped to release memory. + #[serde(default = "default_ml_idle_timeout_secs")] + pub ml_sidecar_idle_timeout_secs: u64, + /// Maximum input characters the ML compressor will accept (larger inputs + /// fall back to a native compressor). + #[serde(default = "default_ml_max_input_chars")] + pub ml_max_input_chars: usize, + /// Inference device: `cpu` or `auto`. + #[serde(default = "default_ml_device")] + pub ml_device: String, +} + +fn default_true() -> bool { + true +} +fn default_max_cache_entries() -> usize { + 256 +} +fn default_max_cache_bytes() -> usize { + 64 * 1024 * 1024 +} +fn default_min_bytes() -> usize { + 2048 +} +fn default_ccr_min_tokens() -> usize { + 500 +} +fn default_ml_model_id() -> String { + "answerdotai/ModernBERT-base".to_string() +} +fn default_ml_target_ratio() -> f64 { + 0.5 +} +fn default_ml_idle_timeout_secs() -> u64 { + 900 +} +fn default_ml_max_input_chars() -> usize { + 200_000 +} +fn default_ml_device() -> String { + "cpu".to_string() +} + +impl Default for TokenjuiceConfig { + fn default() -> Self { + Self { + router_enabled: true, + ccr_enabled: true, + ccr_disk_enabled: false, + max_cache_entries: default_max_cache_entries(), + max_cache_bytes: default_max_cache_bytes(), + ccr_ttl_secs: None, + min_bytes_to_compress: default_min_bytes(), + ccr_min_tokens: default_ccr_min_tokens(), + search_enabled: true, + code_enabled: true, + html_enabled: true, + ml_compression_enabled: false, + ml_model_id: default_ml_model_id(), + ml_target_ratio: default_ml_target_ratio(), + ml_sidecar_idle_timeout_secs: default_ml_idle_timeout_secs(), + ml_max_input_chars: default_ml_max_input_chars(), + ml_device: default_ml_device(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults_are_sane() { + let c = TokenjuiceConfig::default(); + assert!(c.router_enabled); + assert!(c.ccr_enabled); + assert!(c.search_enabled); + assert!(!c.ml_compression_enabled); + assert_eq!(c.ml_device, "cpu"); + } + + #[test] + fn parses_from_toml() { + let c: TokenjuiceConfig = toml::from_str( + r#" + router_enabled = false + ml_compression_enabled = true + max_cache_entries = 12 + ccr_ttl_secs = 300 + "#, + ) + .unwrap(); + assert!(!c.router_enabled); + assert!(c.ml_compression_enabled); + assert_eq!(c.max_cache_entries, 12); + assert_eq!(c.ccr_ttl_secs, Some(300)); + // Untouched fields keep defaults. + assert!(c.code_enabled); + } +} diff --git a/src/openhuman/config/schema/types.rs b/src/openhuman/config/schema/types.rs index 2917210cb..62ad1a489 100644 --- a/src/openhuman/config/schema/types.rs +++ b/src/openhuman/config/schema/types.rs @@ -394,6 +394,10 @@ pub struct Config { #[serde(default)] pub runtime_python: RuntimePythonConfig, + /// TokenJuice content-router / compaction configuration. + #[serde(default)] + pub tokenjuice: TokenjuiceConfig, + #[serde(default)] pub voice_server: VoiceServerConfig, @@ -785,6 +789,7 @@ impl Default for Config { subconscious_provider: None, node: NodeConfig::default(), runtime_python: RuntimePythonConfig::default(), + tokenjuice: TokenjuiceConfig::default(), voice_server: VoiceServerConfig::default(), voice_providers: Vec::new(), stt_provider: None, diff --git a/src/openhuman/harness_init/registry.rs b/src/openhuman/harness_init/registry.rs index 6c85ac211..b76c89056 100644 --- a/src/openhuman/harness_init/registry.rs +++ b/src/openhuman/harness_init/registry.rs @@ -42,6 +42,7 @@ pub fn all_steps() -> Vec { vec![ python_runtime_step(), spacy_step(), + kompress_step(), runtime_python_server_step(), node_runtime_step(), ] @@ -153,6 +154,46 @@ async fn spacy_run(config: &Config) -> Result<(), String> { // ── node_runtime ──────────────────────────────────────────────────────────── +fn kompress_step() -> HarnessInitStep { + HarnessInitStep { + id: "kompress", + label: "TokenJuice ML compressor (torch)", + required: false, + is_done: |config| Box::pin(kompress_is_done(config)), + run: |config| Box::pin(kompress_run(config)), + } +} + +/// Whether this step should provision a *dedicated* Kompress venv. When spaCy is +/// also enabled, the single runtime-python server must share one interpreter, so +/// `runtime_python_server_step` installs torch into the spaCy venv instead — a +/// dedicated venv here would be unused and double the (heavy) provisioning work. +fn kompress_needs_dedicated_venv(config: &Config) -> bool { + config.runtime_python.enabled + && config.tokenjuice.ml_compression_enabled + && !config.memory_tree.spacy_enabled +} + +async fn kompress_is_done(config: &Config) -> bool { + if !kompress_needs_dedicated_venv(config) { + return true; + } + crate::openhuman::runtime_python_server::kompress_provisioned(config) +} + +async fn kompress_run(config: &Config) -> Result<(), String> { + if !kompress_needs_dedicated_venv(config) { + // Shared-venv case (spaCy on) is provisioned by the server launch step. + return Ok(()); + } + crate::openhuman::runtime_python_server::ensure_kompress(config) + .await + .map(|_| { + log::info!("[harness_init] Kompress (torch) provisioned"); + }) + .map_err(|e| format!("{e:#}")) +} + fn node_runtime_step() -> HarnessInitStep { HarnessInitStep { id: "node_runtime", @@ -208,6 +249,7 @@ mod tests { vec![ "python_runtime", "spacy", + "kompress", "runtime_python_server", "node_runtime" ] diff --git a/src/openhuman/runtime_python_server/kompress.rs b/src/openhuman/runtime_python_server/kompress.rs new file mode 100644 index 000000000..83927c3da --- /dev/null +++ b/src/openhuman/runtime_python_server/kompress.rs @@ -0,0 +1,317 @@ +//! Kompress backend — TokenJuice ML plain-text compressor (ModernBERT/torch). +//! +//! Provisions torch + transformers into the runtime python server's venv and +//! pre-downloads the model so the long-lived server can load it offline at +//! startup. The actual compression runs inside the shared `server.py` and is +//! reached via [`request_kompress`] (→ `server::request("kompress.compress")`). +//! +//! Two provisioning entry points so the single-venv server can host Kompress +//! alongside (or instead of) spaCy: +//! - [`ensure_kompress`] creates a dedicated `kompress-venv` when Kompress is +//! the only heavy backend. +//! - [`install_into`] adds torch + transformers to an *existing* venv (e.g. the +//! spaCy venv) when both backends are enabled and must share one interpreter. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{bail, Context, Result}; +use serde::{Deserialize, Serialize}; +use tokio::process::Command; +use tokio::sync::OnceCell; + +use crate::openhuman::config::Config; +use crate::openhuman::runtime_python::PythonBootstrap; + +use super::spacy::python_server_cache_root; + +const VENV_TIMEOUT: Duration = Duration::from_secs(120); +/// torch + transformers wheels are large; allow a generous one-time window. +const PIP_TIMEOUT: Duration = Duration::from_secs(1800); +const MODEL_TIMEOUT: Duration = Duration::from_secs(1800); + +static PROVISION_LOCK: OnceCell> = OnceCell::const_new(); + +async fn provision_lock() -> &'static tokio::sync::Mutex<()> { + PROVISION_LOCK + .get_or_init(|| async { tokio::sync::Mutex::new(()) }) + .await +} + +/// A provisioned Kompress runtime: the venv interpreter with torch+transformers +/// and the HuggingFace cache holding the (pre-downloaded) model weights. +#[derive(Debug, Clone)] +pub struct KompressRuntime { + pub python_bin: PathBuf, + pub hf_home: PathBuf, +} + +/// Result of one `kompress.compress` call. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct KompressResponse { + #[serde(default)] + pub compressed_text: String, + #[serde(default)] + pub input_chars: usize, + #[serde(default)] + pub output_chars: usize, +} + +/// Compress `text` with the Kompress backend over the shared runtime server. +pub async fn request_kompress(config: &Config, text: &str) -> Result { + super::server::request_kompress_compress(config, text).await +} + +/// HF cache directory for Kompress model weights (kept under the server cache). +pub(crate) fn hf_home(config: &Config) -> PathBuf { + python_server_cache_root(config).join("kompress-hf") +} + +fn kompress_venv_dir(config: &Config) -> PathBuf { + python_server_cache_root(config).join("kompress-venv") +} + +fn venv_python_path(venv_dir: &Path) -> PathBuf { + if cfg!(windows) { + venv_dir.join("Scripts").join("python.exe") + } else { + venv_dir.join("bin").join("python") + } +} + +fn marker_path(venv_dir: &Path, model_id: &str) -> PathBuf { + let safe_model: String = model_id + .chars() + .map(|ch| { + if ch.is_ascii_alphanumeric() || matches!(ch, '.' | '-' | '_') { + ch + } else { + '_' + } + }) + .collect(); + venv_dir.join(format!(".openhuman-kompress-ready-{safe_model}")) +} + +/// Cheap, network-free probe: are torch + transformers + the model provisioned? +pub fn kompress_provisioned(config: &Config) -> bool { + let venv = kompress_venv_dir(config); + marker_path(&venv, &config.tokenjuice.ml_model_id).exists() && venv_python_path(&venv).exists() +} + +/// Ensure a dedicated Kompress venv exists (torch + transformers + model). +pub async fn ensure_kompress(config: &Config) -> Result { + let _guard = provision_lock().await.lock().await; + if !config.runtime_python.enabled { + bail!("runtime_python disabled — cannot provision Kompress"); + } + if !config.tokenjuice.ml_compression_enabled { + bail!("tokenjuice.ml_compression_enabled is false"); + } + + let venv_dir = kompress_venv_dir(config); + let venv_python = venv_python_path(&venv_dir); + let hf = hf_home(config); + + if marker_path(&venv_dir, &config.tokenjuice.ml_model_id).exists() && venv_python.exists() { + return Ok(KompressRuntime { + python_bin: venv_python, + hf_home: hf, + }); + } + + tokio::fs::create_dir_all(&hf) + .await + .with_context(|| format!("creating kompress hf home {}", hf.display()))?; + + log::info!( + "[runtime_python_server::kompress] provisioning venv={} model={}", + venv_dir.display(), + config.tokenjuice.ml_model_id + ); + + let base = PythonBootstrap::new(config.runtime_python.clone()) + .resolve() + .await + .context("resolving base python for kompress venv")?; + + run_step( + &base.python_bin, + &["-m", "venv", &venv_dir.to_string_lossy()], + VENV_TIMEOUT, + &hf, + "create venv", + ) + .await?; + if !venv_python.exists() { + bail!( + "venv created but interpreter missing at {}", + venv_python.display() + ); + } + + install_deps_and_model(&venv_python, &hf, &config.tokenjuice.ml_model_id).await?; + + tokio::fs::write( + marker_path(&venv_dir, &config.tokenjuice.ml_model_id), + base.version.as_bytes(), + ) + .await + .with_context(|| "writing kompress ready marker")?; + + log::info!("[runtime_python_server::kompress] provisioning complete"); + Ok(KompressRuntime { + python_bin: venv_python, + hf_home: hf, + }) +} + +/// Install torch + transformers + the model into an *existing* venv (shared with +/// another backend, e.g. spaCy). Idempotent — a marker next to the interpreter +/// records completion so repeat launches skip the heavy step. +pub async fn install_into(config: &Config, venv_python: &Path) -> Result { + let _guard = provision_lock().await.lock().await; + let hf = hf_home(config); + let shared_marker = venv_python + .parent() + .map(|d| marker_path(d, &config.tokenjuice.ml_model_id)) + .unwrap_or_else(|| marker_path(Path::new("."), &config.tokenjuice.ml_model_id)); + if shared_marker.exists() { + return Ok(hf); + } + tokio::fs::create_dir_all(&hf) + .await + .with_context(|| format!("creating kompress hf home {}", hf.display()))?; + log::info!( + "[runtime_python_server::kompress] installing torch+transformers into shared venv {}", + venv_python.display() + ); + install_deps_and_model(venv_python, &hf, &config.tokenjuice.ml_model_id).await?; + let _ = tokio::fs::write(&shared_marker, config.tokenjuice.ml_model_id.as_bytes()).await; + Ok(hf) +} + +/// pip-install torch (CPU wheel) + transformers, then pre-download the model so +/// the long-lived server can load it offline. +async fn install_deps_and_model(venv_python: &Path, hf: &Path, model_id: &str) -> Result<()> { + run_step( + venv_python, + &["-m", "pip", "install", "--upgrade", "pip"], + PIP_TIMEOUT, + hf, + "pip upgrade", + ) + .await?; + run_step( + venv_python, + &[ + "-m", + "pip", + "install", + "--index-url", + "https://download.pytorch.org/whl/cpu", + "torch", + ], + PIP_TIMEOUT, + hf, + "pip install torch (cpu)", + ) + .await?; + run_step( + venv_python, + &["-m", "pip", "install", "transformers", "tokenizers"], + PIP_TIMEOUT, + hf, + "pip install transformers", + ) + .await?; + // Pre-download the model into the HF cache so server startup loads offline. + let preload = format!( + "from transformers import AutoModel, AutoTokenizer; \ + AutoTokenizer.from_pretrained('{model_id}'); AutoModel.from_pretrained('{model_id}')" + ); + run_step( + venv_python, + &["-c", &preload], + MODEL_TIMEOUT, + hf, + "preload model", + ) + .await?; + Ok(()) +} + +async fn run_step( + python_bin: &Path, + args: &[&str], + timeout: Duration, + hf_home: &Path, + label: &str, +) -> Result<()> { + log::debug!( + "[runtime_python_server::kompress] step `{label}`: {} {:?}", + python_bin.display(), + args + ); + let mut cmd = Command::new(python_bin); + cmd.args(args); + cmd.env("HF_HOME", hf_home); + cmd.env("HF_HUB_DISABLE_TELEMETRY", "1"); + cmd.kill_on_drop(true); + + let output = match tokio::time::timeout(timeout, cmd.output()).await { + Ok(Ok(o)) => o, + Ok(Err(e)) => return Err(e).with_context(|| format!("spawning step `{label}`")), + Err(_) => bail!("step `{label}` timed out after {:?}", timeout), + }; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + let tail: String = stderr + .chars() + .rev() + .take(800) + .collect::() + .chars() + .rev() + .collect(); + bail!("step `{label}` failed (status {}): {tail}", output.status); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn venv_python_path_is_platform_specific() { + let p = venv_python_path(Path::new("/tmp/venv")); + if cfg!(windows) { + assert!(p.ends_with("Scripts/python.exe") || p.ends_with("Scripts\\python.exe")); + } else { + assert_eq!(p, PathBuf::from("/tmp/venv/bin/python")); + } + } + + #[test] + fn provisioned_false_on_clean_config() { + let mut config = Config::default(); + config.runtime_python.cache_dir = "/nonexistent/tj-test".to_string(); + assert!(!kompress_provisioned(&config)); + } + + #[test] + fn ready_marker_is_model_specific_and_path_safe() { + let venv = Path::new("/tmp/openhuman-kompress-test-venv"); + let first = marker_path(venv, "answerdotai/ModernBERT-base"); + let second = marker_path(venv, "other/model"); + + assert_ne!(first, second); + assert_eq!(first.parent(), Some(venv)); + assert!(first + .file_name() + .unwrap() + .to_string_lossy() + .contains("answerdotai_ModernBERT-base")); + } +} diff --git a/src/openhuman/runtime_python_server/mod.rs b/src/openhuman/runtime_python_server/mod.rs index fe5c4d2d0..8253f3a44 100644 --- a/src/openhuman/runtime_python_server/mod.rs +++ b/src/openhuman/runtime_python_server/mod.rs @@ -4,12 +4,14 @@ //! process-level server that keeps Python-backed model modules warm and serves //! Rust callers over a private JSONL stdio protocol. +pub mod kompress; pub mod protocol; pub mod registry; pub mod server; pub mod spacy; pub mod types; +pub use kompress::{ensure_kompress, kompress_provisioned, request_kompress, KompressResponse}; pub use registry::{enabled_backends, RuntimePythonBackend}; pub use server::{ensure_started, status, RuntimePythonServer}; pub use spacy::{ diff --git a/src/openhuman/runtime_python_server/registry.rs b/src/openhuman/runtime_python_server/registry.rs index c5846f276..e34760f4a 100644 --- a/src/openhuman/runtime_python_server/registry.rs +++ b/src/openhuman/runtime_python_server/registry.rs @@ -3,12 +3,15 @@ use crate::openhuman::config::Config; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RuntimePythonBackend { Spacy, + /// TokenJuice ML plain-text compressor ("Kompress", ModernBERT via torch). + Kompress, } impl RuntimePythonBackend { pub fn id(self) -> &'static str { match self { Self::Spacy => "spacy", + Self::Kompress => "kompress", } } } @@ -22,6 +25,9 @@ pub fn enabled_backends(config: &Config) -> Vec { if config.memory_tree.spacy_enabled { backends.push(RuntimePythonBackend::Spacy); } + if config.tokenjuice.ml_compression_enabled { + backends.push(RuntimePythonBackend::Kompress); + } backends } @@ -43,4 +49,26 @@ mod tests { config.memory_tree.spacy_enabled = true; assert_eq!(enabled_backends(&config), vec![RuntimePythonBackend::Spacy]); } + + #[test] + fn registry_includes_kompress_when_enabled() { + let mut config = Config::default(); + config.runtime_python.enabled = true; + config.memory_tree.spacy_enabled = false; + config.tokenjuice.ml_compression_enabled = true; + assert_eq!( + enabled_backends(&config), + vec![RuntimePythonBackend::Kompress] + ); + + config.memory_tree.spacy_enabled = true; + assert_eq!( + enabled_backends(&config), + vec![RuntimePythonBackend::Spacy, RuntimePythonBackend::Kompress] + ); + + // Master runtime switch still gates everything. + config.runtime_python.enabled = false; + assert!(enabled_backends(&config).is_empty()); + } } diff --git a/src/openhuman/runtime_python_server/server.py b/src/openhuman/runtime_python_server/server.py index 79bd1ef41..7015d7f0a 100644 --- a/src/openhuman/runtime_python_server/server.py +++ b/src/openhuman/runtime_python_server/server.py @@ -4,16 +4,37 @@ Private JSONL stdio protocol. Rust owns the process and sends one compact JSON request per line. This server keeps expensive Python backends warm for the life of the Rust core process. + +Enabled backends are passed via the OPENHUMAN_RPS_BACKENDS env var (comma +separated, e.g. "spacy,kompress"). Each backend lazy- or eager-loads its model +as appropriate; spaCy loads at startup (fast), Kompress (torch/ModernBERT) +loads on first request so a missing/slow model never blocks the handshake. """ import json +import os +import re import sys PROTOCOL = 1 SPACY_MODEL = "en_core_web_sm" +_BACKENDS = [ + b.strip() + for b in os.environ.get("OPENHUMAN_RPS_BACKENDS", "spacy").split(",") + if b.strip() +] + _spacy_nlp = None +# Kompress (ModernBERT) lazy-loaded state. +_kompress_tok = None +_kompress_model = None +_KOMPRESS_MODEL = os.environ.get("OPENHUMAN_RPS_KOMPRESS_MODEL", "answerdotai/ModernBERT-base") +_KOMPRESS_DEVICE = os.environ.get("OPENHUMAN_RPS_KOMPRESS_DEVICE", "cpu") +_KOMPRESS_TARGET_RATIO = float(os.environ.get("OPENHUMAN_RPS_KOMPRESS_TARGET_RATIO", "0.5")) +_KOMPRESS_MAX_INPUT = int(os.environ.get("OPENHUMAN_RPS_KOMPRESS_MAX_INPUT_CHARS", "200000")) + def _emit(obj): sys.stdout.write(json.dumps(obj, ensure_ascii=False, separators=(",", ":"))) @@ -32,6 +53,9 @@ def _configure_stdio(): sys.stdout.reconfigure(encoding="utf-8") +# ── spaCy backend ──────────────────────────────────────────────────────── + + def _load_spacy(): global _spacy_nlp if _spacy_nlp is not None: @@ -70,24 +94,118 @@ def _spacy_extract(params): return {"entities": entities, "nouns": nouns} +# ── Kompress backend (ModernBERT salience compressor) ──────────────────── + +_SENT_RE = re.compile(r"(?<=[.!?])\s+|\n+") + + +def _pick_device(): + if _KOMPRESS_DEVICE != "auto": + return _KOMPRESS_DEVICE + try: + import torch + + if torch.cuda.is_available(): + return "cuda" + if getattr(torch.backends, "mps", None) and torch.backends.mps.is_available(): + return "mps" + except Exception: + pass + return "cpu" + + +def _load_kompress(): + global _kompress_tok, _kompress_model + if _kompress_model is not None: + return + import torch # noqa: F401 + from transformers import AutoModel, AutoTokenizer + + device = _pick_device() + _kompress_tok = AutoTokenizer.from_pretrained(_KOMPRESS_MODEL) + _kompress_model = AutoModel.from_pretrained(_KOMPRESS_MODEL).to(device) + _kompress_model.eval() + _kompress_model._oh_device = device + + +def _split_sentences(text): + parts = [p.strip() for p in _SENT_RE.split(text)] + return [p for p in parts if p] + + +def _kompress_compress(params): + params = params or {} + text = params.get("text") or "" + target_ratio = float(params.get("target_ratio", _KOMPRESS_TARGET_RATIO)) + max_input = int(params.get("max_input_chars", _KOMPRESS_MAX_INPUT)) + if len(text) > max_input: + text = text[:max_input] + + sentences = _split_sentences(text) + if len(sentences) <= 3: + return {"compressed_text": text, "input_chars": len(text), "output_chars": len(text)} + + _load_kompress() + import torch + + device = getattr(_kompress_model, "_oh_device", "cpu") + with torch.no_grad(): + enc = _kompress_tok( + sentences, + return_tensors="pt", + padding=True, + truncation=True, + max_length=128, + ).to(device) + out = _kompress_model(**enc) + mask = enc["attention_mask"].unsqueeze(-1).float() + summed = (out.last_hidden_state * mask).sum(dim=1) + counts = mask.sum(dim=1).clamp(min=1.0) + emb = summed / counts + doc_mean = emb.mean(dim=0, keepdim=True) + salience = (emb - doc_mean).norm(dim=1) + + n_keep = max(3, int(round(len(sentences) * target_ratio))) + if n_keep >= len(sentences): + out_text = text + else: + order = salience.argsort(descending=True).tolist() + keep_idx = sorted(order[:n_keep]) + out_text = " ".join(sentences[i] for i in keep_idx) + + return { + "compressed_text": out_text, + "input_chars": len(text), + "output_chars": len(out_text), + } + + +# ── Dispatch ───────────────────────────────────────────────────────────── + + def _handle(req): req_id = req.get("id") method = req.get("method") params = req.get("params") or {} if method == "spacy.extract": return {"id": req_id, "ok": True, "result": _spacy_extract(params)} + if method == "kompress.compress": + return {"id": req_id, "ok": True, "result": _kompress_compress(params)} return _error(req_id, "unknown_method", f"unknown runtime_python_server method: {method}") def main(): _configure_stdio() + # Eager-load only the cheap, fast backends at startup. Kompress (torch) is + # lazy-loaded on first request so a heavy import never blocks the handshake. try: - _load_spacy() + if "spacy" in _BACKENDS: + _load_spacy() except Exception as exc: _emit({"ready": False, "error": f"{type(exc).__name__}: {exc}"}) return 1 - _emit({"ready": True, "protocol": PROTOCOL, "backends": ["spacy"]}) + _emit({"ready": True, "protocol": PROTOCOL, "backends": _BACKENDS}) for line in sys.stdin: line = line.strip() if not line: diff --git a/src/openhuman/runtime_python_server/server.rs b/src/openhuman/runtime_python_server/server.rs index b8508a46d..cd13c34a8 100644 --- a/src/openhuman/runtime_python_server/server.rs +++ b/src/openhuman/runtime_python_server/server.rs @@ -16,7 +16,10 @@ use crate::openhuman::config::Config; use crate::openhuman::runtime_python::process::PythonLaunchSpec; const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(30); -const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); +/// Ceiling for a single backend request. spaCy extraction is sub-100ms; the +/// Kompress (torch) backend can take seconds on CPU, so this is sized for the +/// heavier backend (it's a max, not added latency for fast methods). +const REQUEST_TIMEOUT: Duration = Duration::from_secs(60); const START_FAILURE_BACKOFF: Duration = Duration::from_secs(300); static SERVER: OnceLock> = OnceLock::new(); @@ -40,6 +43,8 @@ struct ServerLaunch { python_bin: PathBuf, script_path: PathBuf, backends: Vec, + /// Extra environment for the worker (backend list + Kompress model/device/HF). + env: Vec<(String, String)>, } struct ServerInner { @@ -94,6 +99,7 @@ fn drain_server_stderr(stderr: ChildStderr) { pub struct RuntimePythonServer { launch: ServerLaunch, inner: Mutex>, + last_used: Mutex, } impl RuntimePythonServer { @@ -102,6 +108,7 @@ impl RuntimePythonServer { Ok(Self { launch, inner: Mutex::new(None), + last_used: Mutex::new(Instant::now()), }) } @@ -120,13 +127,18 @@ impl RuntimePythonServer { T: DeserializeOwned, { match self.request_once(method, params.clone()).await { - Ok(value) => Ok(value), + Ok(value) => { + self.mark_used().await; + Ok(value) + } Err(err) => { log::warn!( "[runtime_python_server] request failed; restarting server before retry: {err:#}" ); self.reset().await; - self.request_once(method, params).await + let value = self.request_once(method, params).await?; + self.mark_used().await; + Ok(value) } } } @@ -208,7 +220,22 @@ impl RuntimePythonServer { async fn reset(&self) { let mut guard = self.inner.lock().await; - *guard = None; + if let Some(mut inner) = guard.take() { + if let Err(error) = inner._child.start_kill() { + log::debug!("[runtime_python_server] failed to signal child shutdown: {error}"); + } + } + } + + async fn mark_used(&self) { + *self.last_used.lock().await = Instant::now(); + } + + async fn kompress_idle_expired(&self, timeout: Duration) -> bool { + self.launch + .backends + .contains(&RuntimePythonBackend::Kompress) + && idle_timeout_expired(*self.last_used.lock().await, timeout) } fn status_from_inner(&self, inner: Option<&ServerInner>) -> RuntimePythonServerStatus { @@ -238,10 +265,45 @@ impl RuntimePythonServer { let guard = self.inner.lock().await; self.status_from_inner(guard.as_ref()) } + + /// The backend set this server was launched for. + pub fn backends(&self) -> &[RuntimePythonBackend] { + &self.launch.backends + } } pub async fn ensure_started(config: &Config) -> Result> { let mut guard = server_slot().lock().await; + // If the enabled backend set changed since the server started (e.g. ML was + // toggled on after a spaCy-only launch), the cached process can't serve the + // new backend — it was never provisioned/launched for it. Rebuild instead of + // reusing a stale launch. + let requested_backends = enabled_backends(config); + let cached = match &*guard { + ServerCache::Ready(existing) => Some(existing.clone()), + ServerCache::Empty | ServerCache::Failed { .. } => None, + }; + if let Some(existing) = cached { + if existing.backends() != requested_backends.as_slice() { + log::info!( + "[runtime_python_server] backend set changed ({:?} -> {:?}); rebuilding server", + existing.backends(), + requested_backends + ); + existing.reset().await; + *guard = ServerCache::Empty; + } else { + let idle_timeout = Duration::from_secs(config.tokenjuice.ml_sidecar_idle_timeout_secs); + if existing.kompress_idle_expired(idle_timeout).await { + log::info!( + "[runtime_python_server] kompress backend idle for >= {:?}; rebuilding server", + idle_timeout + ); + existing.reset().await; + *guard = ServerCache::Empty; + } + } + } match &*guard { ServerCache::Ready(existing) => { let existing = existing.clone(); @@ -287,6 +349,10 @@ pub async fn ensure_started(config: &Config) -> Result> } } +fn idle_timeout_expired(last_used: Instant, timeout: Duration) -> bool { + last_used.elapsed() >= timeout +} + async fn start_new_server(config: &Config) -> Result> { let server = Arc::new(RuntimePythonServer::new(config).await?); server.start().await?; @@ -318,29 +384,76 @@ async fn prepare_launch(config: &Config) -> Result { bail!("no runtime python server backends enabled"); } - let spacy_runtime = if backends.contains(&RuntimePythonBackend::Spacy) { - Some(super::spacy::ensure_spacy(config).await?) - } else { - None - }; + let want_spacy = backends.contains(&RuntimePythonBackend::Spacy); + let want_kompress = backends.contains(&RuntimePythonBackend::Kompress); - let python_bin = if let Some(spacy_runtime) = spacy_runtime { + // The server runs ONE interpreter, so the chosen venv must satisfy every + // enabled backend. spaCy (if enabled) owns the venv; Kompress then installs + // torch+transformers into it. If only Kompress is enabled it owns its venv. + let mut env: Vec<(String, String)> = Vec::new(); + let python_bin = if want_spacy { + let spacy_runtime = super::spacy::ensure_spacy(config).await?; + if want_kompress { + let hf = super::kompress::install_into(config, &spacy_runtime.python_bin).await?; + push_kompress_env(&mut env, config, &hf); + } spacy_runtime.python_bin + } else if want_kompress { + let rt = super::kompress::ensure_kompress(config).await?; + push_kompress_env(&mut env, config, &rt.hf_home); + rt.python_bin } else { crate::openhuman::runtime_python::PythonBootstrap::new(config.runtime_python.clone()) .resolve() .await? .python_bin }; + + env.push(( + "OPENHUMAN_RPS_BACKENDS".to_string(), + backends + .iter() + .map(|b| b.id()) + .collect::>() + .join(","), + )); + let script_path = write_server_script(config).await?; Ok(ServerLaunch { python_bin, script_path, backends, + env, }) } +/// Environment the worker needs to load + run the Kompress model offline. +fn push_kompress_env(env: &mut Vec<(String, String)>, config: &Config, hf_home: &std::path::Path) { + env.push(( + "OPENHUMAN_RPS_KOMPRESS_MODEL".to_string(), + config.tokenjuice.ml_model_id.clone(), + )); + env.push(( + "OPENHUMAN_RPS_KOMPRESS_DEVICE".to_string(), + config.tokenjuice.ml_device.clone(), + )); + env.push(( + "OPENHUMAN_RPS_KOMPRESS_TARGET_RATIO".to_string(), + config.tokenjuice.ml_target_ratio.to_string(), + )); + env.push(( + "OPENHUMAN_RPS_KOMPRESS_MAX_INPUT_CHARS".to_string(), + config.tokenjuice.ml_max_input_chars.to_string(), + )); + env.push(("HF_HOME".to_string(), hf_home.display().to_string())); + // Model was pre-downloaded during provisioning; load offline so startup + // never blocks on the network. + env.push(("HF_HUB_OFFLINE".to_string(), "1".to_string())); + env.push(("TRANSFORMERS_OFFLINE".to_string(), "1".to_string())); + env.push(("HF_HUB_DISABLE_TELEMETRY".to_string(), "1".to_string())); +} + async fn write_server_script(config: &Config) -> Result { let root = super::spacy::python_server_cache_root(config); tokio::fs::create_dir_all(&root) @@ -375,7 +488,10 @@ async fn spawn_inner(launch: &ServerLaunch) -> Result { version: "runtime-backend".to_string(), source: crate::openhuman::runtime_python::PythonSource::Managed, }; - let spec = PythonLaunchSpec::new(launch.script_path.clone()); + let mut spec = PythonLaunchSpec::new(launch.script_path.clone()); + for (key, value) in &launch.env { + spec.env.insert(key.clone(), value.clone()); + } let mut child = crate::openhuman::runtime_python::process::spawn_stdio_process(&resolved, &spec) .context("spawning runtime python server")?; @@ -439,6 +555,23 @@ pub async fn request_spacy_extract( .await } +pub async fn request_kompress_compress( + config: &Config, + text: &str, +) -> Result { + let server = ensure_started(config).await?; + server + .request( + "kompress.compress", + json!({ + "text": text, + "target_ratio": config.tokenjuice.ml_target_ratio, + "max_input_chars": config.tokenjuice.ml_max_input_chars, + }), + ) + .await +} + #[cfg(test)] mod tests { use super::*; @@ -450,4 +583,16 @@ mod tests { let err = prepare_launch(&config).await.unwrap_err().to_string(); assert!(err.contains("no runtime python server backends enabled")); } + + #[test] + fn idle_timeout_expiry_uses_last_used_instant() { + assert!(idle_timeout_expired( + Instant::now() - Duration::from_secs(10), + Duration::from_secs(5), + )); + assert!(!idle_timeout_expired( + Instant::now(), + Duration::from_secs(5), + )); + } } diff --git a/src/openhuman/tokenjuice/cache/marker.rs b/src/openhuman/tokenjuice/cache/marker.rs new file mode 100644 index 000000000..716ab1757 --- /dev/null +++ b/src/openhuman/tokenjuice/cache/marker.rs @@ -0,0 +1,133 @@ +//! CCR retrieval markers. +//! +//! When the router offloads an original to the [`super::store`], it embeds a +//! marker carrying the CCR token so the model knows the content is recoverable +//! and how to fetch it. The canonical marker is `⟦tj:⟧`; for backward +//! compatibility we also parse the legacy `retrieve_tool_output("")` form +//! that older histories may still contain. + +/// The retrieve tool's name, surfaced in footers and used by the harness to +/// keep the tool's own output from being re-compacted and to always advertise it. +pub const RETRIEVE_TOOL_NAME: &str = "tokenjuice_retrieve"; + +/// The legacy retrieve tool name (kept as an alias during migration). +pub const LEGACY_RETRIEVE_TOOL_NAME: &str = "retrieve_tool_output"; + +/// All CCR recovery tool names. Both must be (a) always advertised to every +/// agent — any agent that sees a retrieval footer must be able to call the tool +/// — and (b) never re-compacted (their job is to return an original in full). +pub const RECOVERY_TOOL_NAMES: &[&str] = &[RETRIEVE_TOOL_NAME, LEGACY_RETRIEVE_TOOL_NAME]; + +/// Tools whose output must never be re-compacted. See [`RECOVERY_TOOL_NAMES`]. +pub const NEVER_COMPACT_TOOLS: &[&str] = RECOVERY_TOOL_NAMES; + +/// True if `tool_name` is one of the CCR recovery tools. +pub fn is_recovery_tool(tool_name: &str) -> bool { + RECOVERY_TOOL_NAMES.contains(&tool_name) +} + +/// Format the canonical inline marker for a CCR `hash`. +pub fn format_marker(hash: &str) -> String { + format!("⟦tj:{hash}⟧") +} + +/// Build the human-facing recovery footer appended to compacted output. +/// +/// `lossy` distinguishes a partial view (data dropped) from a faithful reformat +/// (no data lost, layout changed); both offer exact recovery. +pub fn recovery_footer(hash: &str, original_bytes: usize, lossy: bool) -> String { + let marker = format_marker(hash); + if lossy { + format!( + "\n\n[compacted tool output — this is a PARTIAL view; the full original \ + ({original_bytes} bytes) is available by calling {RETRIEVE_TOOL_NAME} with \ + token \"{hash}\" (marker {marker})]" + ) + } else { + format!( + "\n\n[reformatted tool output — no data lost, but layout changed; the exact \ + original ({original_bytes} bytes) is available by calling {RETRIEVE_TOOL_NAME} \ + with token \"{hash}\" (marker {marker})]" + ) + } +} + +/// Extract all CCR tokens referenced in `text`, from both the canonical +/// `⟦tj:⟧` markers and the legacy `retrieve_tool_output("")` form. +/// Order-preserving, de-duplicated. +pub fn parse_markers(text: &str) -> Vec { + let mut out: Vec = Vec::new(); + let mut push = |h: &str| { + let h = h.trim(); + if !h.is_empty() && !out.iter().any(|e| e == h) { + out.push(h.to_string()); + } + }; + + // Canonical: ⟦tj:HASH⟧ + let mut rest = text; + while let Some(start) = rest.find("⟦tj:") { + let after = &rest[start + "⟦tj:".len()..]; + if let Some(end) = after.find('⟧') { + push(&after[..end]); + rest = &after[end..]; + } else { + break; + } + } + + // Legacy: retrieve_tool_output("HASH") or tokenjuice_retrieve("HASH") + for needle in [ + "retrieve_tool_output(\"", + "tokenjuice_retrieve(\"", + "token \"", + ] { + let mut rest = text; + while let Some(start) = rest.find(needle) { + let after = &rest[start + needle.len()..]; + if let Some(end) = after.find('"') { + push(&after[..end]); + rest = &after[end..]; + } else { + break; + } + } + } + + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn formats_and_parses_canonical() { + let m = format_marker("ab12cd34"); + assert_eq!(m, "⟦tj:ab12cd34⟧"); + assert_eq!( + parse_markers(&format!("see {m} for more")), + vec!["ab12cd34"] + ); + } + + #[test] + fn parses_legacy_form() { + let text = "partial; call retrieve_tool_output(\"deadbeef00\") to recover"; + assert_eq!(parse_markers(text), vec!["deadbeef00"]); + } + + #[test] + fn parses_multiple_dedup() { + let text = "⟦tj:aaa⟧ and ⟦tj:bbb⟧ and again ⟦tj:aaa⟧"; + assert_eq!(parse_markers(text), vec!["aaa", "bbb"]); + } + + #[test] + fn footer_carries_token() { + let f = recovery_footer("c0ffee", 1234, true); + assert!(f.contains("PARTIAL view")); + assert!(f.contains("c0ffee")); + assert_eq!(parse_markers(&f), vec!["c0ffee"]); + } +} diff --git a/src/openhuman/tokenjuice/cache/mod.rs b/src/openhuman/tokenjuice/cache/mod.rs new file mode 100644 index 000000000..2e09eb04a --- /dev/null +++ b/src/openhuman/tokenjuice/cache/mod.rs @@ -0,0 +1,13 @@ +//! CCR (Compress-Cache-Retrieve) — original storage + retrieval markers. + +pub mod marker; +pub mod store; + +pub use marker::{ + format_marker, is_recovery_tool, parse_markers, recovery_footer, LEGACY_RETRIEVE_TOOL_NAME, + NEVER_COMPACT_TOOLS, RECOVERY_TOOL_NAMES, RETRIEVE_TOOL_NAME, +}; +pub use store::{ + configure, disable_disk_tier, enable_disk_tier, offload, offload_checked, retrieve, + retrieve_range, short_hash, stats, RangeUnit, +}; diff --git a/src/openhuman/tokenjuice/cache/store.rs b/src/openhuman/tokenjuice/cache/store.rs new file mode 100644 index 000000000..be7d84771 --- /dev/null +++ b/src/openhuman/tokenjuice/cache/store.rs @@ -0,0 +1,474 @@ +//! CCR — Compress-Cache-Retrieve store. +//! +//! When a compressor drops data (lossy paths), the router stows the original +//! here keyed by a short content hash and embeds a retrieval marker in the +//! compacted text (see [`super::marker`]). The agent calls the +//! `tokenjuice_retrieve` tool to get the original back on demand — so even +//! aggressive compaction stays reversible and is safe under the always-on +//! default. +//! +//! Process-global and bounded by **both** an entry count and a total byte cap, +//! with an optional TTL. Keyed by content hash, so re-offloading identical +//! content is idempotent. An optional **disk tier** (configured from the core's +//! `workspace_dir`) persists originals across the session so retrieval can +//! survive memory eviction; the agent itself never writes there — only the core +//! does, through this module. + +use sha2::{Digest, Sha256}; +use std::collections::{HashMap, VecDeque}; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, OnceLock, RwLock}; +use std::time::{Duration, Instant}; + +/// Default max originals retained (entry-count cap). +pub const DEFAULT_MAX_ENTRIES: usize = 256; +/// Default total-bytes cap (64 MiB) so a few huge originals can't blow memory. +pub const DEFAULT_MAX_BYTES: usize = 64 * 1024 * 1024; +/// Bytes of the SHA-256 digest used for the key (→ 32 hex chars). Wide enough +/// that collisions are infeasible and the hash doubles as an unguessable +/// capability token. +const HASH_BYTES: usize = 16; + +/// Tunable limits, settable once at startup from the `[tokenjuice]` config. +struct Limits { + max_entries: usize, + max_bytes: usize, + ttl: Option, +} + +impl Default for Limits { + fn default() -> Self { + Self { + max_entries: DEFAULT_MAX_ENTRIES, + max_bytes: DEFAULT_MAX_BYTES, + ttl: None, + } + } +} + +fn limits() -> &'static RwLock { + static LIMITS: OnceLock> = OnceLock::new(); + LIMITS.get_or_init(|| RwLock::new(Limits::default())) +} + +/// Optional on-disk tier root (under the core's workspace). `None` ⇒ in-memory only. +fn disk_root() -> &'static RwLock> { + static ROOT: OnceLock>> = OnceLock::new(); + ROOT.get_or_init(|| RwLock::new(None)) +} + +/// Configure the cache limits (called once from config at startup). +pub fn configure(max_entries: usize, max_bytes: usize, ttl_secs: Option) { + let mut l = limits().write().unwrap_or_else(|p| p.into_inner()); + l.max_entries = max_entries.max(1); + l.max_bytes = max_bytes.max(1); + l.ttl = ttl_secs.map(Duration::from_secs); +} + +/// Enable the on-disk tier rooted at `root` (e.g. `/.tokenjuice/ccr`). +/// Best-effort: directory creation failures disable the tier silently. +pub fn enable_disk_tier(root: PathBuf) { + if std::fs::create_dir_all(&root).is_ok() { + *disk_root().write().unwrap_or_else(|p| p.into_inner()) = Some(root); + } else { + log::warn!("[tokenjuice][ccr] could not create disk tier at {root:?}"); + } +} + +/// Turn the on-disk tier off (e.g. when the setting is toggled off at runtime). +/// New offloads stop writing to disk; already-written files are left in place. +pub fn disable_disk_tier() { + *disk_root().write().unwrap_or_else(|p| p.into_inner()) = None; +} + +struct Entry { + content: String, + created: Instant, +} + +#[derive(Default)] +struct Inner { + map: HashMap, + order: VecDeque, + total_bytes: usize, +} + +impl Inner { + /// Insert `content` under `hash` (idempotent) and evict (FIFO) until both + /// the entry-count and total-byte caps hold. + /// + /// Returns whether `hash` is still resident after eviction. A single + /// original larger than `max_bytes` cannot be retained in memory under the + /// byte cap — eviction would immediately drop the just-inserted entry — so + /// `false` is returned and the caller must not advertise it as recoverable + /// (the router declines lossy compaction or relies on the disk tier). + fn insert( + &mut self, + hash: String, + content: String, + max_entries: usize, + max_bytes: usize, + ) -> bool { + if let Some(entry) = self.map.get_mut(&hash) { + entry.created = Instant::now(); + self.order.retain(|candidate| candidate != &hash); + self.order.push_back(hash); + return true; + } + let bytes = content.len(); + self.total_bytes += bytes; + self.map.insert( + hash.clone(), + Entry { + content, + created: Instant::now(), + }, + ); + self.order.push_back(hash.clone()); + while self.order.len() > max_entries || self.total_bytes > max_bytes { + // Never evict the entry we just inserted to satisfy the cap when it + // is the only thing keeping us over: that would make the original + // unrecoverable the instant its footer is emitted. Stop and report + // non-retention instead (one oversized item is rejected, not the + // whole store wiped). + if self.order.len() == 1 { + break; + } + let Some(evicted) = self.order.pop_front() else { + break; + }; + if let Some(e) = self.map.remove(&evicted) { + self.total_bytes = self.total_bytes.saturating_sub(e.content.len()); + } + } + // Retained iff still present (an oversized single entry over the byte + // cap is dropped below) AND within the byte cap. + if self.total_bytes > max_bytes { + if let Some(e) = self.map.remove(&hash) { + self.total_bytes = self.total_bytes.saturating_sub(e.content.len()); + } + self.order.retain(|h| h != &hash); + return false; + } + self.map.contains_key(&hash) + } +} + +fn global() -> &'static Mutex { + static STORE: OnceLock> = OnceLock::new(); + STORE.get_or_init(|| Mutex::new(Inner::default())) +} + +/// Stash `content` and return its short hash. Idempotent for identical content. +pub fn offload(content: &str) -> String { + offload_checked(content).0 +} + +/// Stash `content`, returning `(hash, retained)`. `retained` is `false` only +/// when the original could be kept neither in memory (it exceeds the byte cap) +/// nor on the disk tier — in which case the caller must NOT advertise it as +/// recoverable. Idempotent for identical content. +pub fn offload_checked(content: &str) -> (String, bool) { + let hash = short_hash(content); + let (max_entries, max_bytes) = { + let l = limits().read().unwrap_or_else(|p| p.into_inner()); + (l.max_entries, l.max_bytes) + }; + let mem_retained = global().lock().unwrap_or_else(|p| p.into_inner()).insert( + hash.clone(), + content.to_string(), + max_entries, + max_bytes, + ); + + // Mirror to the disk tier when enabled (best-effort). A successful disk + // write keeps the original recoverable even when it was too big for memory. + // Rewriting an existing hash intentionally refreshes the file mtime, which + // is the TTL clock for disk-backed CCR entries. + let mut disk_retained = false; + if let Some(root) = disk_root() + .read() + .unwrap_or_else(|p| p.into_inner()) + .clone() + { + let path = root.join(&hash); + match std::fs::write(&path, content) { + Ok(()) => disk_retained = true, + Err(e) => log::debug!("[tokenjuice][ccr] disk write failed for {hash}: {e}"), + } + } + (hash, mem_retained || disk_retained) +} + +/// True if `hash` is a well-formed CCR token (exactly the generated hex digest). +/// Tokens come from agent-controlled tool args, and on a disk-tier miss they are +/// joined onto the CCR root — so anything other than the fixed hex shape (e.g. +/// `../../state/config.toml`) must be rejected before touching the filesystem to +/// prevent path traversal / arbitrary file reads through the recovery tool. +fn is_valid_token(hash: &str) -> bool { + hash.len() == HASH_BYTES * 2 && hash.bytes().all(|b| b.is_ascii_hexdigit()) +} + +/// Retrieve a previously-offloaded original by hash, if still available +/// (memory first, then the disk tier). Honours the TTL for both tiers. +pub fn retrieve(hash: &str) -> Option { + // Reject anything that isn't the generated token shape up front — guards the + // disk-tier `root.join(hash)` below against path traversal. + if !is_valid_token(hash) { + return None; + } + let ttl = limits().read().unwrap_or_else(|p| p.into_inner()).ttl; + { + let mut inner = global().lock().unwrap_or_else(|p| p.into_inner()); + if let Some(entry) = inner.map.get(hash) { + if ttl.is_none_or(|t| entry.created.elapsed() < t) { + return Some(entry.content.clone()); + } + // Expired — drop it and fall through to disk. + if let Some(e) = inner.map.remove(hash) { + inner.total_bytes = inner.total_bytes.saturating_sub(e.content.len()); + } + } + } + // Disk fallback. + let root = disk_root() + .read() + .unwrap_or_else(|p| p.into_inner()) + .clone()?; + let path = root.join(hash); + if disk_entry_expired(&path, ttl) { + log::debug!("[tokenjuice][ccr] disk entry expired for {hash}"); + let _ = std::fs::remove_file(&path); + return None; + } + std::fs::read_to_string(path).ok() +} + +fn disk_entry_expired(path: &Path, ttl: Option) -> bool { + let Some(ttl) = ttl else { + return false; + }; + let Ok(metadata) = std::fs::metadata(path) else { + return false; + }; + let Ok(modified) = metadata.modified() else { + return false; + }; + modified.elapsed().is_ok_and(|elapsed| elapsed >= ttl) +} + +/// The span/unit for a ranged retrieval. +#[derive(Debug, Clone, Copy)] +pub enum RangeUnit { + Bytes, + Lines, +} + +/// Retrieve a slice of a previously-offloaded original. `start`/`end` are +/// 0-based, `end` exclusive; out-of-bounds ends are clamped. Returns `None` +/// only when the original isn't available at all. +pub fn retrieve_range(hash: &str, start: usize, end: usize, unit: RangeUnit) -> Option { + let original = retrieve(hash)?; + if end <= start { + return Some(String::new()); + } + match unit { + RangeUnit::Bytes => { + // Clamp to char boundaries so we never split a UTF-8 sequence. + let s = floor_char_boundary(&original, start.min(original.len())); + let e = floor_char_boundary(&original, end.min(original.len())); + Some(original[s..e].to_string()) + } + RangeUnit::Lines => { + let lines: Vec<&str> = original.lines().collect(); + let e = end.min(lines.len()); + if start >= lines.len() { + return Some(String::new()); + } + Some(lines[start..e].join("\n")) + } + } +} + +/// Largest char boundary ≤ `idx` (std's `floor_char_boundary` is still nightly). +fn floor_char_boundary(s: &str, idx: usize) -> usize { + if idx >= s.len() { + return s.len(); + } + let mut i = idx; + while i > 0 && !s.is_char_boundary(i) { + i -= 1; + } + i +} + +/// Short hex content hash used as the CCR key/token. +pub fn short_hash(content: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(content.as_bytes()); + let digest = hasher.finalize(); + hex::encode(&digest[..HASH_BYTES]) +} + +/// Snapshot of cache occupancy for the debug controller / stats. +pub fn stats() -> (usize, usize) { + let inner = global().lock().unwrap_or_else(|p| p.into_inner()); + (inner.map.len(), inner.total_bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trips() { + let original = "ccr round-trip unique payload alpha ".repeat(50); + let hash = offload(&original); + assert_eq!(hash.len(), HASH_BYTES * 2); + assert_eq!(retrieve(&hash).as_deref(), Some(original.as_str())); + } + + #[test] + fn idempotent_hash() { + let a = offload("ccr idempotent unique payload bravo content"); + let b = offload("ccr idempotent unique payload bravo content"); + assert_eq!(a, b); + } + + #[test] + fn missing_hash_is_none() { + assert!(retrieve("ffffffffffffffffffffffffffffffff").is_none()); + } + + #[test] + fn byte_cap_evicts_oldest() { + let mut inner = Inner::default(); + // 10 entries of 100 bytes; cap at 500 bytes ⇒ keep ~5 newest. + for i in 0..10 { + inner.insert(format!("h{i}"), "x".repeat(100), 1000, 500); + } + assert!( + inner.total_bytes <= 500, + "byte cap held: {}", + inner.total_bytes + ); + assert!(!inner.map.contains_key("h0"), "oldest evicted"); + assert!(inner.map.contains_key("h9"), "newest retained"); + } + + #[test] + fn oversized_single_entry_is_not_retained() { + // One entry larger than the byte cap can't be kept; insert reports + // non-retention so the router won't advertise it as recoverable. + let mut inner = Inner::default(); + let retained = inner.insert("big".into(), "x".repeat(200), 100, 100); + assert!(!retained, "oversized single entry must report not-retained"); + assert!(!inner.map.contains_key("big")); + assert_eq!(inner.total_bytes, 0); + } + + #[test] + fn rejects_path_traversal_tokens() { + // Non-hex / wrong-length tokens are rejected before any disk join. + assert!(!is_valid_token("../../state/config.toml")); + assert!(!is_valid_token("..%2f..%2fetc")); + assert!(!is_valid_token("deadbeef")); // too short + assert!(!is_valid_token(&"g".repeat(32))); // non-hex + assert!(is_valid_token(&"a1b2c3d4".repeat(4))); // 32 hex chars + // retrieve() returns None for an invalid token regardless of cache state. + assert!(retrieve("../../state/config.toml").is_none()); + } + + #[test] + fn within_cap_entry_is_retained() { + let mut inner = Inner::default(); + assert!(inner.insert("ok".into(), "x".repeat(50), 100, 100)); + assert!(inner.map.contains_key("ok")); + } + + #[test] + fn entry_cap_evicts_oldest() { + let mut inner = Inner::default(); + for i in 0..60 { + inner.insert(format!("e{i}"), format!("content-{i}"), 50, usize::MAX); + } + assert!(inner.map.len() <= 50); + assert!(!inner.map.contains_key("e0")); + } + + #[test] + fn reoffloading_existing_entry_refreshes_ttl_and_order() { + let mut inner = Inner::default(); + assert!(inner.insert("old".into(), "old payload".into(), 2, usize::MAX)); + assert!(inner.insert("fresh".into(), "fresh payload".into(), 2, usize::MAX)); + let stale_created = Instant::now() - Duration::from_secs(60); + inner.map.get_mut("old").unwrap().created = stale_created; + + assert!(inner.insert("old".into(), "old payload".into(), 2, usize::MAX)); + assert!( + inner.map["old"].created > stale_created, + "existing entry timestamp should refresh" + ); + assert_eq!(inner.order.back().map(String::as_str), Some("old")); + + assert!(inner.insert("third".into(), "third payload".into(), 2, usize::MAX)); + assert!(inner.map.contains_key("old")); + assert!(!inner.map.contains_key("fresh")); + } + + #[test] + fn range_retrieval_lines_and_bytes() { + let original = "line0\nline1\nline2\nline3\nline4"; + let hash = offload(original); + assert_eq!( + retrieve_range(&hash, 1, 3, RangeUnit::Lines).as_deref(), + Some("line1\nline2") + ); + assert_eq!( + retrieve_range(&hash, 0, 5, RangeUnit::Bytes).as_deref(), + Some("line0") + ); + // Out-of-bounds end clamps. + assert_eq!( + retrieve_range(&hash, 4, 999, RangeUnit::Lines).as_deref(), + Some("line4") + ); + } + + #[test] + fn disk_tier_survives_memory_miss() { + let dir = std::env::temp_dir().join(format!("tj-ccr-{}", short_hash("disk-test-seed"))); + let _ = std::fs::remove_dir_all(&dir); + enable_disk_tier(dir.clone()); + let original = "disk tier unique payload charlie ".repeat(40); + let hash = offload(&original); + // Simulate memory eviction by clearing the in-memory map directly. + { + let mut inner = global().lock().unwrap_or_else(|p| p.into_inner()); + inner.map.remove(&hash); + } + assert_eq!( + retrieve(&hash).as_deref(), + Some(original.as_str()), + "disk fallback" + ); + // Disable the tier for other tests and clean up. + *disk_root().write().unwrap() = None; + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn disk_entry_ttl_uses_file_mtime() { + let dir = std::env::temp_dir().join(format!("tj-ccr-ttl-{}", short_hash("disk-ttl"))); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + std::fs::write(&path, "disk ttl payload").unwrap(); + + assert!(!disk_entry_expired(&path, None)); + assert!(!disk_entry_expired(&path, Some(Duration::from_secs(60)))); + assert!(disk_entry_expired(&path, Some(Duration::ZERO))); + + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/src/openhuman/tokenjuice/compress.rs b/src/openhuman/tokenjuice/compress.rs new file mode 100644 index 000000000..727f857a1 --- /dev/null +++ b/src/openhuman/tokenjuice/compress.rs @@ -0,0 +1,230 @@ +//! Universal content-aware compression entry point. +//! +//! [`compress_content`] is the broadly-usable function: hand it any blob and an +//! optional [`ContentHint`], and it detects the content kind, routes to the +//! right compressor, and — when the result drops data — offloads the original +//! to the CCR cache and appends a `⟦tj:⟧` retrieval footer so nothing is +//! ever silently lost. Use it for tool output, file reads, web/HTML fetches, or +//! any large payload headed for the model context. +//! +//! The tool-output adapter [`crate::openhuman::tokenjuice::compact_tool_output`] +//! builds a [`CompressInput`] with a derived command/argv and calls [`route`]. + +use crate::openhuman::tokenjuice::cache; +use crate::openhuman::tokenjuice::compressors::{compressor_for, generic_compressor}; +use crate::openhuman::tokenjuice::detect::detect_content_kind; +use crate::openhuman::tokenjuice::savings; +use crate::openhuman::tokenjuice::tokens::estimate_tokens; +use crate::openhuman::tokenjuice::types::{ + CompressInput, CompressOptions, CompressOutput, CompressedOutput, ContentHint, ContentKind, +}; + +/// Compress arbitrary content. Detects the kind (honouring `hint`), routes to +/// the matching compressor, and offloads/marks the original via CCR when lossy. +/// +/// Always pass-through safe: returns the original unchanged when the router is +/// disabled, the input is too small, the content kind has no enabled +/// compressor, or compression wouldn't shrink it. +pub async fn compress_content( + content: &str, + hint: Option, + opts: &CompressOptions, +) -> CompressedOutput { + let hint = hint.unwrap_or_default(); + let input = CompressInput { + content, + kind: ContentKind::PlainText, // resolved inside route() + hint: &hint, + exit_code: None, + command: None, + argv: None, + original_bytes: content.len(), + }; + route(input, opts).await +} + +/// Core router: detect (unless the input already carries a resolved kind via the +/// hint's explicit override), pick the compressor honouring config gates, run +/// it, and apply CCR offload + footer. +pub async fn route(mut input: CompressInput<'_>, opts: &CompressOptions) -> CompressedOutput { + let content = input.content; + let original_bytes = content.len(); + + if !opts.router_enabled || original_bytes < opts.min_bytes_to_compress { + let kind = detect_content_kind(content, input.hint); + return CompressedOutput::passthrough(content.to_string(), kind); + } + + let kind = detect_content_kind(content, input.hint); + input.kind = kind; + + // Resolve which compressor to try, honouring per-kind config gates. + let primary: Option<&'static dyn crate::openhuman::tokenjuice::compressors::Compressor> = + match kind { + ContentKind::Search if !opts.search_enabled => None, + ContentKind::Code if !opts.code_enabled => None, + ContentKind::Html if !opts.html_enabled => None, + _ => Some(compressor_for(kind)), + }; + + // Try the primary compressor; if it declines, fall back to the generic + // head/tail path (which itself declines for non-command payloads). + let mut produced: Option = match primary { + Some(c) => c.compress(&input, opts).await, + None => None, + }; + // When the specialised compressor declines (including plain text with the + // ML compressor off), fall back to the generic head/tail path. It runs the + // rule engine for *command* output (so e.g. `git status` still compacts even + // though it carries no log signal) and declines for domain-tool payloads. + if produced.is_none() { + produced = generic_compressor().compress(&input, opts).await; + } + + let Some(out) = produced else { + return CompressedOutput::passthrough(content.to_string(), kind); + }; + if out.text.len() >= original_bytes { + return CompressedOutput::passthrough(content.to_string(), kind); + } + + // CCR threshold: only offload (and therefore only allow *lossy* compaction) + // when the input is large enough to be worth caching. Below the token + // threshold a lossy result can't be made recoverable, so pass it through; + // lossless reformats are still allowed without an offload. + let original_tokens = estimate_tokens(content); + let ccr_for_call = opts.ccr_enabled && original_tokens as usize >= opts.ccr_min_tokens; + if out.lossy && !ccr_for_call { + return CompressedOutput::passthrough(content.to_string(), kind); + } + + // Offload the original and append a recovery footer when CCR is in play. + let (text, ccr_token) = if ccr_for_call { + let (token, retained) = cache::offload_checked(content); + if !retained { + // The original is too large to keep in memory (over the byte cap) + // and the disk tier isn't on, so it can't be recovered. A lossy view + // would be irreversible — decline it. A lossless reformat is still + // safe to return, just without a (dangling) recovery footer. + if out.lossy { + return CompressedOutput::passthrough(content.to_string(), kind); + } + (out.text, None) + } else { + let footer = cache::recovery_footer(&token, original_bytes, out.lossy); + let mut text = out.text; + text.push_str(&footer); + // The footer adds bytes — if it tipped us over the original size, bail. + if text.len() >= original_bytes { + return CompressedOutput::passthrough(content.to_string(), kind); + } + (text, Some(token)) + } + } else { + (out.text, None) + }; + + let compacted_bytes = text.len(); + let compacted_tokens = estimate_tokens(&text); + log::info!( + "[tokenjuice] compacted kind={} compressor={} lossy={} {}->{} bytes (~{}->{} tok)", + kind.as_str(), + out.kind.as_str(), + out.lossy, + original_bytes, + compacted_bytes, + original_tokens, + compacted_tokens, + ); + + // Record savings for the dashboard (tokens + cost saved for the LLM the + // result is being compressed for). + savings::record(kind, out.kind, original_tokens, compacted_tokens); + + CompressedOutput { + text, + content_kind: kind, + compressor: out.kind, + lossy: out.lossy, + applied: true, + ccr_token, + original_bytes, + compacted_bytes, + } +} + +/// Build a [`CompressorKind`] label-free passthrough quickly (used by callers +/// that only need to detect without compressing). +pub fn detect_only(content: &str, hint: &ContentHint) -> ContentKind { + detect_content_kind(content, hint) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::tokenjuice::types::CompressorKind; + + fn opts() -> CompressOptions { + CompressOptions { + min_bytes_to_compress: 64, + ..Default::default() + } + } + + #[tokio::test] + async fn routes_json_and_offloads() { + let mut rows = Vec::new(); + for i in 0..120 { + rows.push(format!( + r#"{{"id":{i},"name":"account_{i}","email":"a{i}@ex.com","tier":"gold"}}"# + )); + } + let original = format!("[{}]", rows.join(",")); + let res = compress_content(&original, None, &opts()).await; + assert!(res.applied); + assert_eq!(res.content_kind, ContentKind::Json); + assert_eq!(res.compressor, CompressorKind::SmartCrusher); + assert!(res.text.len() < original.len()); + let token = res.ccr_token.expect("offloaded"); + assert_eq!(cache::retrieve(&token).as_deref(), Some(original.as_str())); + assert!( + res.text.contains("⟦tj:"), + "footer marker present: {}", + res.text + ); + } + + #[tokio::test] + async fn small_input_passes_through() { + let res = compress_content("tiny", None, &opts()).await; + assert!(!res.applied); + assert_eq!(res.text, "tiny"); + } + + #[tokio::test] + async fn html_hint_extracts_text() { + let mut html = String::from(""); + for i in 0..60 { + html.push_str(&format!("
cell number {i} content
")); + } + html.push_str(""); + let hint = ContentHint { + mime: Some("text/html".into()), + ..Default::default() + }; + let res = compress_content(&html, Some(hint), &opts()).await; + assert!(res.applied); + assert_eq!(res.content_kind, ContentKind::Html); + assert!(res.text.contains("cell number 7 content")); + } + + #[tokio::test] + async fn router_disabled_is_passthrough() { + let mut o = opts(); + o.router_enabled = false; + let big = "x".repeat(5000); + let res = compress_content(&big, None, &o).await; + assert!(!res.applied); + assert_eq!(res.text, big); + } +} diff --git a/src/openhuman/tokenjuice/compressors/code.rs b/src/openhuman/tokenjuice/compressors/code.rs new file mode 100644 index 000000000..41828b4ca --- /dev/null +++ b/src/openhuman/tokenjuice/compressors/code.rs @@ -0,0 +1,359 @@ +//! Source-code compressor — keep signatures, collapse bodies. +//! +//! Inspired by Headroom's `CodeCompressor`. The goal is to keep the structural +//! skeleton an agent needs to navigate a file — imports, type/function/class +//! signatures, top-level constants — while collapsing the deep bodies that +//! dominate byte count. +//! +//! This module currently ships the language-agnostic **brace-depth heuristic**: +//! lines at brace nesting depth 0–1 are kept; deeper bodies collapse to a +//! `{ … N lines … }` placeholder. Lines carrying error/TODO markers are always +//! kept. A higher-fidelity tree-sitter path (Rust/TS/Python) is layered on in a +//! follow-up slice and selected by language; until then every language uses the +//! heuristic. The router offloads the original to CCR for exact recovery. + +use async_trait::async_trait; +use std::fmt::Write as _; + +use super::Compressor; +use crate::openhuman::tokenjuice::types::{ + CompressInput, CompressOptions, CompressOutput, CompressorKind, +}; + +/// Bodies with more than this many collapsed lines get a placeholder; shorter +/// ones are kept verbatim (collapsing tiny bodies isn't worth the marker). +pub const MIN_BODY_LINES_TO_COLLAPSE: usize = 4; + +/// Markers that force a line to be kept even inside a deep body. +const KEEP_MARKERS: &[&str] = &["TODO", "FIXME", "XXX", "error", "panic", "unsafe"]; + +pub struct CodeCompressor; + +#[async_trait] +impl Compressor for CodeCompressor { + fn kind(&self) -> CompressorKind { + CompressorKind::Code + } + + async fn compress( + &self, + input: &CompressInput<'_>, + _opts: &CompressOptions, + ) -> Option { + // Prefer the AST path when a grammar matches the file's language; fall + // back to the language-agnostic heuristic otherwise (or when the AST + // path doesn't shrink the content). + #[cfg(feature = "tokenjuice-treesitter")] + if let Some(ext) = input.hint.extension.as_deref() { + if let Some(out) = treesitter::compress(input.content, ext) { + return Some(out); + } + } + compress_heuristic(input.content) + } +} + +/// Language-agnostic brace-depth compressor. Keeps lines at depth ≤ 1, collapses +/// deeper runs. Returns `None` if it wouldn't shrink the content. +pub fn compress_heuristic(content: &str) -> Option { + let lines: Vec<&str> = content.lines().collect(); + if lines.len() < 12 { + return None; + } + + let mut out = String::with_capacity(content.len() / 2 + 64); + let mut depth: i32 = 0; + let mut collapsed: Vec<&str> = Vec::new(); + + let flush = |out: &mut String, collapsed: &mut Vec<&str>| { + if collapsed.is_empty() { + return; + } + if collapsed.len() >= MIN_BODY_LINES_TO_COLLAPSE { + let _ = writeln!(out, " {{ … {} line(s) … }}", collapsed.len()); + } else { + for l in collapsed.iter() { + let _ = writeln!(out, "{l}"); + } + } + collapsed.clear(); + }; + + for line in &lines { + let (opens, closes) = brace_delta(line); + let start_depth = depth; + // A line that carries signal is always kept, regardless of depth. + let force_keep = KEEP_MARKERS.iter().any(|m| line.contains(m)); + + // Keep top-level lines (depth 0) — imports, signatures, the line that + // opens a block — and collapse the block body (depth ≥ 1). Short bodies + // (e.g. small struct field lists) stay verbatim via the flush threshold, + // so struct/enum fields survive while long function bodies collapse. + if start_depth == 0 || force_keep { + flush(&mut out, &mut collapsed); + let _ = writeln!(out, "{line}"); + } else { + collapsed.push(line); + } + depth += opens - closes; + if depth < 0 { + depth = 0; + } + } + flush(&mut out, &mut collapsed); + + let out = out.trim_end().to_string(); + if out.len() >= content.len() { + return None; + } + log::debug!( + "[tokenjuice][code] heuristic {} -> {} bytes ({} lines)", + content.len(), + out.len(), + lines.len() + ); + Some(CompressOutput::lossy(out, CompressorKind::Code)) +} + +/// Count `{`/`}` (and `(`/`)`) on a line, ignoring those inside string/char +/// literals and line comments — a cheap approximation good enough for the +/// depth heuristic. +fn brace_delta(line: &str) -> (i32, i32) { + let mut opens = 0i32; + let mut closes = 0i32; + let mut in_str: Option = None; + let mut prev = '\0'; + let mut chars = line.chars().peekable(); + while let Some(c) = chars.next() { + match in_str { + Some(q) => { + if c == q && prev != '\\' { + in_str = None; + } + } + None => match c { + '"' | '\'' | '`' => in_str = Some(c), + '/' if chars.peek() == Some(&'/') => break, // line comment + '#' => break, // python/shell comment + '{' => opens += 1, + '}' => closes += 1, + _ => {} + }, + } + prev = c; + } + (opens, closes) +} + +/// AST-aware code compression via tree-sitter (Rust/TS/JS/Python). Keeps full +/// source but replaces function/method bodies longer than a threshold with a +/// `{ … N lines … }` (or `...` for Python) placeholder, preserving signatures, +/// imports, type declarations and struct/enum fields exactly. +#[cfg(feature = "tokenjuice-treesitter")] +mod treesitter { + use super::{CompressOutput, CompressorKind, MIN_BODY_LINES_TO_COLLAPSE}; + use tree_sitter::{Node, Parser}; + + /// Pick the grammar for a file extension. Returns the language plus whether + /// it is brace-delimited (vs. Python's indentation suite). + fn language_for(ext: &str) -> Option<(tree_sitter::Language, bool)> { + let ext = ext.to_ascii_lowercase(); + match ext.as_str() { + "rs" => Some((tree_sitter_rust::LANGUAGE.into(), true)), + "ts" | "mts" | "cts" => { + Some((tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(), true)) + } + "tsx" => Some((tree_sitter_typescript::LANGUAGE_TSX.into(), true)), + "js" | "jsx" | "mjs" | "cjs" => { + // The TypeScript grammar is a superset that parses JS too. + Some((tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(), true)) + } + "py" | "pyi" => Some((tree_sitter_python::LANGUAGE.into(), false)), + _ => None, + } + } + + /// Node kinds whose `body` field is a collapsible function/method body. + const BODY_PARENTS: &[&str] = &[ + "function_item", + "function_declaration", + "function_definition", + "method_definition", + "function", + "arrow_function", + "generator_function_declaration", + ]; + + pub fn compress(content: &str, ext: &str) -> Option { + let (language, braced) = language_for(ext)?; + let mut parser = Parser::new(); + parser.set_language(&language).ok()?; + let tree = parser.parse(content, None)?; + let src = content.as_bytes(); + + // Collect outermost collapsible body byte-ranges. + let mut ranges: Vec<(usize, usize)> = Vec::new(); + collect_bodies(tree.root_node(), src, &mut ranges); + // Sort and drop nested ranges (keep outermost only). + ranges.sort_by_key(|r| r.0); + let mut merged: Vec<(usize, usize)> = Vec::new(); + for r in ranges { + if let Some(last) = merged.last() { + if r.0 < last.1 { + continue; // nested inside a body we're already collapsing + } + } + merged.push(r); + } + if merged.is_empty() { + return None; + } + + let mut out = String::with_capacity(content.len()); + let mut cursor = 0usize; + for (start, end) in merged { + if start < cursor { + continue; + } + out.push_str(&content[cursor..start]); + let body = &content[start..end]; + let n_lines = body.lines().count(); + if n_lines < MIN_BODY_LINES_TO_COLLAPSE { + out.push_str(body); + } else if braced { + out.push_str(&format!("{{ … {n_lines} line(s) … }}")); + } else { + // Python suite — keep an indented ellipsis so it still reads. + out.push_str(&format!("... # {n_lines} line(s) collapsed")); + } + cursor = end; + } + out.push_str(&content[cursor..]); + + let out = out.trim_end().to_string(); + if out.len() >= content.len() { + return None; + } + log::debug!( + "[tokenjuice][code] tree-sitter ext={} {} -> {} bytes", + ext, + content.len(), + out.len() + ); + Some(CompressOutput::lossy(out, CompressorKind::Code)) + } + + /// Recursively collect the byte-ranges of function/method bodies. + fn collect_bodies(node: Node, src: &[u8], out: &mut Vec<(usize, usize)>) { + if BODY_PARENTS.contains(&node.kind()) { + if let Some(body) = node.child_by_field_name("body") { + out.push((body.start_byte(), body.end_byte())); + // Don't descend into a collapsed body. + let _ = src; + return; + } + } + let mut cursor = node.walk(); + for child in node.children(&mut cursor) { + collect_bodies(child, src, out); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keeps_signatures_collapses_bodies() { + let mut src = String::from("use std::collections::HashMap;\n\n"); + src.push_str("pub fn process(items: &[i32]) -> i32 {\n"); + for i in 0..30 { + src.push_str(&format!( + " let tmp_{i} = items.iter().sum::() + {i};\n" + )); + } + src.push_str(" tmp_0\n}\n\n"); + src.push_str("struct Config {\n name: String,\n size: usize,\n}\n"); + let out = compress_heuristic(&src).expect("compresses"); + assert!(out.lossy); + assert!( + out.text.contains("pub fn process"), + "signature kept:\n{}", + out.text + ); + assert!(out.text.contains("struct Config")); + assert!( + out.text.contains("line(s) …"), + "body collapsed:\n{}", + out.text + ); + assert!( + !out.text.contains("tmp_15"), + "deep body should be collapsed" + ); + assert!(out.text.len() < src.len()); + } + + #[test] + fn short_file_passes_through() { + let src = "fn a() {}\nfn b() {}\n"; + assert!(compress_heuristic(src).is_none()); + } + + #[cfg(feature = "tokenjuice-treesitter")] + #[test] + fn treesitter_collapses_rust_body_keeps_struct() { + let mut src = String::from("use std::collections::HashMap;\n\n"); + src.push_str("pub fn process(items: &[i32]) -> i32 {\n"); + for i in 0..30 { + src.push_str(&format!( + " let tmp_{i} = items.iter().sum::() + {i};\n" + )); + } + src.push_str(" tmp_0\n}\n\n"); + src.push_str("pub struct Config {\n pub name: String,\n pub size: usize,\n}\n"); + let out = treesitter::compress(&src, "rs").expect("compresses"); + assert!( + out.text.contains("pub fn process(items: &[i32]) -> i32"), + "{}", + out.text + ); + // Struct fields preserved exactly (not a function body). + assert!(out.text.contains("pub name: String"), "{}", out.text); + assert!(out.text.contains("pub size: usize")); + // Function body collapsed. + assert!(out.text.contains("line(s) …"), "{}", out.text); + assert!(!out.text.contains("tmp_15")); + assert!(out.text.len() < src.len()); + } + + #[cfg(feature = "tokenjuice-treesitter")] + #[test] + fn treesitter_collapses_python_body() { + let mut src = String::from("import os\n\ndef handler(event):\n"); + for i in 0..30 { + src.push_str(&format!(" x_{i} = compute(event, {i})\n")); + } + src.push_str(" return x_0\n"); + let out = treesitter::compress(&src, "py").expect("compresses"); + assert!(out.text.contains("def handler(event):"), "{}", out.text); + assert!(out.text.contains("collapsed"), "{}", out.text); + assert!(!out.text.contains("x_15")); + } + + #[test] + fn keeps_marker_lines_in_body() { + let mut src = String::from("fn f() {\n"); + for i in 0..20 { + if i == 10 { + src.push_str(" // TODO: handle the edge case here\n"); + } else { + src.push_str(&format!(" do_thing({i});\n")); + } + } + src.push_str("}\n"); + let out = compress_heuristic(&src).expect("compresses"); + assert!(out.text.contains("TODO"), "marker line kept:\n{}", out.text); + } +} diff --git a/src/openhuman/agent/harness/compaction/diff.rs b/src/openhuman/tokenjuice/compressors/diff.rs similarity index 70% rename from src/openhuman/agent/harness/compaction/diff.rs rename to src/openhuman/tokenjuice/compressors/diff.rs index 6ce902b1b..61b4c3f4e 100644 --- a/src/openhuman/agent/harness/compaction/diff.rs +++ b/src/openhuman/tokenjuice/compressors/diff.rs @@ -1,40 +1,52 @@ //! Unified-diff compressor. //! -//! Clean-room port of headroom's `DiffCompressor` (Apache-2.0), in the -//! lossy-but-bounded style of [`super::search`] / [`super::logs`] (the caller -//! offloads the original to CCR for retrieval): -//! -//! - **Always keep** structural lines: `diff --git`, `index`, `---`/`+++` -//! file headers, and `@@` hunk headers. -//! - **Always keep** changed lines (`+`/`-`) — they are the signal. -//! - **Collapse** long runs of unchanged context (lines starting with a -//! space) down to a few anchor lines plus a `[... N context lines ...]` -//! marker, so the model still sees where a change sits without paying for -//! the whole untouched neighbourhood. -//! - **Summarize** high-volume / low-value files (lockfiles, minified -//! bundles): the hunk body collapses to a one-line `+A/-B` summary. -//! -//! Changed lines are never dropped, so the diff stays faithful to *what -//! changed* even when the surrounding context is trimmed. +//! Clean-room port of Headroom's `DiffCompressor` (Apache-2.0). Keeps the +//! signal (changed lines, structural headers, hunk headers) and collapses long +//! runs of unchanged context to an anchor + marker. Lockfile/bundle hunks +//! collapse to a one-line `+A/-B` summary. The router offloads the original to +//! CCR, so the dropped context stays recoverable. -use super::Compacted; +use async_trait::async_trait; use std::fmt::Write as _; +use super::Compressor; +use crate::openhuman::tokenjuice::text::ansi::strip_ansi; +use crate::openhuman::tokenjuice::types::{ + CompressInput, CompressOptions, CompressOutput, CompressorKind, +}; + /// Context lines kept on each side of a changed run before collapsing. pub const CONTEXT_ANCHOR: usize = 3; /// A run of unchanged context longer than this collapses to a marker. pub const CONTEXT_COLLAPSE_THRESHOLD: usize = 8; +pub struct DiffCompressor; + +#[async_trait] +impl Compressor for DiffCompressor { + fn kind(&self) -> CompressorKind { + CompressorKind::Diff + } + + async fn compress( + &self, + input: &CompressInput<'_>, + _opts: &CompressOptions, + ) -> Option { + compress(input.content) + } +} + /// Compress a unified diff. Returns `None` when there's nothing structural to -/// work with or compression wouldn't shrink it. Lossy when it fires (collapses -/// context / summarizes hunks); the caller offloads the original to CCR. -pub fn compress(content: &str) -> Option { - let lines: Vec<&str> = content.lines().collect(); +/// work with or compression wouldn't shrink it. +pub fn compress(content: &str) -> Option { + let stripped = strip_ansi(content); + let lines: Vec<&str> = stripped.lines().collect(); if lines.is_empty() { return None; } - let mut out = String::with_capacity(content.len() / 2 + 64); + let mut out = String::with_capacity(stripped.len() / 2 + 64); let mut i = 0usize; let mut current_file_is_noisy = false; let mut saw_hunk = false; @@ -42,7 +54,6 @@ pub fn compress(content: &str) -> Option { while i < lines.len() { let line = lines[i]; - // File header — note whether this file is a lockfile/bundle we summarize. if line.starts_with("diff --git ") { current_file_is_noisy = is_noisy_path(line); let _ = writeln!(out, "{line}"); @@ -51,7 +62,6 @@ pub fn compress(content: &str) -> Option { } if is_structural(line) { saw_hunk |= line.starts_with("@@"); - // For a noisy file, collapse the entire hunk body to a summary. if current_file_is_noisy && line.starts_with("@@") { let _ = writeln!(out, "{line}"); i += 1; @@ -68,7 +78,6 @@ pub fn compress(content: &str) -> Option { continue; } - // Context line — collapse long unchanged runs. if is_context(line) { let start = i; while i < lines.len() && is_context(lines[i]) { @@ -92,7 +101,6 @@ pub fn compress(content: &str) -> Option { continue; } - // Changed line (+/-) or anything else — keep verbatim. let _ = writeln!(out, "{line}"); i += 1; } @@ -104,15 +112,17 @@ pub fn compress(content: &str) -> Option { return None; } log::debug!( - "[compaction][diff] {} -> {} bytes ({} input lines)", + "[tokenjuice][diff] {} -> {} bytes ({} input lines)", content.len(), out.len(), lines.len(), ); - Some(Compacted::lossy(out.trim_end().to_string())) + Some(CompressOutput::lossy( + out.trim_end().to_string(), + CompressorKind::Diff, + )) } -/// Structural diff lines that must always survive. fn is_structural(line: &str) -> bool { line.starts_with("@@") || line.starts_with("--- ") @@ -125,14 +135,10 @@ fn is_structural(line: &str) -> bool { || line.starts_with("Binary files") } -/// A unified-diff context line: leading space, not a `+`/`-` change and not a -/// `---`/`+++` header (those are structural and handled first). fn is_context(line: &str) -> bool { line.starts_with(' ') } -/// Count added/removed lines in a hunk body until the next hunk/file header, -/// returning how many lines were consumed. fn summarize_hunk_body(lines: &[&str]) -> (usize, usize, usize) { let mut added = 0usize; let mut removed = 0usize; @@ -151,8 +157,6 @@ fn summarize_hunk_body(lines: &[&str]) -> (usize, usize, usize) { (added, removed, n) } -/// Lockfiles and generated bundles whose diff body is rarely worth reading in -/// full. Matched against the `diff --git a/ b/` header. fn is_noisy_path(diff_git_line: &str) -> bool { let l = diff_git_line.to_ascii_lowercase(); const NOISY: &[&str] = &[ @@ -205,8 +209,6 @@ mod tests { } let out = compress(&s).expect("compresses").text; assert!(out.contains("lockfile/bundle hunk"), "{out}"); - assert!(out.contains("Cargo.lock")); - // Individual dep lines are gone. assert!(!out.contains("new dep entry 7"), "{out}"); assert!(out.len() < s.len()); } diff --git a/src/openhuman/tokenjuice/compressors/generic.rs b/src/openhuman/tokenjuice/compressors/generic.rs new file mode 100644 index 000000000..aac0f85d3 --- /dev/null +++ b/src/openhuman/tokenjuice/compressors/generic.rs @@ -0,0 +1,39 @@ +//! Generic line-oriented fallback compressor. +//! +//! Used by the router when a specialised compressor declines (or the ML text +//! compressor is unavailable). It runs the rule engine's `generic/fallback` +//! head/tail summariser — but **only for command output**. For domain-tool +//! payloads (no derived command/argv) it declines, preserving the long-standing +//! guard that large structured tool results must reach the downstream +//! progressive-disclosure handoff rather than being blindly head/tail clamped. + +use async_trait::async_trait; + +use super::Compressor; +use crate::openhuman::tokenjuice::compressors::log::compress_command_fallback; +use crate::openhuman::tokenjuice::types::{ + CompressInput, CompressOptions, CompressOutput, CompressorKind, +}; + +pub struct GenericCompressor; + +#[async_trait] +impl Compressor for GenericCompressor { + fn kind(&self) -> CompressorKind { + CompressorKind::Generic + } + + async fn compress( + &self, + input: &CompressInput<'_>, + opts: &CompressOptions, + ) -> Option { + let has_command = + input.command.is_some() || input.argv.as_ref().is_some_and(|a| !a.is_empty()); + if !has_command { + // Domain-tool payload — decline rather than blind-truncate. + return None; + } + compress_command_fallback(input, opts) + } +} diff --git a/src/openhuman/tokenjuice/compressors/html.rs b/src/openhuman/tokenjuice/compressors/html.rs new file mode 100644 index 000000000..b99de09db --- /dev/null +++ b/src/openhuman/tokenjuice/compressors/html.rs @@ -0,0 +1,262 @@ +//! HTML → readable-text extractor. +//! +//! Strips markup and returns the readable text content, in the spirit of +//! Headroom's `HTMLExtractor`. Linear-time, allocation-light (no DOM, no +//! regex): it scans once, dropping `

Title

Hello world.

"; + let text = html_to_text(html); + assert!(text.contains("Title")); + assert!(text.contains("Hello")); + assert!(text.contains("world")); + assert!( + !text.contains("alert"), + "script body must be dropped: {text}" + ); + assert!(!text.contains("color:red"), "style body must be dropped"); + } + + #[test] + fn decodes_entities() { + let text = html_to_text("

a & b < c > d  e

"); + assert!(text.contains("a & b < c > d"), "{text}"); + } + + #[test] + fn block_tags_insert_newlines() { + let text = html_to_text("

one

two

  • three
  • "); + let lines: Vec<&str> = text.lines().filter(|l| !l.trim().is_empty()).collect(); + assert!(lines.len() >= 3, "expected separate lines, got {lines:?}"); + } + + #[test] + fn compress_shrinks_real_doc() { + let mut html = String::from(""); + for i in 0..50 { + html.push_str(&format!( + "
    cell {i}
    " + )); + } + html.push_str(""); + let out = compress(&html).expect("compresses"); + assert!(out.lossy); + assert!(out.text.len() < html.len()); + assert!(out.text.contains("cell 7")); + } +} diff --git a/src/openhuman/tokenjuice/compressors/json.rs b/src/openhuman/tokenjuice/compressors/json.rs new file mode 100644 index 000000000..3a305f737 --- /dev/null +++ b/src/openhuman/tokenjuice/compressors/json.rs @@ -0,0 +1,310 @@ +//! JSON-array crusher (SmartCrusher). +//! +//! Clean-room port of Headroom's `SmartCrusher` (Apache-2.0), extended with +//! variance-aware row preservation. An array of objects that repeat the same +//! keys is the single most common bloated tool output (API list responses, DB +//! rows, search manifests). Re-rendering it as a table emits each key **once** +//! instead of per row. +//! +//! Up to [`ROW_DROP_THRESHOLD`] rows every value is preserved (faithful +//! reformat). Above the threshold the table is row-dropped — but rather than a +//! blind head/tail window, rows carrying **error indicators** or **numeric +//! outliers** are always kept, so anomalies survive even when the bulk of a +//! homogeneous array is dropped. The router offloads the full original to CCR, +//! so the dropped rows stay recoverable. + +use async_trait::async_trait; +use serde_json::Value; +use std::collections::BTreeSet; +use std::fmt::Write as _; + +use super::signals::has_error_indicators; +use super::Compressor; +use crate::openhuman::tokenjuice::types::{ + CompressInput, CompressOptions, CompressOutput, CompressorKind, +}; + +/// Minimum rows before tabular rendering is worth the header overhead. +pub const MIN_ROWS: usize = 3; +/// Above this many rows the table is additionally row-dropped. +pub const ROW_DROP_THRESHOLD: usize = 40; +/// Rows kept from the head when row-dropping. +pub const HEAD_ROWS: usize = 20; +/// Rows kept from the tail when row-dropping. +pub const TAIL_ROWS: usize = 10; +/// Z-score beyond which a numeric cell is treated as an outlier worth keeping. +pub const OUTLIER_SIGMA: f64 = 2.0; + +pub struct JsonCompressor; + +#[async_trait] +impl Compressor for JsonCompressor { + fn kind(&self) -> CompressorKind { + CompressorKind::SmartCrusher + } + + async fn compress( + &self, + input: &CompressInput<'_>, + _opts: &CompressOptions, + ) -> Option { + compress(input.content) + } +} + +/// Compress a JSON array-of-objects into a compact table. Returns `None` when +/// the content isn't a uniform-enough array of objects or wouldn't shrink. +pub fn compress(content: &str) -> Option { + let value: Value = serde_json::from_str(content.trim()).ok()?; + let array = value.as_array()?; + if array.len() < MIN_ROWS { + return None; + } + if !array.iter().all(Value::is_object) { + return None; + } + + // Column order = first-seen key order across all rows (union, stable). + let mut columns: Vec = Vec::new(); + for item in array { + if let Some(obj) = item.as_object() { + for key in obj.keys() { + if !columns.iter().any(|c| c == key) { + columns.push(key.clone()); + } + } + } + } + if columns.len() < 2 { + return None; + } + + // Render every row's cells up front so we can choose full vs. row-dropped. + let mut rows: Vec = Vec::with_capacity(array.len()); + for item in array { + let obj = item.as_object()?; + let cells: Vec = columns + .iter() + .map(|col| match obj.get(col) { + None => String::new(), + Some(v) => render_cell(v), + }) + .collect(); + rows.push(cells.join(" | ")); + } + + let lossy = rows.len() > ROW_DROP_THRESHOLD; + let mut out = String::with_capacity(content.len()); + let _ = writeln!( + out, + "[json table: {} rows × {} cols · blank=absent key · exact original via retrieve footer]", + rows.len(), + columns.len() + ); + let _ = writeln!(out, "{}", columns.join(" | ")); + + if lossy { + // Keep head + tail PLUS any anomalous rows (errors / numeric outliers) + // so the signal in a large homogeneous array survives row-dropping. + let keep = rows_to_keep(array, &columns, rows.len()); + let mut prev: Option = None; + for &i in &keep { + if let Some(p) = prev { + let gap = i - p - 1; + if gap > 0 { + let _ = writeln!(out, "[... {gap} row(s) omitted ...]"); + } + } else if i > 0 { + let _ = writeln!(out, "[... {i} row(s) omitted ...]"); + } + let _ = writeln!(out, "{}", rows[i]); + prev = Some(i); + } + if let Some(p) = prev { + let tail = rows.len().saturating_sub(p + 1); + if tail > 0 { + let _ = writeln!(out, "[... {tail} row(s) omitted ...]"); + } + } + } else { + for row in &rows { + let _ = writeln!(out, "{row}"); + } + } + + let out = out.trim_end().to_string(); + if out.len() >= content.len() { + return None; + } + log::debug!( + "[tokenjuice][json] {} rows × {} cols, lossy={} ({} -> {} bytes)", + rows.len(), + columns.len(), + lossy, + content.len(), + out.len(), + ); + if lossy { + Some(CompressOutput::lossy(out, CompressorKind::SmartCrusher)) + } else { + // All values preserved, but the array→table reformat changes layout. + Some(CompressOutput::reformatted( + out, + CompressorKind::SmartCrusher, + )) + } +} + +/// Pick the row indices to keep when row-dropping: the head/tail windows plus +/// any row flagged as anomalous (error text or a numeric outlier in any +/// column). Returns ascending, de-duplicated indices. +fn rows_to_keep(array: &[Value], columns: &[String], n: usize) -> Vec { + let mut keep: BTreeSet = BTreeSet::new(); + for i in 0..HEAD_ROWS.min(n) { + keep.insert(i); + } + for i in n.saturating_sub(TAIL_ROWS)..n { + keep.insert(i); + } + + // Error-text rows: any string/scalar cell carrying an error indicator. + for (i, item) in array.iter().enumerate() { + if let Some(obj) = item.as_object() { + let row_has_error = obj.values().any(|v| match v { + Value::String(s) => has_error_indicators(s), + other => has_error_indicators(&other.to_string()), + }); + if row_has_error { + keep.insert(i); + } + } + } + + // Numeric outliers: for each column, compute mean/std over numeric cells and + // keep rows whose value is beyond OUTLIER_SIGMA. Bounded by a cap so a wide + // anomalous tail can't defeat the point of dropping. + for col in columns { + let nums: Vec<(usize, f64)> = array + .iter() + .enumerate() + .filter_map(|(i, item)| { + item.as_object() + .and_then(|o| o.get(col)) + .and_then(Value::as_f64) + .map(|x| (i, x)) + }) + .collect(); + if nums.len() < 4 { + continue; + } + let mean = nums.iter().map(|(_, x)| x).sum::() / nums.len() as f64; + let var = nums.iter().map(|(_, x)| (x - mean).powi(2)).sum::() / nums.len() as f64; + let std = var.sqrt(); + if std <= f64::EPSILON { + continue; + } + for (i, x) in nums { + if ((x - mean) / std).abs() >= OUTLIER_SIGMA { + keep.insert(i); + } + } + } + + keep.into_iter().collect() +} + +/// Render a single cell. Scalars print bare-ish; nested values stay as compact +/// JSON so the table remains lossless. +fn render_cell(v: &Value) -> String { + match v { + Value::String(s) if !s.contains('|') && !s.contains('\n') => s.clone(), + Value::Bool(b) => b.to_string(), + Value::Number(n) => n.to_string(), + other => serde_json::to_string(other).unwrap_or_default(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn crushes_uniform_array() { + let mut rows = Vec::new(); + for i in 0..20 { + rows.push(format!( + r#"{{"id":{i},"name":"item number {i}","status":"active","owner":"team-alpha"}}"# + )); + } + let input = format!("[{}]", rows.join(",")); + let out = compress(&input).expect("compresses").text; + assert_eq!(out.matches("status").count(), 1, "{out}"); + assert!(out.contains("item number 7")); + assert!(out.len() < input.len(), "expected shrink"); + } + + #[test] + fn large_array_row_drops_and_is_marked_lossy() { + let mut rows = Vec::new(); + for i in 0..200 { + rows.push(format!( + r#"{{"id":{i},"name":"record number {i}","status":"active","note":"some detail {i}"}}"# + )); + } + let input = format!("[{}]", rows.join(",")); + let c = compress(&input).expect("compresses"); + assert!(c.lossy, "row-dropped output must be lossy"); + assert!(c.text.contains("record number 0"), "{}", c.text); + assert!(c.text.contains("record number 199"), "{}", c.text); + assert!(c.text.contains("omitted")); + assert!(c.text.len() < input.len()); + } + + #[test] + fn keeps_error_row_in_dropped_middle() { + // A homogeneous array with a single error row buried in the middle: the + // SmartCrusher must keep that row even though it's in the drop window. + let mut rows = Vec::new(); + for i in 0..120 { + let status = if i == 75 { "error: timeout" } else { "ok" }; + rows.push(format!( + r#"{{"id":{i},"name":"job {i}","status":"{status}","note":"detail {i}"}}"# + )); + } + let input = format!("[{}]", rows.join(",")); + let c = compress(&input).expect("compresses"); + assert!(c.lossy); + assert!( + c.text.contains("job 75"), + "error row must survive:\n{}", + c.text + ); + assert!(c.text.contains("error: timeout")); + } + + #[test] + fn keeps_numeric_outlier_row() { + let mut rows = Vec::new(); + for i in 0..120 { + // Most latencies ~10ms; row 88 is a 9999ms outlier. + let latency = if i == 88 { 9999 } else { 10 + (i % 3) }; + rows.push(format!( + r#"{{"id":{i},"endpoint":"/api/{i}","latency_ms":{latency},"region":"us"}}"# + )); + } + let input = format!("[{}]", rows.join(",")); + let c = compress(&input).expect("compresses"); + assert!( + c.text.contains("9999"), + "outlier row must survive:\n{}", + c.text + ); + } + + #[test] + fn non_array_returns_none() { + assert!(compress(r#"{"a":1}"#).is_none()); + assert!(compress("[1,2,3]").is_none()); + assert!(compress(r#"[{"a":1}]"#).is_none()); + } +} diff --git a/src/openhuman/agent/harness/compaction/logs.rs b/src/openhuman/tokenjuice/compressors/log.rs similarity index 56% rename from src/openhuman/agent/harness/compaction/logs.rs rename to src/openhuman/tokenjuice/compressors/log.rs index f8497c6c0..a8e3a12ec 100644 --- a/src/openhuman/agent/harness/compaction/logs.rs +++ b/src/openhuman/tokenjuice/compressors/log.rs @@ -1,54 +1,135 @@ //! Build/test/lint log compressor. //! -//! Clean-room port of headroom's `LogCompressor` (Apache-2.0). Keeps the -//! lines an agent acts on and drops the ceremony: +//! Two paths, chosen by whether the content is *command* output: //! -//! - **errors** — keep up to [`MAX_ERRORS`] (biased to the first and last, -//! which are usually the root cause and the abort line). -//! - **warnings** — keep up to [`MAX_WARNINGS`], de-duplicated (the "same -//! deprecation × 47" case collapses to one). -//! - **stack traces** — keep up to [`MAX_STACK_TRACES`] runs of indented -//! frames, each capped at [`STACK_TRACE_MAX_LINES`]. -//! - **summary lines** — always keep test/build summaries -//! (`test result: ...`, `N passed; M failed`, `error: aborting`). -//! - hard cap of [`MAX_TOTAL_LINES`] kept lines overall. +//! - **Command output** (the [`CompressInput`] carries a derived command/argv): +//! run the 100-rule reduction engine ([`reduce_execution_with_rules`]). This +//! is TokenJuice's original behaviour — git/cargo/npm/docker-aware rules with +//! failure preservation. +//! - **Non-command logs** (a blob detected as a log with no command context): +//! the signal-based keep-failures/drop-noise compressor, a clean-room port of +//! Headroom's `LogCompressor` (Apache-2.0). //! -//! Output preserves original line order and notes how many lines were -//! dropped. Lossy-but-bounded: first/last errors and the summary always -//! survive. +//! The signal path declines (returns `None`) when a blob has no error/warning/ +//! summary/stack signal at all — that almost certainly isn't a log (a file +//! listing, CSV, generated data) and must not be head/tail truncated. -use super::signals::{severity, Severity}; -use super::Compacted; +use async_trait::async_trait; +use once_cell::sync::Lazy; use std::collections::HashSet; use std::fmt::Write as _; +use super::signals::{severity, Severity}; +use super::Compressor; +use crate::openhuman::tokenjuice::reduce::reduce_execution_with_rules; +use crate::openhuman::tokenjuice::rules::load_builtin_rules; +use crate::openhuman::tokenjuice::types::{ + CompiledRule, CompressInput, CompressOptions, CompressOutput, CompressorKind, ReduceOptions, + ToolExecutionInput, +}; + pub const MAX_ERRORS: usize = 10; pub const MAX_WARNINGS: usize = 5; pub const MAX_STACK_TRACES: usize = 3; pub const STACK_TRACE_MAX_LINES: usize = 20; pub const MAX_TOTAL_LINES: usize = 100; -/// Compress a build/test/lint log. Returns `None` when nothing can be saved -/// (few lines, or no reduction possible) so the caller passes it through. -/// Lossy when it fires (drops lines); the caller offloads the original to CCR. -pub fn compress(content: &str) -> Option { +static BUILTIN_RULES: Lazy> = Lazy::new(load_builtin_rules); + +pub struct LogCompressor; + +#[async_trait] +impl Compressor for LogCompressor { + fn kind(&self) -> CompressorKind { + CompressorKind::Log + } + + async fn compress( + &self, + input: &CompressInput<'_>, + opts: &CompressOptions, + ) -> Option { + let has_command = + input.command.is_some() || input.argv.as_ref().is_some_and(|a| !a.is_empty()); + if has_command { + compress_command(input, opts) + } else { + compress_signal(input.content) + } + } +} + +/// Command output → run the rule engine, tagged as [`CompressorKind::Log`]. +fn compress_command(input: &CompressInput<'_>, opts: &CompressOptions) -> Option { + run_rule_engine(input, opts, CompressorKind::Log) +} + +/// The generic fallback path: same rule engine, tagged [`CompressorKind::Generic`]. +/// Exposed for [`super::generic::GenericCompressor`] so the router can fall back +/// to head/tail summarisation of command output without re-implementing it. +pub fn compress_command_fallback( + input: &CompressInput<'_>, + opts: &CompressOptions, +) -> Option { + run_rule_engine(input, opts, CompressorKind::Generic) +} + +/// Run the 100-rule reduction engine over command output, reporting `kind`. +/// Returns `None` when reduction wouldn't shrink the payload. +fn run_rule_engine( + input: &CompressInput<'_>, + opts: &CompressOptions, + kind: CompressorKind, +) -> Option { + let exec = ToolExecutionInput { + tool_name: input + .hint + .source_tool + .clone() + .unwrap_or_else(|| "shell".to_string()), + command: input.command.clone(), + argv: input.argv.clone(), + stdout: Some(input.content.to_string()), + exit_code: input.exit_code, + ..Default::default() + }; + let reduce_opts = ReduceOptions { + max_inline_chars: opts.max_inline_chars, + ..Default::default() + }; + let result = reduce_execution_with_rules(exec, &BUILTIN_RULES, &reduce_opts); + if result.inline_text.len() >= input.content.len() { + return None; + } + let rule_label = result + .classification + .matched_reducer + .unwrap_or(result.classification.family); + log::debug!( + "[tokenjuice][log] command rule={} kind={} {} -> {} bytes", + rule_label, + kind.as_str(), + input.content.len(), + result.inline_text.len() + ); + Some(CompressOutput::lossy(result.inline_text, kind)) +} + +/// Signal-based log compression for non-command blobs detected as logs. +pub fn compress_signal(content: &str) -> Option { let lines: Vec<&str> = content.lines().collect(); if lines.len() <= MAX_TOTAL_LINES { - // Short enough already — the byte budget (if any) will handle it. return None; } - // Indices we decide to keep. BTreeSet keeps output in original order. let mut keep: std::collections::BTreeSet = std::collections::BTreeSet::new(); - // 1. Summary lines — always keep. for (i, line) in lines.iter().enumerate() { if is_summary_line(line) { keep.insert(i); } } - // 2. Errors — first MAX_ERRORS/2 and last MAX_ERRORS/2 by appearance. let error_idx: Vec = lines .iter() .enumerate() @@ -59,7 +140,6 @@ pub fn compress(content: &str) -> Option { keep.insert(i); } - // 3. Warnings — de-duplicated by normalized text, capped. let mut seen_warn: HashSet = HashSet::new(); let mut warn_kept = 0usize; for (i, line) in lines.iter().enumerate() { @@ -75,8 +155,6 @@ pub fn compress(content: &str) -> Option { } } - // 4. Stack traces — runs of indented / "at "/"#n " frames following an - // error, capped in count and per-trace length. let mut traces_kept = 0usize; let mut i = 0usize; while i < lines.len() && traces_kept < MAX_STACK_TRACES { @@ -99,16 +177,10 @@ pub fn compress(content: &str) -> Option { } if keep.is_empty() { - // No errors, warnings, stack traces, or summary lines — this almost - // certainly isn't a log (e.g. generic `shell` output: a file listing, - // CSV, or a script printing data). Do NOT head/tail-truncate it, which - // would silently drop the middle of legitimate data. Pass it through to - // the byte budget instead. + // Not a log (no signal) — never head/tail truncate legitimate data. return None; } - // Enforce the global line cap, keeping the earliest + latest kept lines so - // the root cause and the final summary both survive. let kept_vec: Vec = keep.iter().copied().collect(); let kept_vec = if kept_vec.len() > MAX_TOTAL_LINES { select_first_last(&kept_vec, MAX_TOTAL_LINES) @@ -117,19 +189,15 @@ pub fn compress(content: &str) -> Option { }; let kept_set: std::collections::BTreeSet = kept_vec.into_iter().collect(); - // Render with gap markers for runs of dropped lines. let mut out = String::with_capacity(content.len() / 2 + 64); let mut prev: Option = None; - let mut total_dropped = 0usize; for &i in &kept_set { if let Some(p) = prev { let gap = i - p - 1; if gap > 0 { - total_dropped += gap; let _ = writeln!(out, "[... {gap} line(s) omitted ...]"); } } else if i > 0 { - total_dropped += i; let _ = writeln!(out, "[... {i} line(s) omitted ...]"); } let _ = writeln!(out, "{}", lines[i]); @@ -138,7 +206,6 @@ pub fn compress(content: &str) -> Option { if let Some(p) = prev { let tail = lines.len().saturating_sub(p + 1); if tail > 0 { - total_dropped += tail; let _ = writeln!(out, "[... {tail} line(s) omitted ...]"); } } @@ -147,17 +214,16 @@ pub fn compress(content: &str) -> Option { return None; } log::debug!( - "[compaction][logs] kept {} of {} line(s), dropped {}", + "[tokenjuice][log] signal kept {} of {} line(s)", kept_set.len(), lines.len(), - total_dropped, ); - Some(Compacted::lossy(out.trim_end().to_string())) + Some(CompressOutput::lossy( + out.trim_end().to_string(), + CompressorKind::Log, + )) } -/// Choose at most `cap` indices, biased to the first and last by value. -/// `idx` must be ascending. Keeps `ceil(cap/2)` from the front and the rest -/// from the back, preserving order and avoiding duplicates. fn select_first_last(idx: &[usize], cap: usize) -> Vec { if idx.len() <= cap { return idx.to_vec(); @@ -177,7 +243,6 @@ fn select_first_last(idx: &[usize], cap: usize) -> Vec { out.into_iter().collect() } -/// Test/build summary lines worth always keeping. fn is_summary_line(line: &str) -> bool { let l = line.to_ascii_lowercase(); let l = l.trim(); @@ -195,7 +260,6 @@ fn is_summary_line(line: &str) -> bool { || (l.contains("npm") && l.contains("err")) } -/// A stack-trace frame: leading whitespace + `at `/`#n `/`File "...` etc. fn is_stack_frame(line: &str) -> bool { let trimmed = line.trim_start(); if trimmed.is_empty() { @@ -208,8 +272,6 @@ fn is_stack_frame(line: &str) -> bool { || (trimmed.starts_with('#') && trimmed[1..].starts_with(|c: char| c.is_ascii_digit()))) } -/// Normalize a warning line for de-duplication: lowercase and strip digits so -/// "warning at line 12" and "warning at line 88" collapse together. fn normalize_for_dedupe(line: &str) -> String { line.chars() .filter(|c| !c.is_ascii_digit()) @@ -237,58 +299,22 @@ mod tests { } #[test] - fn keeps_errors_and_summary_drops_noise() { + fn signal_keeps_errors_and_summary_drops_noise() { let input = noisy_log(); - let out = compress(&input).expect("compresses").text; + let out = compress_signal(&input).expect("compresses").text; assert!(out.contains("error[E0382]"), "{out}"); assert!(out.contains("error: aborting"), "{out}"); assert!(out.contains("test result: FAILED"), "{out}"); - assert!(out.lines().count() <= MAX_TOTAL_LINES + 10); assert!(out.len() < input.len()); assert!(out.contains("omitted")); } #[test] - fn dedupes_warnings() { - let mut s = String::new(); - for i in 0..150 { - let _ = writeln!(s, "warning: unused variable at line {i}"); - } - let _ = writeln!(s, "test result: ok. 1 passed; 0 failed"); - let out = compress(&s).expect("compresses").text; - let warns = out.matches("unused variable").count(); - assert!(warns <= MAX_WARNINGS, "kept {warns} warnings"); - } - - #[test] - fn non_log_data_passes_through_not_head_tail_truncated() { - // Generic shell-style output with no errors/warnings/summary: a long - // listing. Must pass through (None) rather than dropping the middle. + fn non_log_data_passes_through() { let mut s = String::new(); for i in 0..400 { let _ = writeln!(s, "/var/data/file_{i:04}.bin\t{i}\trwxr-xr-x"); } - assert!(compress(&s).is_none(), "non-log data must not be truncated"); - } - - #[test] - fn short_log_passes_through() { - let s = "line1\nline2\nerror: boom\n"; - assert!(compress(s).is_none()); - } - - #[test] - fn keeps_stack_trace_capped() { - let mut s = String::new(); - let _ = writeln!(s, "panicked at 'boom'"); - for i in 0..50 { - let _ = writeln!(s, " at frame_{i} (src/x.rs:{i})"); - } - for i in 0..120 { - let _ = writeln!(s, "info: step {i}"); - } - let out = compress(&s).expect("compresses").text; - let frames = out.matches(" at frame_").count(); - assert!(frames <= STACK_TRACE_MAX_LINES, "{frames} frames kept"); + assert!(compress_signal(&s).is_none()); } } diff --git a/src/openhuman/tokenjuice/compressors/ml_text.rs b/src/openhuman/tokenjuice/compressors/ml_text.rs new file mode 100644 index 000000000..65e17f9a3 --- /dev/null +++ b/src/openhuman/tokenjuice/compressors/ml_text.rs @@ -0,0 +1,48 @@ +//! ML plain-text compressor ("Kompress") — trait slot. +//! +//! Plain text has no structural skeleton to exploit, so high-quality +//! compression needs a learned model (Headroom uses ModernBERT token +//! classification to drop low-salience spans). That path runs the `kompress` +//! backend of the shared `runtime_python_server` and is **opt-in** behind the +//! `tokenjuice.ml_compression_enabled` config flag. +//! +//! This module is the [`Compressor`] slot; it delegates to +//! [`crate::openhuman::tokenjuice::ml`]. Whenever the flag is off or the Python +//! runtime is unavailable, `compress` declines so the router falls back to the +//! generic compressor — never an error in the agent loop. + +use async_trait::async_trait; + +use super::Compressor; +use crate::openhuman::tokenjuice::types::{ + CompressInput, CompressOptions, CompressOutput, CompressorKind, +}; + +pub struct MlTextCompressor; + +#[async_trait] +impl Compressor for MlTextCompressor { + fn kind(&self) -> CompressorKind { + CompressorKind::MlText + } + + async fn compress( + &self, + input: &CompressInput<'_>, + opts: &CompressOptions, + ) -> Option { + if !opts.ml_text_enabled { + return None; + } + match crate::openhuman::tokenjuice::ml::compress(input.content, opts).await { + Ok(Some(text)) if text.len() < input.content.len() => { + Some(CompressOutput::lossy(text, CompressorKind::MlText)) + } + Ok(_) => None, + Err(e) => { + log::debug!("[tokenjuice][ml] unavailable, falling back: {e:#}"); + None + } + } + } +} diff --git a/src/openhuman/tokenjuice/compressors/mod.rs b/src/openhuman/tokenjuice/compressors/mod.rs new file mode 100644 index 000000000..8cc2b28c6 --- /dev/null +++ b/src/openhuman/tokenjuice/compressors/mod.rs @@ -0,0 +1,77 @@ +//! Per-content-kind compressors and the registry that maps a [`ContentKind`] +//! to the [`Compressor`] that handles it. +//! +//! Each compressor preserves the signal its kind carries — errors in logs, +//! changed hunks in diffs, signatures in code, anomalous rows in JSON — and +//! drops the rest. Lossy compressors leave recovery to the router +//! ([`crate::openhuman::tokenjuice::compress`]), which offloads the original to +//! the CCR cache and appends a retrieval marker. Compressors therefore return +//! only the compacted body and a `lossy` flag; they never touch the cache. + +pub mod code; +pub mod diff; +pub mod generic; +pub mod html; +pub mod json; +pub mod log; +pub mod ml_text; +pub mod search; +pub mod signals; + +use async_trait::async_trait; + +use crate::openhuman::tokenjuice::types::{ + CompressInput, CompressOptions, CompressOutput, CompressorKind, ContentKind, +}; + +/// A content-aware compressor. Implementations are stateless and zero-sized; +/// the registry hands out `&'static` references. +#[async_trait] +pub trait Compressor: Send + Sync { + /// Which [`CompressorKind`] this is (for stats/logs). + fn kind(&self) -> CompressorKind; + + /// Compress `input`. Return `None` to decline (the router passes the + /// original through). `Some` carries the compacted body and whether data + /// was dropped. Async so the ML compressor can talk to its Python sidecar + /// without a blocking bridge; native compressors complete synchronously. + async fn compress( + &self, + input: &CompressInput<'_>, + opts: &CompressOptions, + ) -> Option; +} + +static JSON_COMPRESSOR: json::JsonCompressor = json::JsonCompressor; +static CODE_COMPRESSOR: code::CodeCompressor = code::CodeCompressor; +static LOG_COMPRESSOR: log::LogCompressor = log::LogCompressor; +static SEARCH_COMPRESSOR: search::SearchCompressor = search::SearchCompressor; +static DIFF_COMPRESSOR: diff::DiffCompressor = diff::DiffCompressor; +static HTML_COMPRESSOR: html::HtmlCompressor = html::HtmlCompressor; +static ML_TEXT_COMPRESSOR: ml_text::MlTextCompressor = ml_text::MlTextCompressor; +static GENERIC_COMPRESSOR: generic::GenericCompressor = generic::GenericCompressor; + +/// Map a detected [`ContentKind`] to the compressor that handles it. +/// +/// `PlainText` routes to the ML compressor; whether it actually runs is gated +/// by `opts.ml_text_enabled` (and runtime Python/runtime_python_server +/// availability), and it falls back to [`generic::GenericCompressor`] otherwise +/// — that gating lives in [`crate::openhuman::tokenjuice::compress`], so this +/// function is a pure static mapping. +pub fn compressor_for(kind: ContentKind) -> &'static dyn Compressor { + match kind { + ContentKind::Json => &JSON_COMPRESSOR, + ContentKind::Code => &CODE_COMPRESSOR, + ContentKind::Log => &LOG_COMPRESSOR, + ContentKind::Search => &SEARCH_COMPRESSOR, + ContentKind::Diff => &DIFF_COMPRESSOR, + ContentKind::Html => &HTML_COMPRESSOR, + ContentKind::PlainText => &ML_TEXT_COMPRESSOR, + } +} + +/// The generic line-oriented fallback compressor (head/tail summariser). Used +/// by the router when a specialised compressor declines or is disabled. +pub fn generic_compressor() -> &'static dyn Compressor { + &GENERIC_COMPRESSOR +} diff --git a/src/openhuman/tokenjuice/compressors/search.rs b/src/openhuman/tokenjuice/compressors/search.rs new file mode 100644 index 000000000..fc3d464f8 --- /dev/null +++ b/src/openhuman/tokenjuice/compressors/search.rs @@ -0,0 +1,221 @@ +//! Search-results compressor (relevance ranking). +//! +//! grep / ripgrep output is `path:line:body` matches. For very large result +//! sets the compressor groups matches by file, ranks them, keeps the top-K per +//! file plus a `[+N more in ]` tally, and (via the router) offloads the +//! full result set to CCR so the complete list is one `retrieve` away. +//! +//! Ranking: when the caller supplies a `query` in the [`ContentHint`], matches +//! are scored by query-term density in the body; otherwise by body length / +//! uniqueness (longer, more-distinctive lines first), with importance signals +//! (error/TODO) always boosted. Group/file order is preserved (first-seen). +//! +//! NOTE: this compressor is gated by `opts.search_enabled` in the router. It is +//! a behaviour change from the historical "never compact grep" stance — kept +//! lossless by the CCR offload — and is enabled by default per project decision. + +use async_trait::async_trait; +use std::fmt::Write as _; + +use super::signals::line_score; +use super::Compressor; +use crate::openhuman::tokenjuice::detect::parse_search_line; +use crate::openhuman::tokenjuice::types::{ + CompressInput, CompressOptions, CompressOutput, CompressorKind, +}; + +/// Only compress result sets with more than this many matching lines. +pub const MIN_MATCHES: usize = 40; +/// Matches kept per file before the "+N more" tally. +pub const TOP_K_PER_FILE: usize = 5; + +pub struct SearchCompressor; + +#[async_trait] +impl Compressor for SearchCompressor { + fn kind(&self) -> CompressorKind { + CompressorKind::Search + } + + async fn compress( + &self, + input: &CompressInput<'_>, + _opts: &CompressOptions, + ) -> Option { + compress(input.content, input.hint.query.as_deref()) + } +} + +struct Match<'a> { + line_no: u64, + body: &'a str, + score: f32, + raw: &'a str, +} + +/// Compress search output. `query` (when known) ranks matches by term density. +pub fn compress(content: &str, query: Option<&str>) -> Option { + // Preserve any non-match preamble/summary lines (e.g. "80 match(es)") and + // group match lines by file in first-seen order. + let mut preamble: Vec<&str> = Vec::new(); + let mut files: Vec<(&str, Vec>)> = Vec::new(); + let mut match_count = 0usize; + + let query_terms: Vec = query + .map(|q| { + q.split_whitespace() + .map(|t| t.to_ascii_lowercase()) + .filter(|t| t.len() >= 2) + .collect() + }) + .unwrap_or_default(); + + for line in content.lines() { + match parse_search_line(line) { + Some((path, line_no, body)) => { + match_count += 1; + let score = score_match(body, &query_terms); + let m = Match { + line_no, + body, + score, + raw: line, + }; + if let Some((_, v)) = files.iter_mut().find(|(p, _)| *p == path) { + v.push(m); + } else { + files.push((path, vec![m])); + } + } + None => { + if !line.trim().is_empty() && files.is_empty() { + // Only keep preamble that appears before any match. + preamble.push(line); + } + } + } + } + + if match_count < MIN_MATCHES || files.is_empty() { + return None; + } + + let mut out = String::with_capacity(content.len() / 2 + 64); + for line in &preamble { + let _ = writeln!(out, "{line}"); + } + let _ = writeln!( + out, + "[search: {} match(es) across {} file(s) · top {} per file · full set via retrieve footer]", + match_count, + files.len(), + TOP_K_PER_FILE + ); + + for (path, mut matches) in files { + let total = matches.len(); + if total <= TOP_K_PER_FILE { + // Keep all in original (line-number) order. + matches.sort_by_key(|m| m.line_no); + for m in &matches { + let _ = writeln!(out, "{}", m.raw); + } + continue; + } + // Rank by score (desc), keep top-K, then re-sort kept by line number so + // the output reads top-to-bottom within the file. + matches.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + let mut kept: Vec<&Match<'_>> = matches.iter().take(TOP_K_PER_FILE).collect(); + kept.sort_by_key(|m| m.line_no); + for m in &kept { + let _ = writeln!(out, "{}:{}:{}", path, m.line_no, m.body); + } + let _ = writeln!( + out, + "[+{} more match(es) in {path}]", + total - TOP_K_PER_FILE + ); + } + + let out = out.trim_end().to_string(); + if out.len() >= content.len() { + return None; + } + log::debug!( + "[tokenjuice][search] {} matches -> {} bytes (from {} bytes)", + match_count, + out.len(), + content.len() + ); + Some(CompressOutput::lossy(out, CompressorKind::Search)) +} + +/// Score a match body. With query terms, density of those terms dominates; +/// otherwise distinctiveness (length) with an importance bump for error/TODO. +fn score_match(body: &str, query_terms: &[String]) -> f32 { + let importance = line_score(body); + if query_terms.is_empty() { + // No query: favour longer, more-distinctive lines, plus importance. + let len_score = (body.trim().len() as f32 / 80.0).min(1.0); + return importance.max(0.2 + 0.8 * len_score); + } + let lower = body.to_ascii_lowercase(); + let hits = query_terms.iter().filter(|t| lower.contains(*t)).count(); + let density = hits as f32 / query_terms.len() as f32; + importance.max(density) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn big_results() -> String { + let mut s = String::from("120 match(es); scanned 3 file(s)\n"); + for i in 0..60 { + let _ = writeln!(s, "src/a.rs:{i}:let value_{i} = compute_long_name_{i}();"); + } + for i in 0..60 { + let _ = writeln!(s, "src/b.rs:{i}:fn helper_function_number_{i}() {{}}"); + } + s + } + + #[test] + fn keeps_top_k_per_file_and_tally() { + let input = big_results(); + let out = compress(&input, None).expect("compresses").text; + assert!(out.contains("more match(es) in src/a.rs"), "{out}"); + assert!(out.contains("more match(es) in src/b.rs")); + // Preamble survives. + assert!(out.contains("120 match(es)")); + assert!(out.len() < input.len()); + } + + #[test] + fn query_ranks_relevant_matches() { + let mut s = String::new(); + for i in 0..50 { + let body = if i == 7 { + "the special needle token appears here".to_string() + } else { + format!("ordinary line content number {i}") + }; + let _ = writeln!(s, "src/x.rs:{i}:{body}"); + } + let out = compress(&s, Some("needle token")).expect("compresses").text; + assert!( + out.contains("special needle token"), + "ranked-in match missing:\n{out}" + ); + } + + #[test] + fn small_result_set_passes_through() { + let s = "a.rs:1:hit\nb.rs:2:hit\n"; + assert!(compress(s, None).is_none()); + } +} diff --git a/src/openhuman/agent/harness/compaction/signals.rs b/src/openhuman/tokenjuice/compressors/signals.rs similarity index 67% rename from src/openhuman/agent/harness/compaction/signals.rs rename to src/openhuman/tokenjuice/compressors/signals.rs index 0ff777a6b..eb21069b7 100644 --- a/src/openhuman/agent/harness/compaction/signals.rs +++ b/src/openhuman/tokenjuice/compressors/signals.rs @@ -1,18 +1,15 @@ -//! Shared importance signals for the compaction compressors. +//! Shared importance signals for the content-router compressors. //! -//! A small, deterministic keyword registry + per-line scorer used by -//! [`super::search`] and [`super::log`] to decide which lines to keep when a -//! tool output is over budget. No ML, no regex compilation cost on the hot -//! path beyond simple case-insensitive substring scans. +//! A small, deterministic keyword registry + per-line scorer used by the +//! search, log, and JSON compressors to decide which lines/rows to keep when a +//! tool output is over budget. No ML, no regex on the hot path beyond simple +//! case-insensitive substring scans. //! -//! Behavior is a clean-room port of headroom's `error_detection` priority -//! signals (Apache-2.0): error/fatal lines score highest, warnings next, -//! importance markers (security/TODO) a small bump, everything else baseline. +//! Clean-room port of Headroom's `error_detection` priority signals +//! (Apache-2.0): error/fatal lines score highest, warnings next, importance +//! markers (security/TODO) a small bump, everything else baseline. -/// Keywords that mark a hard failure. Matched case-insensitively as -/// substrings. Kept deliberately small and high-precision — a false positive -/// just means we keep a line we could have dropped, which is the safe -/// direction. +/// Keywords that mark a hard failure (case-insensitive substrings). const ERROR_KEYWORDS: &[&str] = &[ "error", "fatal", @@ -32,8 +29,7 @@ const ERROR_KEYWORDS: &[&str] = &[ /// Keywords that mark a warning. Lower weight than errors. const WARNING_KEYWORDS: &[&str] = &["warning", "warn:", "[warn]", "deprecated"]; -/// Keywords that bump importance regardless of severity — things an agent -/// almost always wants to see even in a truncated view. +/// Keywords that bump importance regardless of severity. const IMPORTANCE_KEYWORDS: &[&str] = &[ "security", "vulnerability", @@ -51,16 +47,13 @@ pub const SCORE_WARNING: f32 = 0.6; pub const SCORE_IMPORTANCE: f32 = 0.4; pub const SCORE_BASELINE: f32 = 0.1; -/// True if any error keyword appears in `text` (case-insensitive). Cheap -/// pre-check used to decide whether a blob is worth the log compressor. +/// True if any error keyword appears in `text` (case-insensitive). pub fn has_error_indicators(text: &str) -> bool { let lower = text.to_ascii_lowercase(); ERROR_KEYWORDS.iter().any(|kw| lower.contains(kw)) } -/// Importance score for a single line in `[0.0, 1.0]`. Errors dominate, -/// then warnings, then importance markers; a plain line gets the baseline so -/// ordering is stable and "keep highest N" never discards everything. +/// Importance score for a single line in `[0.0, 1.0]`. pub fn line_score(line: &str) -> f32 { let lower = line.to_ascii_lowercase(); let mut score = SCORE_BASELINE; @@ -84,8 +77,7 @@ pub enum Severity { Other, } -/// Bucket a line into [`Severity`]. Used by [`super::log`] to keep error and -/// warning lines under separate caps. +/// Bucket a line into [`Severity`]. pub fn severity(line: &str) -> Severity { let lower = line.to_ascii_lowercase(); if ERROR_KEYWORDS.iter().any(|kw| lower.contains(kw)) { @@ -120,12 +112,6 @@ mod tests { assert_eq!(line_score(" Compiling foo v0.1.0"), SCORE_BASELINE); } - #[test] - fn importance_markers_bump() { - assert!(line_score("TODO: handle retry") > SCORE_BASELINE); - assert!(line_score("potential security issue here") > SCORE_BASELINE); - } - #[test] fn severity_buckets() { assert_eq!(severity("error[E0382]: borrow"), Severity::Error); diff --git a/src/openhuman/tokenjuice/config_patch.rs b/src/openhuman/tokenjuice/config_patch.rs new file mode 100644 index 000000000..98b3cae8d --- /dev/null +++ b/src/openhuman/tokenjuice/config_patch.rs @@ -0,0 +1,127 @@ +//! Partial-update patch for the `[tokenjuice]` config block, used by the +//! `tokenjuice.settings_update` RPC. Only fields present in the JSON are +//! applied, so the UI can flip a single toggle without resending everything. + +use serde::Deserialize; + +use crate::openhuman::config::TokenjuiceConfig; + +// Field names are snake_case to match the `[tokenjuice]` config keys that +// `tokenjuice.settings_get` returns, so the UI reads and writes the same shape. +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +pub struct TokenjuiceSettingsPatch { + pub router_enabled: Option, + pub ccr_enabled: Option, + pub ccr_disk_enabled: Option, + pub max_cache_entries: Option, + pub max_cache_bytes: Option, + /// TTL seconds; `0` clears the TTL (no expiry). Absent leaves it unchanged. + pub ccr_ttl_secs: Option, + pub min_bytes_to_compress: Option, + pub ccr_min_tokens: Option, + pub search_enabled: Option, + pub code_enabled: Option, + pub html_enabled: Option, + pub ml_compression_enabled: Option, + pub ml_model_id: Option, + pub ml_target_ratio: Option, + pub ml_sidecar_idle_timeout_secs: Option, + pub ml_max_input_chars: Option, + pub ml_device: Option, +} + +impl TokenjuiceSettingsPatch { + /// Apply present fields onto `cfg`, leaving absent ones untouched. + pub fn apply(&self, cfg: &mut TokenjuiceConfig) { + if let Some(v) = self.router_enabled { + cfg.router_enabled = v; + } + if let Some(v) = self.ccr_enabled { + cfg.ccr_enabled = v; + } + if let Some(v) = self.ccr_disk_enabled { + cfg.ccr_disk_enabled = v; + } + if let Some(v) = self.max_cache_entries { + cfg.max_cache_entries = v.max(1); + } + if let Some(v) = self.max_cache_bytes { + cfg.max_cache_bytes = v.max(1); + } + if let Some(v) = self.ccr_ttl_secs { + cfg.ccr_ttl_secs = if v == 0 { None } else { Some(v) }; + } + if let Some(v) = self.min_bytes_to_compress { + cfg.min_bytes_to_compress = v; + } + if let Some(v) = self.ccr_min_tokens { + cfg.ccr_min_tokens = v; + } + if let Some(v) = self.search_enabled { + cfg.search_enabled = v; + } + if let Some(v) = self.code_enabled { + cfg.code_enabled = v; + } + if let Some(v) = self.html_enabled { + cfg.html_enabled = v; + } + if let Some(v) = self.ml_compression_enabled { + cfg.ml_compression_enabled = v; + } + if let Some(v) = &self.ml_model_id { + if !v.trim().is_empty() { + cfg.ml_model_id = v.clone(); + } + } + if let Some(v) = self.ml_target_ratio { + if (0.0..=1.0).contains(&v) { + cfg.ml_target_ratio = v; + } + } + if let Some(v) = self.ml_sidecar_idle_timeout_secs { + cfg.ml_sidecar_idle_timeout_secs = v; + } + if let Some(v) = self.ml_max_input_chars { + cfg.ml_max_input_chars = v; + } + if let Some(v) = &self.ml_device { + if !v.trim().is_empty() { + cfg.ml_device = v.clone(); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn applies_only_present_fields() { + let mut cfg = TokenjuiceConfig::default(); + let patch: TokenjuiceSettingsPatch = + serde_json::from_str(r#"{ "ccr_min_tokens": 1200, "search_enabled": false }"#).unwrap(); + patch.apply(&mut cfg); + assert_eq!(cfg.ccr_min_tokens, 1200); + assert!(!cfg.search_enabled); + // Untouched fields keep defaults. + assert!(cfg.router_enabled); + assert!(cfg.code_enabled); + } + + #[test] + fn ttl_can_be_set_and_cleared() { + let mut cfg = TokenjuiceConfig::default(); + let set: TokenjuiceSettingsPatch = + serde_json::from_str(r#"{ "ccr_ttl_secs": 300 }"#).unwrap(); + set.apply(&mut cfg); + assert_eq!(cfg.ccr_ttl_secs, Some(300)); + // 0 clears the TTL. + let clear: TokenjuiceSettingsPatch = + serde_json::from_str(r#"{ "ccr_ttl_secs": 0 }"#).unwrap(); + clear.apply(&mut cfg); + assert_eq!(cfg.ccr_ttl_secs, None); + } +} diff --git a/src/openhuman/tokenjuice/detect/hint.rs b/src/openhuman/tokenjuice/detect/hint.rs new file mode 100644 index 000000000..a79497ca0 --- /dev/null +++ b/src/openhuman/tokenjuice/detect/hint.rs @@ -0,0 +1,119 @@ +//! Tool-name → content-prior mapping for the content router. +//! +//! The producing tool name is a strong prior on what kind of content a blob +//! holds, so the detector doesn't have to work from scratch for the common +//! case. Ported from the compaction router (`hint_for_tool`) and folded into +//! the richer [`ContentHint`] the new router uses. + +use crate::openhuman::tokenjuice::types::ContentKind; + +/// A coarse prior derived purely from the producing tool name. `Auto` means +/// "no strong prior — run full structural detection". +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ToolPrior { + Search, + Log, + Diff, + Json, + Auto, +} + +/// Map an agent-level tool name to its content prior. Unknown tools fall +/// through to [`ToolPrior::Auto`]. +/// +/// `shell` is deliberately `Auto` — its output is frequently NOT a log (a +/// `find`, a `seq`, a `cat` of CSV, a script printing a list), so it is routed +/// through detection rather than forced to the log compressor. +pub fn tool_prior(tool_name: &str) -> ToolPrior { + match tool_name { + "grep" | "glob_search" | "ripgrep" | "rg" => ToolPrior::Search, + "run_tests" | "run_linter" | "npm_exec" | "node_exec" | "install_tool" | "lsp" => { + ToolPrior::Log + } + "read_diff" | "git_operations" => ToolPrior::Diff, + _ => ToolPrior::Auto, + } +} + +/// Translate a strong tool prior straight into a [`ContentKind`] without +/// structural detection. Returns `None` for `Auto` (detector must decide). +pub fn prior_to_kind(prior: ToolPrior) -> Option { + match prior { + ToolPrior::Search => Some(ContentKind::Search), + ToolPrior::Log => Some(ContentKind::Log), + ToolPrior::Diff => Some(ContentKind::Diff), + ToolPrior::Json => Some(ContentKind::Json), + ToolPrior::Auto => None, + } +} + +/// Map a file extension (no dot, lower-cased by the caller) to a content kind, +/// when it is unambiguous. Returns `None` for extensions that don't pin a kind. +pub fn extension_to_kind(ext: &str) -> Option { + match ext { + "json" | "jsonl" | "ndjson" => Some(ContentKind::Json), + "html" | "htm" | "xhtml" => Some(ContentKind::Html), + "diff" | "patch" => Some(ContentKind::Diff), + "log" => Some(ContentKind::Log), + // Source-code extensions we recognise for the code compressor. Grammar + // availability is checked later by the compressor; unknown languages + // fall back to the brace-depth heuristic. + "rs" | "ts" | "tsx" | "js" | "jsx" | "mjs" | "cjs" | "py" | "pyi" | "go" | "java" + | "kt" | "kts" | "c" | "h" | "cc" | "cpp" | "cxx" | "hpp" | "rb" | "php" | "swift" + | "scala" | "cs" => Some(ContentKind::Code), + _ => None, + } +} + +/// Map a MIME type to a content kind, when unambiguous. Tolerates a `; charset` +/// suffix. +pub fn mime_to_kind(mime: &str) -> Option { + let base = mime.split(';').next().unwrap_or(mime).trim(); + match base { + "application/json" | "text/json" => Some(ContentKind::Json), + "text/html" | "application/xhtml+xml" => Some(ContentKind::Html), + "text/x-diff" | "text/x-patch" => Some(ContentKind::Diff), + _ => { + // application/ and text/x- source types → Code. + if base.starts_with("text/x-") || base == "application/javascript" { + Some(ContentKind::Code) + } else { + None + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tool_priors_map_known_tools() { + assert_eq!(tool_prior("grep"), ToolPrior::Search); + assert_eq!(tool_prior("run_tests"), ToolPrior::Log); + assert_eq!(tool_prior("read_diff"), ToolPrior::Diff); + assert_eq!(tool_prior("file_read"), ToolPrior::Auto); + assert_eq!(tool_prior("shell"), ToolPrior::Auto); + } + + #[test] + fn extensions_map() { + assert_eq!(extension_to_kind("rs"), Some(ContentKind::Code)); + assert_eq!(extension_to_kind("json"), Some(ContentKind::Json)); + assert_eq!(extension_to_kind("html"), Some(ContentKind::Html)); + assert_eq!(extension_to_kind("patch"), Some(ContentKind::Diff)); + assert_eq!(extension_to_kind("xyz"), None); + } + + #[test] + fn mimes_map() { + assert_eq!(mime_to_kind("application/json"), Some(ContentKind::Json)); + assert_eq!( + mime_to_kind("text/html; charset=utf-8"), + Some(ContentKind::Html) + ); + assert_eq!(mime_to_kind("text/x-rust"), Some(ContentKind::Code)); + assert_eq!(mime_to_kind("text/plain"), None); + } +} diff --git a/src/openhuman/tokenjuice/detect/kind.rs b/src/openhuman/tokenjuice/detect/kind.rs new file mode 100644 index 000000000..6593aee2b --- /dev/null +++ b/src/openhuman/tokenjuice/detect/kind.rs @@ -0,0 +1,433 @@ +//! Content-kind detection for the TokenJuice content router. +//! +//! Cheap structural heuristics that classify a blob so the router can pick the +//! right compressor. Ported and extended from the compaction router's +//! `detect.rs` (clean-room port of Headroom's `content_detector`, Apache-2.0), +//! adding HTML and source-code detection and a richer [`ContentHint`] that can +//! carry a MIME type, file extension, producing tool, and a hard override. +//! +//! Resolution precedence (cheap → expensive): +//! 1. `hint.explicit` — hard override, returned verbatim. +//! 2. `hint.mime` / `hint.extension` — unambiguous type tags. +//! 3. `hint.source_tool` — strong tool prior (grep→Search, …). +//! 4. structural detection — JSON → Diff → HTML → Search → Code → Log. + +use crate::openhuman::tokenjuice::detect::hint::{ + extension_to_kind, mime_to_kind, prior_to_kind, tool_prior, ToolPrior, +}; +use crate::openhuman::tokenjuice::types::{ContentHint, ContentKind}; + +/// Resolve the [`ContentKind`] for `content` given a caller [`ContentHint`]. +pub fn detect_content_kind(content: &str, hint: &ContentHint) -> ContentKind { + // 1. Hard override. + if let Some(kind) = hint.explicit { + return kind; + } + // 2. MIME / extension tags. For a Code/Json/Html tag we still sanity-check + // diff bodies (a `shell` cat of a `.json` that is actually a patch is + // rare, but a diff body is unmistakable and cheap to confirm). + if let Some(kind) = hint.mime.as_deref().and_then(mime_to_kind) { + return reconcile_tag(kind, content); + } + if let Some(kind) = hint + .extension + .as_deref() + .map(str::to_ascii_lowercase) + .as_deref() + .and_then(extension_to_kind) + { + return reconcile_tag(kind, content); + } + // 3. Tool prior. + if let Some(tool) = hint.source_tool.as_deref() { + match tool_prior(tool) { + // A Search prior is absolute: grep output is never re-routed. + ToolPrior::Search => return ContentKind::Search, + ToolPrior::Auto => {} + other => { + if let Some(kind) = prior_to_kind(other) { + return reconcile_tag(kind, content); + } + } + } + } + // 4. Full structural detection. + detect(content) +} + +/// A type tag (from MIME/extension/tool) is trusted unless the body is clearly +/// a diff — diffs are unmistakable and worth preferring even when the file +/// extension says otherwise. +fn reconcile_tag(tagged: ContentKind, content: &str) -> ContentKind { + if tagged != ContentKind::Diff && looks_like_diff(content) { + ContentKind::Diff + } else { + tagged + } +} + +/// Full structural detection, in priority order: JSON → diff → HTML → search → +/// code → log → plain text. +pub fn detect(content: &str) -> ContentKind { + let trimmed = content.trim_start(); + if trimmed.is_empty() { + return ContentKind::PlainText; + } + if looks_like_json(content) { + return ContentKind::Json; + } + if looks_like_diff(content) { + return ContentKind::Diff; + } + if looks_like_html(content) { + return ContentKind::Html; + } + if search_line_ratio(content) >= 0.6 { + return ContentKind::Search; + } + if looks_like_code(content) { + return ContentKind::Code; + } + if log_line_ratio(content) >= 0.5 { + return ContentKind::Log; + } + ContentKind::PlainText +} + +/// True if `content` parses as a JSON array of objects (crusher input) or a +/// single non-trivial JSON object/array. Scalars and tiny payloads are ignored. +pub fn looks_like_json(content: &str) -> bool { + let trimmed = content.trim_start(); + let first = trimmed.as_bytes().first().copied(); + if first != Some(b'[') && first != Some(b'{') { + return false; + } + match serde_json::from_str::(trimmed.trim_end()) { + Ok(serde_json::Value::Array(items)) => { + items.len() >= 2 && items.iter().any(|v| v.is_object()) + } + // A standalone object is JSON worth routing when it has enough keys to + // be worth pretty-handling (the JSON compressor declines small ones). + Ok(serde_json::Value::Object(map)) => map.len() >= 2, + _ => false, + } +} + +/// True if `content` parses as a JSON array of objects specifically (the table +/// crusher's strict input). Kept for callers that need the narrow check. +pub fn looks_like_json_array(content: &str) -> bool { + let trimmed = content.trim_start(); + if !trimmed.starts_with('[') { + return false; + } + matches!( + serde_json::from_str::(trimmed.trim_end()), + Ok(serde_json::Value::Array(items)) if items.len() >= 2 && items.iter().any(|v| v.is_object()) + ) +} + +/// True if `content` looks like a unified diff: a `diff --git` header or at +/// least one hunk header (`@@ ... @@`). +pub fn looks_like_diff(content: &str) -> bool { + for line in content.lines().take(400) { + if line.starts_with("diff --git ") || line.starts_with("Index: ") { + return true; + } + if line.starts_with("@@ ") && line[3..].contains("@@") { + return true; + } + } + false +} + +/// True if `content` looks like an HTML document: a doctype / ` bool { + let head = &content[..content.len().min(8192)]; + let lower = head.to_ascii_lowercase(); + if lower.contains("") + || lower.contains("` openings over non-blank lines. + let mut tags = 0usize; + let mut lines = 0usize; + for line in head.lines().take(200) { + if line.trim().is_empty() { + continue; + } + lines += 1; + tags += count_html_tags(line); + } + lines >= 3 && (tags as f32 / lines as f32) >= 1.0 +} + +/// Cheap count of ` usize { + let bytes = line.as_bytes(); + let mut count = 0usize; + let mut i = 0usize; + while i + 1 < bytes.len() { + if bytes[i] == b'<' { + let next = bytes[i + 1]; + if next.is_ascii_alphabetic() || next == b'/' || next == b'!' { + count += 1; + } + } + i += 1; + } + count +} + +/// True if `content` heuristically looks like source code: a meaningful share +/// of lines carry code structure (keywords, braces, semicolons, indentation +/// with operators). Deliberately conservative — detection only fires for `Auto` +/// content with no extension/MIME hint, so false positives are rare. +pub fn looks_like_code(content: &str) -> bool { + const KEYWORDS: &[&str] = &[ + "fn ", + "function ", + "class ", + "def ", + "impl ", + "struct ", + "enum ", + "trait ", + "interface ", + "import ", + "export ", + "package ", + "public ", + "private ", + "const ", + "let ", + "var ", + "return ", + "#include", + "using ", + "namespace ", + ]; + let mut total = 0usize; + let mut code_like = 0usize; + let mut brace_lines = 0usize; + for line in content.lines().take(400) { + let t = line.trim(); + if t.is_empty() { + continue; + } + total += 1; + let has_kw = KEYWORDS.iter().any(|kw| line.contains(kw)); + let ends_struct = t.ends_with('{') || t.ends_with(';') || t.ends_with('}'); + if t.contains('{') || t.contains('}') { + brace_lines += 1; + } + if has_kw || ends_struct { + code_like += 1; + } + } + if total < 5 { + return false; + } + let ratio = code_like as f32 / total as f32; + // Require both a decent code-signal ratio and some brace structure so prose + // with the odd "return" or "class" doesn't trip it. + ratio >= 0.4 && brace_lines >= 2 +} + +/// Fraction of non-empty lines that look like `path:line:...` search hits. +fn search_line_ratio(content: &str) -> f32 { + let mut total = 0usize; + let mut hits = 0usize; + for line in content.lines().take(2000) { + if line.trim().is_empty() { + continue; + } + total += 1; + if parse_search_line(line).is_some() { + hits += 1; + } + } + if total == 0 { + 0.0 + } else { + hits as f32 / total as f32 + } +} + +/// Fraction of lines carrying an error/warning indicator — the log signal. +fn log_line_ratio(content: &str) -> f32 { + use crate::openhuman::tokenjuice::compressors::signals::{severity, Severity}; + let mut total = 0usize; + let mut hits = 0usize; + for line in content.lines().take(2000) { + if line.trim().is_empty() { + continue; + } + total += 1; + if severity(line) != Severity::Other { + hits += 1; + } + } + if total == 0 { + 0.0 + } else { + hits as f32 / total as f32 + } +} + +/// Parse a single grep/ripgrep line into `(path, line_number, content)`. +/// +/// Anchors on the earliest `::` marker, skipping a leading Windows +/// drive prefix (`C:`), so paths may contain `:` (drive), `-`, and spaces. +/// Returns `None` for context lines and non-matches. +pub fn parse_search_line(line: &str) -> Option<(&str, u64, &str)> { + let scan_from = if line.len() >= 2 { + let bytes = line.as_bytes(); + if bytes[0].is_ascii_alphabetic() && bytes[1] == b':' { + 2 + } else { + 0 + } + } else { + 0 + }; + + let rest = &line[scan_from..]; + let mut search_start = 0usize; + while let Some(rel) = rest[search_start..].find(':') { + let colon = search_start + rel; + let after = &rest[colon + 1..]; + let digits_len = after.chars().take_while(|c| c.is_ascii_digit()).count(); + if digits_len > 0 && after.as_bytes().get(digits_len) == Some(&b':') { + let path = &line[..scan_from + colon]; + let num: u64 = after[..digits_len].parse().ok()?; + let body = &after[digits_len + 1..]; + if path.is_empty() { + return None; + } + return Some((path, num, body)); + } + search_start = colon + 1; + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + fn hint() -> ContentHint { + ContentHint::default() + } + + #[test] + fn explicit_override_wins() { + let h = ContentHint { + explicit: Some(ContentKind::Html), + ..Default::default() + }; + // Body is JSON but the explicit hint forces Html. + assert_eq!( + detect_content_kind(r#"[{"a":1},{"b":2}]"#, &h), + ContentKind::Html + ); + } + + #[test] + fn mime_and_extension_route() { + let h = ContentHint { + mime: Some("application/json".into()), + ..Default::default() + }; + assert_eq!( + detect_content_kind("{not even json}", &h), + ContentKind::Json + ); + let h = ContentHint { + extension: Some("RS".into()), + ..Default::default() + }; + assert_eq!(detect_content_kind("anything", &h), ContentKind::Code); + } + + #[test] + fn tool_prior_routes_and_search_is_absolute() { + let h = ContentHint::for_tool("grep"); + // Even a diff-looking body stays Search under a grep prior. + assert_eq!( + detect_content_kind("diff --git a/x b/x\n@@ -1 +1 @@", &h), + ContentKind::Search + ); + let h = ContentHint::for_tool("read_diff"); + assert_eq!( + detect_content_kind("diff --git a/x b/x\n@@ -1 +1 @@\n+a", &h), + ContentKind::Diff + ); + } + + #[test] + fn detect_search_results() { + let c = + "src/main.rs:42:fn process() {\nsrc/lib.rs:7:pub use foo;\nsrc/x.rs:99: let y = 1;"; + assert_eq!(detect_content_kind(c, &hint()), ContentKind::Search); + } + + #[test] + fn detect_diff_json_log() { + assert_eq!( + detect_content_kind("diff --git a/x.rs b/x.rs\n@@ -1,3 +1,4 @@\n+added", &hint()), + ContentKind::Diff + ); + assert_eq!( + detect_content_kind(r#"[{"id":1,"name":"a"},{"id":2,"name":"b"}]"#, &hint()), + ContentKind::Json + ); + assert_eq!( + detect_content_kind( + "Compiling foo\nwarning: unused\nerror[E0382]: borrow of moved value\nerror: aborting", + &hint() + ), + ContentKind::Log + ); + } + + #[test] + fn detect_html() { + let c = "\nx

    hi

    "; + assert_eq!(detect_content_kind(c, &hint()), ContentKind::Html); + } + + #[test] + fn detect_code() { + let c = "use std::fmt;\n\npub fn add(a: i32, b: i32) -> i32 {\n let c = a + b;\n return c;\n}\n\nstruct Foo {\n x: i32,\n}"; + assert_eq!(detect_content_kind(c, &hint()), ContentKind::Code); + } + + #[test] + fn plain_text_passes_through() { + assert_eq!( + detect_content_kind("just some prose about a topic at length here", &hint()), + ContentKind::PlainText + ); + } + + #[test] + fn parse_unix_and_windows_paths() { + assert_eq!( + parse_search_line("src/main.rs:42:fn process() {"), + Some(("src/main.rs", 42, "fn process() {")) + ); + assert_eq!( + parse_search_line(r"C:\Users\me\a.rs:10:let x = 1;"), + Some((r"C:\Users\me\a.rs", 10, "let x = 1;")) + ); + assert_eq!( + parse_search_line("pre-commit-config.yaml:3:foo"), + Some(("pre-commit-config.yaml", 3, "foo")) + ); + assert_eq!(parse_search_line("just a sentence"), None); + } +} diff --git a/src/openhuman/tokenjuice/detect/mod.rs b/src/openhuman/tokenjuice/detect/mod.rs new file mode 100644 index 000000000..da632b49f --- /dev/null +++ b/src/openhuman/tokenjuice/detect/mod.rs @@ -0,0 +1,10 @@ +//! Content-kind detection + tool-name priors for the TokenJuice content router. + +pub mod hint; +pub mod kind; + +pub use hint::{extension_to_kind, mime_to_kind, prior_to_kind, tool_prior, ToolPrior}; +pub use kind::{ + detect, detect_content_kind, looks_like_code, looks_like_diff, looks_like_html, + looks_like_json, looks_like_json_array, parse_search_line, +}; diff --git a/src/openhuman/tokenjuice/ml/mod.rs b/src/openhuman/tokenjuice/ml/mod.rs new file mode 100644 index 000000000..438e59860 --- /dev/null +++ b/src/openhuman/tokenjuice/ml/mod.rs @@ -0,0 +1,70 @@ +//! TokenJuice ML plain-text compressor ("Kompress"). +//! +//! Plain text has no structural skeleton to exploit, so high-quality +//! compression needs a learned model (ModernBERT token/sentence salience). That +//! runs inside the shared [`crate::openhuman::runtime_python_server`] as the +//! `kompress` backend — this module is just the thin Rust entry the +//! [`crate::openhuman::tokenjuice::compressors::ml_text`] compressor calls. +//! +//! Opt-in at runtime via `config.tokenjuice.ml_compression_enabled` (default +//! off) — there is no build-time feature gate, since torch is provisioned at +//! runtime (pip), never linked. Degrades gracefully (`Ok(None)` / `Err`) when +//! the flag is off, the runtime python server is unavailable, or the input is +//! too large — the agent loop never fails because ML compression is missing. + +use std::sync::{OnceLock, RwLock}; + +use anyhow::Result; + +use crate::openhuman::config::Config; +use crate::openhuman::tokenjuice::types::CompressOptions; + +/// Global config snapshot the Kompress backend runs against. Held behind a +/// `RwLock` (not a `OnceLock`) so a live settings update — e.g. toggling +/// `ml_compression_enabled` on from Settings — is picked up without a restart. +fn config_cell() -> &'static RwLock> { + static CONFIG: OnceLock>> = OnceLock::new(); + CONFIG.get_or_init(|| RwLock::new(None)) +} + +/// Install (or replace) the config snapshot. Called at startup and on every +/// `tokenjuice.settings_update` so the runtime sees current values. +pub fn configure(config: Config) { + *config_cell().write().unwrap_or_else(|p| p.into_inner()) = Some(config); +} + +/// Compress `text` via the Kompress backend of the runtime python server. +/// +/// Returns `Ok(Some(compacted))` on a useful result, `Ok(None)` when the flag +/// is off / input too large / output wouldn't help, and `Err` when the backend +/// is unavailable (caller degrades to a native compressor). +pub async fn compress(text: &str, _opts: &CompressOptions) -> Result> { + // Snapshot the current config under the read lock (live-updated by + // `configure` on settings changes), then release it before the await. + let config = { + let guard = config_cell().read().unwrap_or_else(|p| p.into_inner()); + match guard.as_ref() { + Some(c) => c.clone(), + None => anyhow::bail!("tokenjuice ml not configured"), + } + }; + let tj = &config.tokenjuice; + if !tj.ml_compression_enabled { + return Ok(None); + } + if text.len() > tj.ml_max_input_chars { + // Too large for the model — let a native compressor handle it. + return Ok(None); + } + + let resp = crate::openhuman::runtime_python_server::request_kompress(&config, text).await?; + if resp.compressed_text.is_empty() || resp.compressed_text.len() >= text.len() { + return Ok(None); + } + log::debug!( + "[tokenjuice::ml] kompress {} -> {} chars", + resp.input_chars, + resp.output_chars + ); + Ok(Some(resp.compressed_text)) +} diff --git a/src/openhuman/tokenjuice/mod.rs b/src/openhuman/tokenjuice/mod.rs index e9e60fbcb..ba2f3d40c 100644 --- a/src/openhuman/tokenjuice/mod.rs +++ b/src/openhuman/tokenjuice/mod.rs @@ -41,18 +41,94 @@ //! //! When two layers define the same rule `id`, the higher-priority layer wins. +pub mod cache; pub mod classify; +pub mod compress; +pub mod compressors; +pub mod config_patch; +pub mod detect; +pub mod ml; pub mod reduce; pub mod rules; +pub mod savings; +pub mod schemas; pub mod text; +pub mod tokens; pub mod tool_integration; +pub mod tools; pub mod types; +/// Install the full TokenJuice runtime from a [`Config`] in one call: router / +/// compressor options + CCR cache limits + disk tier, savings attribution + +/// snapshot path, and the ML backend config snapshot. Used at startup and after +/// a live settings update. +/// +/// Note: toggling `ml_compression_enabled` and the live compressor/CCR flags +/// takes effect immediately; the ML model id / device snapshot is read once and +/// only changes on restart. +pub fn install_from_config(config: &crate::openhuman::config::Config) { + let tj = &config.tokenjuice; + let options = types::CompressOptions { + router_enabled: tj.router_enabled, + ccr_enabled: tj.ccr_enabled, + search_enabled: tj.search_enabled, + code_enabled: tj.code_enabled, + html_enabled: tj.html_enabled, + ml_text_enabled: tj.ml_compression_enabled, + min_bytes_to_compress: tj.min_bytes_to_compress, + ccr_min_tokens: tj.ccr_min_tokens, + max_inline_chars: None, + }; + let disk_root = tj + .ccr_disk_enabled + .then(|| config.workspace_dir.join(".tokenjuice").join("ccr")); + install_config( + options, + tj.max_cache_entries, + tj.max_cache_bytes, + tj.ccr_ttl_secs, + disk_root, + ); + savings::configure( + config + .default_model + .clone() + .unwrap_or_else(|| crate::openhuman::config::DEFAULT_MODEL.to_string()), + &config.workspace_dir, + ); + ml::configure(config.clone()); +} + +/// All read-only TokenJuice debug controllers (detect / compress / cache_stats +/// / retrieve), for registration in `src/core/all.rs`. +pub fn all_tokenjuice_registered_controllers() -> Vec { + schemas::all_registered_controllers() +} + +/// Declared schemas for the TokenJuice debug controllers. +pub fn all_tokenjuice_controller_schemas() -> Vec { + schemas::all_controller_schemas() +} + #[cfg(test)] #[path = "text_tests.rs"] mod text_tests; +pub use cache::{ + is_recovery_tool, LEGACY_RETRIEVE_TOOL_NAME, NEVER_COMPACT_TOOLS, RECOVERY_TOOL_NAMES, + RETRIEVE_TOOL_NAME, +}; +pub use compress::{compress_content, route}; +pub use compressors::{compressor_for, generic_compressor, Compressor}; +pub use detect::detect_content_kind; pub use reduce::reduce_execution_with_rules; pub use rules::{load_builtin_rules, load_rules, LoadRuleOptions}; -pub use tool_integration::{compact_tool_output, CompactionStats}; -pub use types::{CompactResult, ReduceOptions, ToolExecutionInput}; +pub use tool_integration::{ + compact_output, compact_tool_output, configure, current_options, install_config, + CompactionStats, +}; +pub use tools::TokenjuiceRetrieveTool; +pub use types::{ + CompactResult, CompressInput, CompressOptions, CompressOutput, CompressedOutput, + CompressorKind, ContentHint, ContentKind, ReduceOptions, ToolExecutionInput, +}; diff --git a/src/openhuman/tokenjuice/savings.rs b/src/openhuman/tokenjuice/savings.rs new file mode 100644 index 000000000..db891e513 --- /dev/null +++ b/src/openhuman/tokenjuice/savings.rs @@ -0,0 +1,214 @@ +//! Compaction savings accounting — how many tokens (and $$) the content router +//! has saved. +//! +//! Every time the router compacts a tool result it records the estimated tokens +//! before/after and the cost that would have been paid to send the dropped +//! tokens as **input** to the LLM the result is being compressed for. Cost uses +//! the per-model input price from [`crate::openhuman::agent::cost`]. +//! +//! Aggregates are kept process-global and snapshotted to +//! `workspace_dir/state/tokenjuice_savings.json` so the dashboard survives +//! restarts. Attribution model + snapshot path are installed once at startup +//! via [`configure`]. + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::{Mutex, OnceLock}; + +use serde::{Deserialize, Serialize}; + +use crate::openhuman::tokenjuice::types::{CompressorKind, ContentKind}; + +/// Per-key (model / compressor) rolled-up savings. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SavingsBucket { + pub events: u64, + pub original_tokens: u64, + pub compacted_tokens: u64, + pub tokens_saved: u64, + pub cost_saved_usd: f64, +} + +impl SavingsBucket { + fn add(&mut self, original: u64, compacted: u64, cost: f64) { + self.events += 1; + self.original_tokens += original; + self.compacted_tokens += compacted; + self.tokens_saved += original.saturating_sub(compacted); + self.cost_saved_usd += cost; + } +} + +/// The full savings snapshot returned to callers / the dashboard. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SavingsAggregate { + /// Overall totals across every compaction. + pub total: SavingsBucket, + /// Breakdown by the model the savings were attributed to. + pub by_model: HashMap, + /// Breakdown by which compressor produced the saving. + pub by_compressor: HashMap, +} + +struct State { + aggregate: SavingsAggregate, + /// Model used to price the saved input tokens (the configured default). + attribution_model: String, + /// Where the snapshot is persisted; `None` ⇒ in-memory only. + snapshot_path: Option, +} + +impl Default for State { + fn default() -> Self { + Self { + aggregate: SavingsAggregate::default(), + attribution_model: crate::openhuman::config::DEFAULT_MODEL.to_string(), + snapshot_path: None, + } + } +} + +fn state() -> &'static Mutex { + static STATE: OnceLock> = OnceLock::new(); + STATE.get_or_init(|| Mutex::new(State::default())) +} + +/// Install the attribution model and snapshot location, loading any prior +/// snapshot. Called once at startup from [`crate::openhuman::tokenjuice::install_config`]. +pub fn configure(attribution_model: String, workspace_dir: &std::path::Path) { + let path = workspace_dir.join("state").join("tokenjuice_savings.json"); + let loaded = std::fs::read_to_string(&path) + .ok() + .and_then(|s| serde_json::from_str::(&s).ok()); + let mut st = state().lock().unwrap_or_else(|p| p.into_inner()); + if !attribution_model.trim().is_empty() { + st.attribution_model = attribution_model; + } + st.snapshot_path = Some(path); + if let Some(agg) = loaded { + st.aggregate = agg; + } +} + +/// Record one compaction's savings. `original_tokens`/`compacted_tokens` are the +/// pre/post estimates; the cost saved prices the dropped tokens as input to the +/// attribution model. +pub fn record( + content_kind: ContentKind, + compressor: CompressorKind, + original_tokens: u64, + compacted_tokens: u64, +) { + if original_tokens <= compacted_tokens { + return; + } + let saved = original_tokens - compacted_tokens; + + let mut st = state().lock().unwrap_or_else(|p| p.into_inner()); + let model = st.attribution_model.clone(); + let cost = cost_saved_usd(&model, saved); + + st.aggregate + .total + .add(original_tokens, compacted_tokens, cost); + st.aggregate + .by_model + .entry(model) + .or_default() + .add(original_tokens, compacted_tokens, cost); + st.aggregate + .by_compressor + .entry(compressor.as_str().to_string()) + .or_default() + .add(original_tokens, compacted_tokens, cost); + + let _ = content_kind; // reserved for a future by-kind breakdown + persist(&st); +} + +/// Cost (USD) of sending `tokens_saved` as input to `model`, using the per-model +/// input price. Tool results enter the next turn's context as input tokens, so +/// the input price is the relevant rate. +fn cost_saved_usd(model: &str, tokens_saved: u64) -> f64 { + let pricing = crate::openhuman::agent::cost::lookup_pricing(model); + (tokens_saved as f64) / 1_000_000.0 * pricing.input_per_mtok_usd +} + +fn persist(st: &State) { + let Some(path) = st.snapshot_path.as_ref() else { + return; + }; + if let Some(parent) = path.parent() { + let _ = std::fs::create_dir_all(parent); + } + match serde_json::to_string(&st.aggregate) { + Ok(json) => { + if let Err(e) = std::fs::write(path, json) { + log::debug!("[tokenjuice][savings] snapshot write failed: {e}"); + } + } + Err(e) => log::debug!("[tokenjuice][savings] snapshot serialize failed: {e}"), + } +} + +/// Snapshot the current savings aggregate. +pub fn stats() -> SavingsAggregate { + state() + .lock() + .unwrap_or_else(|p| p.into_inner()) + .aggregate + .clone() +} + +/// The model savings are currently attributed to. +pub fn attribution_model() -> String { + state() + .lock() + .unwrap_or_else(|p| p.into_inner()) + .attribution_model + .clone() +} + +/// Clear all recorded savings (and the persisted snapshot). +pub fn reset() { + let mut st = state().lock().unwrap_or_else(|p| p.into_inner()); + st.aggregate = SavingsAggregate::default(); + persist(&st); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn records_and_aggregates() { + // Use a fresh local state to avoid clobbering the process-global one. + let mut agg = SavingsAggregate::default(); + let cost = cost_saved_usd("agentic-v1", 1000); + agg.total.add(2000, 1000, cost); + agg.by_compressor + .entry("smartcrusher".into()) + .or_default() + .add(2000, 1000, cost); + assert_eq!(agg.total.tokens_saved, 1000); + assert!(agg.total.cost_saved_usd > 0.0); + assert_eq!(agg.by_compressor["smartcrusher"].events, 1); + } + + #[test] + fn cost_uses_input_price() { + // agentic-v1 input is $3/Mtok → 1M saved tokens ≈ $3. + let c = cost_saved_usd("agentic-v1", 1_000_000); + assert!((c - 3.0).abs() < 1e-6, "got {c}"); + } + + #[test] + fn no_record_when_not_smaller() { + let before = stats().total.events; + record(ContentKind::Json, CompressorKind::SmartCrusher, 100, 100); + record(ContentKind::Json, CompressorKind::SmartCrusher, 50, 100); + assert_eq!(stats().total.events, before, "no-op when not smaller"); + } +} diff --git a/src/openhuman/tokenjuice/schemas.rs b/src/openhuman/tokenjuice/schemas.rs new file mode 100644 index 000000000..294cffe4f --- /dev/null +++ b/src/openhuman/tokenjuice/schemas.rs @@ -0,0 +1,381 @@ +//! Read-only RPC controller for inspecting the TokenJuice content router. +//! +//! Diagnostic only — none of these run in the per-tool-output hot path. They +//! let the CLI / debug surfaces see what the router would do (detect a kind, +//! dry-run a compression and show the marker/stats), inspect CCR occupancy, and +//! fetch an offloaded original. Mirrors the controller pattern in +//! `threads/schemas.rs`. + +use serde_json::{Map, Value}; + +use crate::core::all::{ControllerFuture, RegisteredController}; +use crate::core::{ControllerSchema, FieldSchema, TypeSchema}; + +use super::cache; +use super::compress::route; +use super::detect::detect_content_kind; +use super::tool_integration::current_options; +use super::types::{CompressInput, ContentHint}; + +pub fn all_controller_schemas() -> Vec { + vec![ + schemas("detect"), + schemas("compress"), + schemas("cache_stats"), + schemas("retrieve"), + schemas("settings_get"), + schemas("settings_update"), + schemas("savings_stats"), + schemas("savings_reset"), + ] +} + +pub fn all_registered_controllers() -> Vec { + vec![ + RegisteredController { + schema: schemas("detect"), + handler: handle_detect, + }, + RegisteredController { + schema: schemas("compress"), + handler: handle_compress, + }, + RegisteredController { + schema: schemas("cache_stats"), + handler: handle_cache_stats, + }, + RegisteredController { + schema: schemas("retrieve"), + handler: handle_retrieve, + }, + RegisteredController { + schema: schemas("settings_get"), + handler: handle_settings_get, + }, + RegisteredController { + schema: schemas("settings_update"), + handler: handle_settings_update, + }, + RegisteredController { + schema: schemas("savings_stats"), + handler: handle_savings_stats, + }, + RegisteredController { + schema: schemas("savings_reset"), + handler: handle_savings_reset, + }, + ] +} + +pub fn schemas(function: &str) -> ControllerSchema { + match function { + "detect" => ControllerSchema { + namespace: "tokenjuice", + function: "detect", + description: + "Detect the content kind of a blob (json/code/log/search/diff/html/plain_text).", + inputs: vec![ + FieldSchema { + name: "content", + ty: TypeSchema::String, + comment: "The content to classify.", + required: true, + }, + FieldSchema { + name: "tool_name", + ty: TypeSchema::String, + comment: "Optional producing tool name (prior hint).", + required: false, + }, + FieldSchema { + name: "extension", + ty: TypeSchema::String, + comment: "Optional file extension hint (no dot).", + required: false, + }, + ], + outputs: vec![FieldSchema { + name: "kind", + ty: TypeSchema::String, + comment: "Detected content kind.", + required: true, + }], + }, + "compress" => ControllerSchema { + namespace: "tokenjuice", + function: "compress", + description: "Dry-run the content router over a blob and report stats + marker.", + inputs: vec![ + FieldSchema { + name: "content", + ty: TypeSchema::String, + comment: "The content to compress.", + required: true, + }, + FieldSchema { + name: "tool_name", + ty: TypeSchema::String, + comment: "Optional producing tool name (prior hint).", + required: false, + }, + ], + outputs: vec![FieldSchema { + name: "result", + ty: TypeSchema::Json, + comment: "Applied flag, kind, compressor, byte counts, CCR token, and text.", + required: true, + }], + }, + "cache_stats" => ControllerSchema { + namespace: "tokenjuice", + function: "cache_stats", + description: "Report CCR cache occupancy (entry count and total bytes).", + inputs: vec![], + outputs: vec![FieldSchema { + name: "result", + ty: TypeSchema::Json, + comment: "{ entries, bytes }.", + required: true, + }], + }, + "retrieve" => ControllerSchema { + namespace: "tokenjuice", + function: "retrieve", + description: "Fetch a previously-offloaded original from the CCR cache by token.", + inputs: vec![FieldSchema { + name: "token", + ty: TypeSchema::String, + comment: "The CCR token (hash) from a ⟦tj:…⟧ marker.", + required: true, + }], + outputs: vec![FieldSchema { + name: "result", + ty: TypeSchema::Json, + comment: "{ found, content }.", + required: true, + }], + }, + "settings_get" => ControllerSchema { + namespace: "tokenjuice", + function: "settings_get", + description: "Get the current [tokenjuice] configuration block.", + inputs: vec![], + outputs: vec![FieldSchema { + name: "settings", + ty: TypeSchema::Json, + comment: "The tokenjuice config (router/CCR/compressor toggles + ML fields).", + required: true, + }], + }, + "settings_update" => ControllerSchema { + namespace: "tokenjuice", + function: "settings_update", + description: "Patch the [tokenjuice] config (any subset of fields), persist, and apply live.", + inputs: vec![FieldSchema { + name: "patch", + ty: TypeSchema::Json, + comment: "Partial tokenjuice settings; only present fields are changed.", + required: true, + }], + outputs: vec![FieldSchema { + name: "settings", + ty: TypeSchema::Json, + comment: "The full settings after applying the patch.", + required: true, + }], + }, + "savings_stats" => ControllerSchema { + namespace: "tokenjuice", + function: "savings_stats", + description: "Token + cost savings the content router has accrued (total + by model + by compressor).", + inputs: vec![], + outputs: vec![FieldSchema { + name: "result", + ty: TypeSchema::Json, + comment: "{ attributionModel, total, byModel, byCompressor, cache }.", + required: true, + }], + }, + "savings_reset" => ControllerSchema { + namespace: "tokenjuice", + function: "savings_reset", + description: "Clear all recorded savings statistics.", + inputs: vec![], + outputs: vec![FieldSchema { + name: "ok", + ty: TypeSchema::Bool, + comment: "True once reset.", + required: true, + }], + }, + _other => ControllerSchema { + namespace: "tokenjuice", + function: "unknown", + description: "Unknown tokenjuice controller function.", + inputs: vec![], + outputs: vec![FieldSchema { + name: "error", + ty: TypeSchema::String, + comment: "Lookup error details.", + required: true, + }], + }, + } +} + +// ── Handlers ───────────────────────────────────────────────────────── + +fn str_param(params: &Map, key: &str) -> Option { + params.get(key).and_then(Value::as_str).map(str::to_string) +} + +fn handle_detect(params: Map) -> ControllerFuture { + Box::pin(async move { + let content = str_param(¶ms, "content").ok_or("missing 'content'")?; + let hint = ContentHint { + source_tool: str_param(¶ms, "tool_name"), + extension: str_param(¶ms, "extension"), + ..Default::default() + }; + let kind = detect_content_kind(&content, &hint); + Ok(serde_json::json!({ "kind": kind.as_str() })) + }) +} + +fn handle_compress(params: Map) -> ControllerFuture { + Box::pin(async move { + let content = str_param(¶ms, "content").ok_or("missing 'content'")?; + let hint = ContentHint { + source_tool: str_param(¶ms, "tool_name"), + ..Default::default() + }; + let opts = current_options(); + let input = CompressInput { + content: &content, + kind: super::types::ContentKind::PlainText, + hint: &hint, + exit_code: None, + command: None, + argv: None, + original_bytes: content.len(), + }; + let res = route(input, &opts).await; + Ok(serde_json::json!({ + "applied": res.applied, + "kind": res.content_kind.as_str(), + "compressor": res.compressor.as_str(), + "lossy": res.lossy, + "originalBytes": res.original_bytes, + "compactedBytes": res.compacted_bytes, + "ccrToken": res.ccr_token, + "text": res.text, + })) + }) +} + +fn handle_cache_stats(_params: Map) -> ControllerFuture { + Box::pin(async move { + let (entries, bytes) = cache::stats(); + Ok(serde_json::json!({ "entries": entries, "bytes": bytes })) + }) +} + +fn handle_retrieve(params: Map) -> ControllerFuture { + Box::pin(async move { + let token = str_param(¶ms, "token").ok_or("missing 'token'")?; + match cache::retrieve(&token) { + Some(content) => Ok(serde_json::json!({ "found": true, "content": content })), + None => Ok(serde_json::json!({ "found": false, "content": Value::Null })), + } + }) +} + +fn handle_settings_get(_params: Map) -> ControllerFuture { + Box::pin(async move { + let config = crate::openhuman::config::Config::load_or_init() + .await + .map_err(|e| format!("load config: {e}"))?; + let settings = serde_json::to_value(&config.tokenjuice) + .map_err(|e| format!("serialize tokenjuice settings: {e}"))?; + Ok(serde_json::json!({ "settings": settings })) + }) +} + +fn handle_settings_update(params: Map) -> ControllerFuture { + Box::pin(async move { + // Accept either {"patch": {...}} or a bare object of fields. + let patch = params + .get("patch") + .cloned() + .unwrap_or_else(|| Value::Object(params.clone())); + let patch: super::config_patch::TokenjuiceSettingsPatch = + serde_json::from_value(patch).map_err(|e| format!("invalid patch: {e}"))?; + + let mut config = crate::openhuman::config::Config::load_or_init() + .await + .map_err(|e| format!("load config: {e}"))?; + patch.apply(&mut config.tokenjuice); + config + .save() + .await + .map_err(|e| format!("save config: {e}"))?; + + // Re-install so router flags / CCR limits / threshold take effect live. + crate::openhuman::tokenjuice::install_from_config(&config); + + let settings = serde_json::to_value(&config.tokenjuice) + .map_err(|e| format!("serialize tokenjuice settings: {e}"))?; + Ok(serde_json::json!({ "settings": settings })) + }) +} + +fn handle_savings_stats(_params: Map) -> ControllerFuture { + Box::pin(async move { + let agg = super::savings::stats(); + let (entries, bytes) = cache::stats(); + Ok(serde_json::json!({ + "attributionModel": super::savings::attribution_model(), + "total": agg.total, + "byModel": agg.by_model, + "byCompressor": agg.by_compressor, + "cache": { "entries": entries, "bytes": bytes }, + })) + }) +} + +fn handle_savings_reset(_params: Map) -> ControllerFuture { + Box::pin(async move { + super::savings::reset(); + Ok(serde_json::json!({ "ok": true })) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn detect_handler_classifies_json() { + let mut p = Map::new(); + p.insert( + "content".into(), + Value::String(r#"[{"a":1,"b":2},{"a":3,"b":4}]"#.into()), + ); + let out = handle_detect(p).await.unwrap(); + assert_eq!(out["kind"], "json"); + } + + #[tokio::test] + async fn cache_stats_handler_returns_counts() { + cache::offload("tokenjuice controller stats unique payload here"); + let out = handle_cache_stats(Map::new()).await.unwrap(); + assert!(out["entries"].as_u64().unwrap() >= 1); + } + + #[test] + fn all_schemas_have_namespace() { + for s in all_controller_schemas() { + assert_eq!(s.namespace, "tokenjuice"); + } + } +} diff --git a/src/openhuman/tokenjuice/tokens.rs b/src/openhuman/tokenjuice/tokens.rs new file mode 100644 index 000000000..f9b0335aa --- /dev/null +++ b/src/openhuman/tokenjuice/tokens.rs @@ -0,0 +1,46 @@ +//! Lightweight token estimation for compaction savings accounting. +//! +//! The agent harness gets authoritative token counts from the provider, but +//! the content router runs *before* any provider call, so it has no exact +//! count for a tool result. For savings insights we use the standard ~4 +//! characters-per-token heuristic (close enough for English text / code / +//! JSON to drive cost estimates and the savings dashboard). This is the same +//! order of approximation Headroom reports its savings with. + +/// Average characters per token used by the estimate. +pub const CHARS_PER_TOKEN: f64 = 4.0; + +/// Estimate the number of tokens in `text` (≈ chars / 4, minimum 1 for any +/// non-empty input). Uses `chars().count()` so multi-byte text isn't +/// over-counted by byte length. +pub fn estimate_tokens(text: &str) -> u64 { + if text.is_empty() { + return 0; + } + let chars = text.chars().count() as f64; + (chars / CHARS_PER_TOKEN).ceil().max(1.0) as u64 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_is_zero() { + assert_eq!(estimate_tokens(""), 0); + } + + #[test] + fn rough_quarter_of_chars() { + assert_eq!(estimate_tokens("abcd"), 1); + assert_eq!(estimate_tokens(&"x".repeat(400)), 100); + // Any non-empty input is at least one token. + assert_eq!(estimate_tokens("a"), 1); + } + + #[test] + fn counts_chars_not_bytes() { + // 4 multi-byte chars → 1 token, not 12 bytes → 3 tokens. + assert_eq!(estimate_tokens("日本語訳"), 1); + } +} diff --git a/src/openhuman/tokenjuice/tool_integration.rs b/src/openhuman/tokenjuice/tool_integration.rs index 4aab1f29e..79a3fd1cd 100644 --- a/src/openhuman/tokenjuice/tool_integration.rs +++ b/src/openhuman/tokenjuice/tool_integration.rs @@ -1,41 +1,88 @@ -//! Glue between the agent tool loop and the tokenjuice reduction engine. +//! Glue between the agent tool loop and the TokenJuice content router. //! -//! Exposes a single entry point — [`compact_tool_output`] — that the agent -//! loop calls after a tool returns its output. It builds a -//! [`ToolExecutionInput`] from whatever metadata the caller has (tool name, -//! JSON arguments, exit code) and runs the reduction pipeline with the -//! lazily-cached builtin rule set. +//! Exposes the entry points the agent loop calls after a tool returns output: //! -//! The function is **pass-through safe**: if reduction does not meaningfully -//! shrink the payload (below [`MIN_COMPACT_RATIO`]) or if the input is already -//! under [`MIN_COMPACT_INPUT_BYTES`], the original string is returned -//! untouched. Callers do not need to guard the call site. +//! - [`compact_tool_output`] — full version with the tool's JSON arguments and +//! exit code; derives a command/argv and content hint, routes through the +//! content router, and returns `(text, CompactionStats)`. +//! - [`compact_output`] — minimal version (content + tool name + enable flag) +//! for call sites that only have those, returning just the text. +//! +//! Both are **pass-through safe**: if compression doesn't meaningfully shrink +//! the payload, or the input is under the byte floor, or the router/CCR is +//! disabled, the original string is returned untouched. +//! +//! Runtime options (the `[tokenjuice]` config block) are installed once at +//! startup via [`configure`]; callers don't thread `Config` through. -use once_cell::sync::Lazy; +use once_cell::sync::OnceCell; use serde_json::Value; +use std::sync::RwLock; -use super::reduce::reduce_execution_with_rules; -use super::rules::load_builtin_rules; -use super::types::{CompiledRule, ReduceOptions, ToolExecutionInput}; +use super::compress::route; +use super::types::{CompressInput, CompressOptions, ContentHint}; -/// Skip compaction for outputs smaller than this (bytes). Tiny outputs have -/// no headroom to benefit from head/tail summarisation and risk being -/// distorted by rule matches that were designed for long logs. -const MIN_COMPACT_INPUT_BYTES: usize = 512; +/// Skip compaction for outputs smaller than this (bytes) by default. Tiny +/// outputs have no headroom and risk distortion. Overridable per the config's +/// `min_bytes_to_compress` once [`configure`] runs. +const DEFAULT_MIN_COMPACT_INPUT_BYTES: usize = 512; -/// Keep the compacted form only if it is at most this fraction of the -/// original length. Between `MIN_COMPACT_RATIO` and 1.0 the compaction is -/// considered not worthwhile and the raw output is returned. -const MIN_COMPACT_RATIO: f64 = 0.95; +/// Process-global runtime options, installed from config at startup. +fn options_cell() -> &'static RwLock { + static OPTS: OnceCell> = OnceCell::new(); + OPTS.get_or_init(|| { + RwLock::new(CompressOptions { + min_bytes_to_compress: DEFAULT_MIN_COMPACT_INPUT_BYTES, + ..Default::default() + }) + }) +} -static BUILTIN_RULES: Lazy> = Lazy::new(load_builtin_rules); +/// Install the runtime [`CompressOptions`] (called once from config at startup). +/// Also configures the CCR cache limits/disk tier indirectly via the caller. +pub fn configure(opts: CompressOptions) { + *options_cell().write().unwrap_or_else(|p| p.into_inner()) = opts; +} -/// Statistics for a single compaction call. +/// Snapshot the current runtime options. +pub fn current_options() -> CompressOptions { + options_cell() + .read() + .unwrap_or_else(|p| p.into_inner()) + .clone() +} + +/// Install the full TokenJuice runtime configuration in one call at startup: +/// router/compressor options, CCR cache limits, and the optional on-disk tier. +/// Kept free of the config-schema type so `tokenjuice` stays decoupled — the +/// caller maps `Config.tokenjuice` into these primitives. +#[allow(clippy::too_many_arguments)] +pub fn install_config( + options: CompressOptions, + max_cache_entries: usize, + max_cache_bytes: usize, + ccr_ttl_secs: Option, + disk_tier_root: Option, +) { + configure(options); + super::cache::configure(max_cache_entries, max_cache_bytes, ccr_ttl_secs); + // Enable or disable the disk tier to match the setting — a `None` here means + // the user turned it off, so clear any previously-installed disk root rather + // than leaving the process writing originals to disk until restart. + match disk_tier_root { + Some(root) => super::cache::enable_disk_tier(root), + None => super::cache::disable_disk_tier(), + } + log::debug!("[tokenjuice] runtime config installed"); +} + +/// Statistics for a single compaction call (back-compat shape). #[derive(Debug, Clone)] pub struct CompactionStats { pub tool_name: String, pub original_bytes: usize, pub compacted_bytes: usize, + /// The compressor kind (or `none/...`) that handled the output. pub rule_id: String, pub applied: bool, } @@ -50,23 +97,17 @@ impl CompactionStats { } } -/// Compact a tool call's output using tokenjuice's builtin rule set. +/// Compact a tool call's output through the content router. /// -/// * `tool_name` — the agent-level tool name (e.g. `"shell"`, -/// `"browser_navigate"`). When the tool is a shell wrapper, callers should -/// pass the *underlying* tool name (e.g. `"git"`) by extracting it from -/// `arguments`, but passing the agent tool name also works — rules also -/// match on `commandIncludes` / `argvIncludes`. -/// * `arguments` — the raw JSON arguments the agent passed to the tool. -/// Used to heuristically derive `command` / `argv` for shell-style tools. +/// * `tool_name` — the agent-level tool name (`shell`, `grep`, `browser_navigate`). +/// * `arguments` — the raw JSON arguments; used to derive command/argv (for the +/// log/command rule path) and a file extension (for code/JSON/HTML hints). /// * `output` — the captured tool output (already credential-scrubbed). -/// * `exit_code` — if known; enables failure-preserving behaviour (rules -/// with a `failure` block use `failure.head`/`failure.tail` instead of the -/// default summarise window when this is non-zero). +/// * `exit_code` — enables failure-preserving behaviour in the log compressor. /// -/// Returns `(compacted_text, stats)`. When `stats.applied == false` the -/// returned string is the untouched original. -pub fn compact_tool_output( +/// Returns `(text, stats)`. When `stats.applied == false` the text is the +/// untouched original. +pub async fn compact_tool_output( tool_name: &str, arguments: Option<&Value>, output: &str, @@ -74,118 +115,68 @@ pub fn compact_tool_output( ) -> (String, CompactionStats) { let original_bytes = output.len(); - if original_bytes < MIN_COMPACT_INPUT_BYTES { - log::debug!( - "[tokenjuice] skipping tool={} bytes={} reason=too-small", - tool_name, - original_bytes - ); + // A recovery tool's output is the original we previously offloaded — never + // re-compact it, or the agent could never see the full data it asked for. + if super::cache::is_recovery_tool(tool_name) { return ( - output.to_owned(), + output.to_string(), CompactionStats { - tool_name: tool_name.to_owned(), + tool_name: tool_name.to_string(), original_bytes, compacted_bytes: original_bytes, - rule_id: "none/too-small".to_owned(), + rule_id: "none/recovery-tool".to_string(), applied: false, }, ); } + let opts = current_options(); let (command, argv) = extract_command_argv(arguments); - // Whether this execution looks like a shell command (vs. a domain tool that - // returns structured/large data, e.g. a Composio action). Domain tools carry - // no `command`/`argv` in their arguments. - let has_command = command.is_some() || argv.as_ref().is_some_and(|a| !a.is_empty()); - - let input = ToolExecutionInput { - tool_name: tool_name.to_owned(), - command, - argv, - stdout: Some(output.to_owned()), - exit_code, + let hint = ContentHint { + source_tool: Some(tool_name.to_string()), + extension: extract_extension(arguments), + query: extract_query(arguments), ..Default::default() }; - let result = reduce_execution_with_rules(input, &BUILTIN_RULES, &ReduceOptions::default()); - let compacted_bytes = result.inline_text.len(); - let rule_id = result - .classification - .matched_reducer - .clone() - .unwrap_or_else(|| result.classification.family.clone()); - - let ratio = if original_bytes == 0 { - 1.0 - } else { - compacted_bytes as f64 / original_bytes as f64 + let input = CompressInput { + content: output, + kind: super::types::ContentKind::PlainText, + hint: &hint, + exit_code, + command, + argv, + original_bytes, }; - // The `generic/fallback` reducer is a line-oriented head/tail summariser - // meant for *command* output tokenjuice has no specific rule for. It must - // NOT compact domain-tool output (no command/argv): those payloads are - // structured and are handled downstream by the sub-agent progressive - // -disclosure handoff / payload summariser, not by blind head/tail - // truncation. Letting the generic fallback fire here clamps every large - // tool result to ~1.2k chars and silently preempts the handoff. Specific, - // tool/argv-matched rules still apply to domain tools as before. - let generic_fallback_on_domain_tool = rule_id == "generic/fallback" && !has_command; + let res = route(input, &opts).await; + let stats = CompactionStats { + tool_name: tool_name.to_string(), + original_bytes, + compacted_bytes: res.compacted_bytes, + rule_id: if res.applied { + res.compressor.as_str().to_string() + } else { + format!("none/{}", res.content_kind.as_str()) + }, + applied: res.applied, + }; + (res.text, stats) +} - let applied = !generic_fallback_on_domain_tool - && ratio <= MIN_COMPACT_RATIO - && compacted_bytes < original_bytes; - - if applied { - log::info!( - "[tokenjuice] compacted tool={} rule={} {}->{} bytes (ratio={:.2})", - tool_name, - rule_id, - original_bytes, - compacted_bytes, - ratio - ); - ( - result.inline_text, - CompactionStats { - tool_name: tool_name.to_owned(), - original_bytes, - compacted_bytes, - rule_id, - applied: true, - }, - ) - } else { - log::debug!( - "[tokenjuice] pass-through tool={} rule={} {}->{} bytes (ratio={:.2} > {})", - tool_name, - rule_id, - original_bytes, - compacted_bytes, - ratio, - MIN_COMPACT_RATIO - ); - ( - output.to_owned(), - CompactionStats { - tool_name: tool_name.to_owned(), - original_bytes, - compacted_bytes: original_bytes, - rule_id, - applied: false, - }, - ) +/// Minimal compaction for call sites that only have content + tool name. The +/// `enabled` flag is an explicit kill-switch on top of the configured options. +pub async fn compact_output(content: String, tool_name: &str, enabled: bool) -> String { + // The call-site `enabled` flag and the configured router switch are both + // hard off-switches; either one short-circuits to the untouched original. + if !enabled || !current_options().router_enabled { + return content; } + let (text, _stats) = compact_tool_output(tool_name, None, &content, None).await; + text } /// Derive `(command, argv)` from a tool's JSON arguments. -/// -/// Handles the common shapes: -/// * `{"command": "git status"}` — string command (whitespace-split into argv). -/// * `{"command": "git", "args": ["status"]}` — explicit split. -/// * `{"argv": ["git", "status"]}` — pre-built argv. -/// * `{"cmd": "..."}` — alternate field name. -/// -/// Returns `(None, None)` when the arguments don't look shell-like. fn extract_command_argv(arguments: Option<&Value>) -> (Option, Option>) { let Some(Value::Object(map)) = arguments else { return (None, None); @@ -213,7 +204,6 @@ fn extract_command_argv(arguments: Option<&Value>) -> (Option, Option = cmd.split_whitespace().map(|s| s.to_owned()).collect(); return (Some(cmd.to_owned()), (!argv.is_empty()).then_some(argv)); } @@ -221,98 +211,94 @@ fn extract_command_argv(arguments: Option<&Value>) -> (Option, Option) -> Option { + let Some(Value::Object(map)) = arguments else { + return None; + }; + let path = ["path", "file_path", "file", "filename"] + .iter() + .find_map(|k| map.get(*k).and_then(Value::as_str))?; + let ext = std::path::Path::new(path) + .extension() + .and_then(|e| e.to_str())?; + Some(ext.to_ascii_lowercase()) +} + +/// Derive a search-query hint from common query-bearing argument shapes. +fn extract_query(arguments: Option<&Value>) -> Option { + let Some(Value::Object(map)) = arguments else { + return None; + }; + ["query", "pattern", "search", "q", "regex"] + .iter() + .find_map(|k| map.get(*k).and_then(Value::as_str)) + .map(str::to_string) +} + #[cfg(test)] mod tests { use super::*; use serde_json::json; - #[test] - fn skips_short_output() { - let (out, stats) = compact_tool_output("shell", None, "hello world", Some(0)); + #[tokio::test] + async fn skips_short_output() { + let (out, stats) = compact_tool_output("shell", None, "hello world", Some(0)).await; assert_eq!(out, "hello world"); assert!(!stats.applied); - assert_eq!(stats.rule_id, "none/too-small"); assert_eq!(stats.original_bytes, 11); } - #[test] - fn compacts_long_git_status_via_argv() { + #[tokio::test] + async fn compacts_long_git_status_via_argv() { let mut lines = vec!["On branch main".to_owned()]; for i in 0..200 { lines.push(format!("\tmodified: src/file_{i}.rs")); } let output = lines.join("\n"); let args = json!({"command": "git status"}); - let (compacted, stats) = compact_tool_output("shell", Some(&args), &output, Some(0)); + let (compacted, stats) = compact_tool_output("shell", Some(&args), &output, Some(0)).await; assert!(stats.applied, "expected compaction, got {:?}", stats); assert!(compacted.len() < output.len()); - assert!(stats.rule_id.starts_with("git/")); } - #[test] - fn passes_through_incompressible_output() { + #[tokio::test] + async fn passes_through_incompressible_output() { let unique_lines: Vec = (0..200) .map(|i| format!("unique-payload-chunk-{i}-{}", "x".repeat(30))) .collect(); let output = unique_lines.join("\n"); - let (returned, stats) = compact_tool_output("unknown_tool", None, &output, Some(0)); - // Either the fallback rule compacted it (applied == true) or it - // passed through because ratio > threshold. Both are valid; we only - // assert the function never loses data silently. - if stats.applied { - assert_ne!(returned, output); - assert!(stats.compacted_bytes < stats.original_bytes); - } else { + let (returned, stats) = compact_tool_output("unknown_tool", None, &output, Some(0)).await; + if !stats.applied { assert_eq!(returned, output); } } + #[tokio::test] + async fn disabled_flag_is_passthrough() { + let big = "x".repeat(5000); + assert_eq!(compact_output(big.clone(), "grep", false).await, big); + } + #[test] fn extract_argv_handles_common_shapes() { let (cmd, argv) = extract_command_argv(Some(&json!({"command": "git status"}))); assert_eq!(cmd.as_deref(), Some("git status")); assert_eq!(argv.unwrap(), vec!["git", "status"]); - let (cmd, argv) = extract_command_argv(Some(&json!({ - "command": "cargo", - "args": ["test", "--lib"], - }))); - assert_eq!(cmd.as_deref(), Some("cargo test --lib")); - assert_eq!(argv.unwrap(), vec!["cargo", "test", "--lib"]); - - let (cmd, argv) = extract_command_argv(Some(&json!({ - "argv": ["npm", "install"], - }))); - assert_eq!(cmd.as_deref(), Some("npm install")); - assert_eq!(argv.unwrap(), vec!["npm", "install"]); - - let (cmd, argv) = extract_command_argv(Some(&json!({"unrelated": 1}))); - assert!(cmd.is_none()); - assert!(argv.is_none()); - - let (cmd, argv) = extract_command_argv(None); - assert!(cmd.is_none()); - assert!(argv.is_none()); + let (cmd, _) = extract_command_argv(Some(&json!({"command": "cargo", "args": ["test"]}))); + assert_eq!(cmd.as_deref(), Some("cargo test")); } #[test] - fn ratio_computation() { - let stats = CompactionStats { - tool_name: "x".into(), - original_bytes: 1000, - compacted_bytes: 250, - rule_id: "r".into(), - applied: true, - }; - assert!((stats.ratio() - 0.25).abs() < 1e-9); - - let empty = CompactionStats { - tool_name: "x".into(), - original_bytes: 0, - compacted_bytes: 0, - rule_id: "r".into(), - applied: false, - }; - assert!((empty.ratio() - 1.0).abs() < 1e-9); + fn extract_extension_and_query() { + assert_eq!( + extract_extension(Some(&json!({"path": "src/lib.rs"}))).as_deref(), + Some("rs") + ); + assert_eq!( + extract_query(Some(&json!({"pattern": "foo bar"}))).as_deref(), + Some("foo bar") + ); } } diff --git a/src/openhuman/tokenjuice/tools.rs b/src/openhuman/tokenjuice/tools.rs new file mode 100644 index 000000000..8b547ee16 --- /dev/null +++ b/src/openhuman/tokenjuice/tools.rs @@ -0,0 +1,166 @@ +//! Agent tool: `tokenjuice_retrieve` — fetch the original of a compacted result. +//! +//! The content router may replace a large tool result with a compacted view and +//! a `⟦tj:⟧` marker, stashing the original in the CCR store +//! ([`crate::openhuman::tokenjuice::cache::store`]). This tool hands the +//! original back on demand — fully or by a byte/line range — so even lossy +//! compaction stays reversible. +//! +//! Read-only, no side effects, no path/network access. + +use async_trait::async_trait; +use serde_json::{json, Value}; + +use crate::openhuman::tokenjuice::cache::{self, store::RangeUnit}; +use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolResult}; + +pub struct TokenjuiceRetrieveTool; + +impl TokenjuiceRetrieveTool { + pub fn new() -> Self { + Self + } +} + +impl Default for TokenjuiceRetrieveTool { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl Tool for TokenjuiceRetrieveTool { + fn name(&self) -> &str { + cache::RETRIEVE_TOOL_NAME + } + + fn description(&self) -> &str { + "Retrieve the full, original text of a tool result that was compacted to save \ + context. When output shows a marker like `⟦tj:a1b2c3d4⟧` (or a legacy \ + `retrieve_tool_output(\"…\")` footer), call this with that token to get the \ + complete original back. Optionally pass a `range` to fetch just a byte or line \ + slice. Use it only when you actually need the dropped detail — the compacted \ + view is usually enough." + } + + fn parameters_schema(&self) -> Value { + json!({ + "type": "object", + "properties": { + "token": { + "type": "string", + "description": "The hash from a ⟦tj:…⟧ marker (or legacy retrieve footer)." + }, + "range": { + "type": "object", + "description": "Optional slice of the original to return.", + "properties": { + "start": { "type": "integer", "minimum": 0 }, + "end": { "type": "integer", "minimum": 0 }, + "unit": { "type": "string", "enum": ["bytes", "lines"] } + }, + "required": ["start", "end"] + } + }, + "required": ["token"] + }) + } + + fn permission_level(&self) -> PermissionLevel { + PermissionLevel::ReadOnly + } + + async fn execute(&self, args: Value) -> anyhow::Result { + // Accept `token` (canonical) or `hash` (legacy arg name). + let token = args + .get("token") + .or_else(|| args.get("hash")) + .and_then(|v| v.as_str()) + .map(str::trim) + .filter(|s| !s.is_empty()); + let Some(token) = token else { + return Ok(ToolResult::error( + "tokenjuice_retrieve: missing required 'token' argument".to_string(), + )); + }; + + // Optional range. + if let Some(range) = args.get("range").filter(|v| v.is_object()) { + let start = range.get("start").and_then(Value::as_u64).unwrap_or(0) as usize; + let end = range.get("end").and_then(Value::as_u64).unwrap_or(u64::MAX) as usize; + let unit = match range.get("unit").and_then(Value::as_str) { + Some("bytes") => RangeUnit::Bytes, + _ => RangeUnit::Lines, + }; + return match cache::retrieve_range(token, start, end, unit) { + Some(slice) => { + log::debug!( + "[tokenjuice][ccr] retrieved range token={token} {start}..{end} {} bytes", + slice.len() + ); + Ok(ToolResult::success(slice)) + } + None => Ok(ToolResult::error(miss_message(token))), + }; + } + + match cache::retrieve(token) { + Some(original) => { + log::debug!( + "[tokenjuice][ccr] retrieved token={token} bytes={}", + original.len() + ); + Ok(ToolResult::success(original)) + } + None => Ok(ToolResult::error(miss_message(token))), + } + } +} + +fn miss_message(token: &str) -> String { + format!( + "tokenjuice_retrieve: no cached original for token '{token}' \ + (it may have been evicted; re-run the tool to regenerate it)" + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::tokenjuice::cache::store; + + #[tokio::test] + async fn retrieves_offloaded_original() { + let original = "ORIGINAL TOKENJUICE PAYLOAD ".repeat(20); + let hash = store::offload(&original); + let tool = TokenjuiceRetrieveTool::new(); + let res = tool.execute(json!({ "token": hash })).await.unwrap(); + assert!(!res.is_error); + assert_eq!(res.output(), original); + } + + #[tokio::test] + async fn retrieves_line_range() { + let original = "r0\nr1\nr2\nr3\nr4"; + let hash = store::offload(original); + let tool = TokenjuiceRetrieveTool::new(); + let res = tool + .execute(json!({ "token": hash, "range": { "start": 1, "end": 3, "unit": "lines" } })) + .await + .unwrap(); + assert!(!res.is_error); + assert_eq!(res.output(), "r1\nr2"); + } + + #[tokio::test] + async fn missing_token_is_error() { + let tool = TokenjuiceRetrieveTool::new(); + let res = tool + .execute(json!({ "token": "deadbeefcafe" })) + .await + .unwrap(); + assert!(res.is_error); + let res2 = tool.execute(json!({})).await.unwrap(); + assert!(res2.is_error); + } +} diff --git a/src/openhuman/tokenjuice/types.rs b/src/openhuman/tokenjuice/types.rs index 05d40654e..4355c81a2 100644 --- a/src/openhuman/tokenjuice/types.rs +++ b/src/openhuman/tokenjuice/types.rs @@ -301,6 +301,265 @@ pub struct CompactResult { pub classification: ClassificationResult, } +// --------------------------------------------------------------------------- +// Content Router (TokenJuice 2.0) — content-kind detection + compressor dispatch +// --------------------------------------------------------------------------- + +/// The kind of content a blob holds, as decided by the detector. Drives which +/// [`crate::openhuman::tokenjuice::compressors::Compressor`] the router picks. +/// +/// Inspired by Headroom's content router: each kind has a specialised +/// compressor tuned to preserve the signal that kind carries (errors in logs, +/// changed hunks in diffs, signatures in code, …). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ContentKind { + /// JSON array/object payload → tabular SmartCrusher. + Json, + /// Source code → AST/heuristic signature keeper. + Code, + /// Build / test / lint log → keep failures, drop passing noise. + Log, + /// grep / ripgrep style `path:line:content` matches → relevance rank. + Search, + /// Unified git diff / patch → keep changed hunks, collapse context. + Diff, + /// HTML document → strip markup to readable text. + Html, + /// Anything else → ML text compressor (if enabled) or pass-through. + PlainText, +} + +impl ContentKind { + /// Stable lower-case label for logs / RPC / stats. + pub fn as_str(self) -> &'static str { + match self { + ContentKind::Json => "json", + ContentKind::Code => "code", + ContentKind::Log => "log", + ContentKind::Search => "search", + ContentKind::Diff => "diff", + ContentKind::Html => "html", + ContentKind::PlainText => "plain_text", + } + } +} + +/// A caller-supplied prior about a blob's content, so the detector doesn't have +/// to work from scratch. Any field may be `None`; the detector resolves what it +/// can and falls back to structural heuristics. An `explicit` kind is a hard +/// override and skips detection entirely. +#[derive(Debug, Clone, Default)] +pub struct ContentHint { + /// MIME type if known (`text/html`, `application/json`, …). + pub mime: Option, + /// File extension without the dot (`rs`, `ts`, `py`, `json`, `html`, `diff`). + pub extension: Option, + /// The agent-level tool that produced the content (`grep`, `run_tests`, …). + pub source_tool: Option, + /// A search/query string associated with the content, when known (used by + /// the search compressor to rank matches by query-term density). + pub query: Option, + /// Hard override — when set, detection returns this kind verbatim. + pub explicit: Option, +} + +impl ContentHint { + /// Convenience: a hint carrying only the producing tool name. + pub fn for_tool(tool_name: impl Into) -> Self { + Self { + source_tool: Some(tool_name.into()), + ..Default::default() + } + } +} + +/// Which compressor actually produced an output. Recorded in stats / logs so a +/// human (or the debug controller) can see what the router chose. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum CompressorKind { + /// JSON array→table crusher. + SmartCrusher, + /// AST/heuristic code-signature keeper. + Code, + /// Log keep-failures compressor (and the rule engine for command output). + Log, + /// Search relevance ranker. + Search, + /// Unified-diff context collapser. + Diff, + /// HTML→text extractor. + Html, + /// ML (Python/ModernBERT) plain-text compressor. + MlText, + /// Line-oriented head/tail fallback. + Generic, + /// No compressor fired — pass-through. + None, +} + +impl CompressorKind { + /// Stable lower-case label for stats / logs / RPC. + pub fn as_str(self) -> &'static str { + match self { + CompressorKind::SmartCrusher => "smartcrusher", + CompressorKind::Code => "code", + CompressorKind::Log => "log", + CompressorKind::Search => "search", + CompressorKind::Diff => "diff", + CompressorKind::Html => "html", + CompressorKind::MlText => "ml_text", + CompressorKind::Generic => "generic", + CompressorKind::None => "none", + } + } +} + +/// Input handed to a [`crate::openhuman::tokenjuice::compressors::Compressor`]. +/// Borrows the content to avoid copies on the hot path. +#[derive(Debug, Clone)] +pub struct CompressInput<'a> { + /// The raw content to compress. + pub content: &'a str, + /// The detected (or hinted) content kind. + pub kind: ContentKind, + /// The original caller hint (carries `query`, `source_tool`, argv-derived + /// command for the log/command path, …). + pub hint: &'a ContentHint, + /// Process exit code if this is command output — enables failure-preserving + /// behaviour in the log compressor. + pub exit_code: Option, + /// Derived shell command (joined argv) for the log/command rule path, if any. + pub command: Option, + /// Derived argv for the log/command rule path, if any. + pub argv: Option>, + /// Original byte length (== `content.len()`; cached for convenience). + pub original_bytes: usize, +} + +/// Output of a compressor. `text` is the compacted body **without** any CCR +/// retrieval footer — the router ([`crate::openhuman::tokenjuice::compress`]) +/// adds the marker after offloading the original. +#[derive(Debug, Clone)] +pub struct CompressOutput { + /// The compacted body. + pub text: String, + /// True when data was dropped (vs. a faithful reformat). Changes the footer + /// wording and whether the original is mandatory for fidelity. + pub lossy: bool, + /// Which compressor produced this. + pub kind: CompressorKind, + /// Optional named counts (e.g. error/warning tallies). + pub facts: Option>, +} + +impl CompressOutput { + /// A faithful reformat — every value preserved, only layout changed. + pub fn reformatted(text: String, kind: CompressorKind) -> Self { + Self { + text, + lossy: false, + kind, + facts: None, + } + } + + /// A lossy view — data was dropped; the original must be offloaded for recovery. + pub fn lossy(text: String, kind: CompressorKind) -> Self { + Self { + text, + lossy: true, + kind, + facts: None, + } + } +} + +/// Knobs for the router and compressors, built by the caller from the +/// `[tokenjuice]` config block. TokenJuice stays decoupled from the config +/// schema crate by taking this plain struct rather than `Config`. +#[derive(Debug, Clone)] +pub struct CompressOptions { + /// Master switch — when false, [`crate::openhuman::tokenjuice::compress_content`] + /// is a pass-through. + pub router_enabled: bool, + /// Whether to offload originals to CCR and emit retrieval markers. + pub ccr_enabled: bool, + /// Per-compressor toggles. + pub search_enabled: bool, + pub code_enabled: bool, + pub html_enabled: bool, + /// Whether the ML plain-text compressor may be used (further gated at + /// runtime by Python/runtime_python_server availability). + pub ml_text_enabled: bool, + /// Outputs below this many bytes are never compressed. + pub min_bytes_to_compress: usize, + /// CCR only fires (offload original + lossy compression) when the input is + /// estimated to be at least this many tokens. Below it, the result passes + /// through (lossless reformats may still apply without offload). Lets small + /// tool results skip the cache entirely. + pub ccr_min_tokens: usize, + /// Maximum inline character count for the generic/rule fallback path. + pub max_inline_chars: Option, +} + +impl Default for CompressOptions { + fn default() -> Self { + Self { + router_enabled: true, + ccr_enabled: true, + search_enabled: true, + code_enabled: true, + html_enabled: true, + ml_text_enabled: false, + min_bytes_to_compress: 2048, + ccr_min_tokens: 500, + max_inline_chars: None, + } + } +} + +/// The result of the universal [`crate::openhuman::tokenjuice::compress_content`] +/// entry point: the compacted text (with any CCR footer already appended), plus +/// metadata for callers/stats. +#[derive(Debug, Clone)] +pub struct CompressedOutput { + /// Final text to inline into context (includes the retrieval footer when lossy). + pub text: String, + /// The detected content kind. + pub content_kind: ContentKind, + /// Which compressor fired (`None` ⇒ pass-through). + pub compressor: CompressorKind, + /// Whether the output dropped data. + pub lossy: bool, + /// True if the router actually changed the content. + pub applied: bool, + /// CCR token for the offloaded original, if one was stored. + pub ccr_token: Option, + /// Original byte length. + pub original_bytes: usize, + /// Compacted byte length (of `text`). + pub compacted_bytes: usize, +} + +impl CompressedOutput { + /// Build a pass-through result that didn't change `content`. + pub fn passthrough(content: String, kind: ContentKind) -> Self { + let len = content.len(); + Self { + text: content, + content_kind: kind, + compressor: CompressorKind::None, + lossy: false, + applied: false, + ccr_token: None, + original_bytes: len, + compacted_bytes: len, + } + } +} + // --------------------------------------------------------------------------- // RuleFixture — used by integration tests // --------------------------------------------------------------------------- diff --git a/src/openhuman/tools/impl/system/retrieve_tool_output.rs b/src/openhuman/tools/impl/system/retrieve_tool_output.rs index decb6589d..4934083f9 100644 --- a/src/openhuman/tools/impl/system/retrieve_tool_output.rs +++ b/src/openhuman/tools/impl/system/retrieve_tool_output.rs @@ -69,7 +69,7 @@ impl Tool for RetrieveToolOutputTool { )); }; - match crate::openhuman::agent::harness::compaction::store::retrieve(hash) { + match crate::openhuman::tokenjuice::cache::retrieve(hash) { Some(original) => { log::debug!( "[compaction][ccr] retrieved hash={} bytes={}", @@ -89,7 +89,7 @@ impl Tool for RetrieveToolOutputTool { #[cfg(test)] mod tests { use super::*; - use crate::openhuman::agent::harness::compaction::store; + use crate::openhuman::tokenjuice::cache::store; #[tokio::test] async fn retrieves_offloaded_original() { diff --git a/src/openhuman/tools/ops.rs b/src/openhuman/tools/ops.rs index 54ad05e2a..7efa190fc 100644 --- a/src/openhuman/tools/ops.rs +++ b/src/openhuman/tools/ops.rs @@ -209,6 +209,10 @@ pub fn all_tools_with_runtime( // large result is compacted with a `retrieve_tool_output("")` // marker, this hands the original back from the CCR store on demand. Box::new(RetrieveToolOutputTool::new()), + // TokenJuice 2.0 content-router retrieval: fetches the original (full or + // by byte/line range) for a `⟦tj:⟧` marker from the CCR cache. + // Supersedes `retrieve_tool_output`; both are kept live during migration. + Box::new(crate::openhuman::tokenjuice::TokenjuiceRetrieveTool::new()), // Deterministic time-expression → timestamp resolver. `current_time` // only returns *now*, leaving the model to do epoch arithmetic by hand // (a real incident had an agent compute "24h ago" ~10 months off, then diff --git a/tests/json_rpc_e2e.rs b/tests/json_rpc_e2e.rs index b24756cf3..8e8a18ba2 100644 --- a/tests/json_rpc_e2e.rs +++ b/tests/json_rpc_e2e.rs @@ -981,6 +981,120 @@ fn ensure_test_rpc_auth() { }); } +#[tokio::test] +async fn json_rpc_tokenjuice_detect_and_cache_stats() { + let _env_lock = json_rpc_e2e_env_lock(); + let (rpc_addr, rpc_join) = serve_on_ephemeral(build_core_http_router(false)).await; + let rpc_base = format!("http://{rpc_addr}"); + + // detect: a JSON array of objects classifies as `json`. + let detect = post_json_rpc( + &rpc_base, + 1860_1, + "openhuman.tokenjuice_detect", + json!({ "content": r#"[{"a":1,"b":2},{"a":3,"b":4}]"# }), + ) + .await; + let detect_result = assert_no_jsonrpc_error(&detect, "tokenjuice_detect"); + assert_eq!( + detect_result.get("kind").and_then(Value::as_str), + Some("json") + ); + + // cache_stats: returns numeric occupancy fields. + let stats = post_json_rpc( + &rpc_base, + 1860_2, + "openhuman.tokenjuice_cache_stats", + json!({}), + ) + .await; + let stats_result = assert_no_jsonrpc_error(&stats, "tokenjuice_cache_stats"); + assert!(stats_result + .get("entries") + .and_then(Value::as_u64) + .is_some()); + assert!(stats_result.get("bytes").and_then(Value::as_u64).is_some()); + + rpc_join.abort(); +} + +#[tokio::test] +async fn json_rpc_tokenjuice_settings_and_savings() { + let _env_lock = json_rpc_e2e_env_lock(); + let (rpc_addr, rpc_join) = serve_on_ephemeral(build_core_http_router(false)).await; + let rpc_base = format!("http://{rpc_addr}"); + + // settings_get: returns the [tokenjuice] block with the configurable + // CCR token threshold (default 500) and the router master switch. + let get = post_json_rpc( + &rpc_base, + 1861_1, + "openhuman.tokenjuice_settings_get", + json!({}), + ) + .await; + let get_result = assert_no_jsonrpc_error(&get, "tokenjuice_settings_get"); + let settings = get_result + .get("settings") + .expect("settings_get returns a settings object"); + assert!(settings + .get("router_enabled") + .and_then(Value::as_bool) + .is_some()); + assert!(settings + .get("ccr_min_tokens") + .and_then(Value::as_u64) + .is_some()); + + // settings_update: flip the CCR token threshold and confirm it round-trips. + let updated = post_json_rpc( + &rpc_base, + 1861_2, + "openhuman.tokenjuice_settings_update", + json!({ "patch": { "ccr_min_tokens": 750 } }), + ) + .await; + let updated_result = assert_no_jsonrpc_error(&updated, "tokenjuice_settings_update"); + assert_eq!( + updated_result + .get("settings") + .and_then(|s| s.get("ccr_min_tokens")) + .and_then(Value::as_u64), + Some(750) + ); + + // savings_stats: returns the aggregate shape (total + attribution model + cache). + let savings = post_json_rpc( + &rpc_base, + 1861_3, + "openhuman.tokenjuice_savings_stats", + json!({}), + ) + .await; + let savings_result = assert_no_jsonrpc_error(&savings, "tokenjuice_savings_stats"); + assert!(savings_result.get("attributionModel").is_some()); + assert!(savings_result + .get("total") + .and_then(|t| t.get("tokensSaved")) + .and_then(Value::as_u64) + .is_some()); + assert!(savings_result.get("cache").is_some()); + + // savings_reset: zeroes the totals. + let reset = post_json_rpc( + &rpc_base, + 1861_4, + "openhuman.tokenjuice_savings_reset", + json!({}), + ) + .await; + let reset_result = assert_no_jsonrpc_error(&reset, "tokenjuice_savings_reset"); + assert_eq!(reset_result.get("ok").and_then(Value::as_bool), Some(true)); + + rpc_join.abort(); +} + #[tokio::test] async fn json_rpc_tool_registry_lists_and_gets_entries() { let _env_lock = json_rpc_e2e_env_lock(); diff --git a/tests/memory_threads_raw_coverage_e2e.rs b/tests/memory_threads_raw_coverage_e2e.rs index 7a44af300..bfa6cdbeb 100644 --- a/tests/memory_threads_raw_coverage_e2e.rs +++ b/tests/memory_threads_raw_coverage_e2e.rs @@ -4026,6 +4026,10 @@ async fn memory_sources_registry_rpc_and_schema_handlers_cover_crud_edges() { #[tokio::test] async fn memory_ops_public_handlers_cover_document_file_kv_graph_and_envelopes() { + Box::pin(memory_ops_public_handlers_cover_document_file_kv_graph_and_envelopes_body()).await; +} + +async fn memory_ops_public_handlers_cover_document_file_kv_graph_and_envelopes_body() { let _lock = env_lock(); let tmp = TempDir::new().expect("tempdir"); let _workspace = EnvVarGuard::set_to_path("OPENHUMAN_WORKSPACE", tmp.path());