diff --git a/app/test/e2e/specs/smoke.spec.ts b/app/test/e2e/specs/smoke.spec.ts index bc7f147f1..b9ac3abdf 100644 --- a/app/test/e2e/specs/smoke.spec.ts +++ b/app/test/e2e/specs/smoke.spec.ts @@ -14,6 +14,7 @@ import { waitForApp, waitForAppReady } from '../helpers/app-helpers'; import { hasAppChrome } from '../helpers/element-helpers'; import { resetApp } from '../helpers/reset-app'; import { waitForHomePage } from '../helpers/shared-flows'; +import { startMockServer, stopMockServer } from '../mock-server'; const USER_ID = 'e2e-smoke'; @@ -21,10 +22,15 @@ describe('Smoke', function () { this.timeout(120_000); before(async () => { + await startMockServer(); await waitForApp(); await resetApp(USER_ID); }); + after(async () => { + await stopMockServer(); + }); + it('has a live WebDriver session', async () => { const sessionId = browser.sessionId; expect(sessionId).toBeDefined(); diff --git a/src/bin/gmail_backfill_3d.rs b/src/bin/gmail_backfill_3d.rs index 4400388c2..64c4da508 100644 --- a/src/bin/gmail_backfill_3d.rs +++ b/src/bin/gmail_backfill_3d.rs @@ -41,13 +41,13 @@ use openhuman_core::openhuman::composio::providers::registry::{ get_provider, init_default_providers, }; use openhuman_core::openhuman::config::Config; -use openhuman_core::openhuman::memory_tree::content_store::read::{ - verify_chunk_file, verify_summary_file, VerifyResult, -}; -use openhuman_core::openhuman::memory_tree::jobs::drain_until_idle; -use openhuman_core::openhuman::memory_tree::store::{ +use openhuman_core::openhuman::memory::jobs::drain_until_idle; +use openhuman_core::openhuman::memory_store::chunks::store::{ get_chunk_content_pointers, list_chunks, list_summaries_with_content_path, ListChunksQuery, }; +use openhuman_core::openhuman::memory_store::content::read::{ + verify_chunk_file, verify_summary_file, VerifyResult, +}; #[derive(Parser, Debug)] #[command( diff --git a/src/bin/memory_tree_init_smoke.rs b/src/bin/memory_tree_init_smoke.rs index 0e916d56a..bddd11e9c 100644 --- a/src/bin/memory_tree_init_smoke.rs +++ b/src/bin/memory_tree_init_smoke.rs @@ -30,7 +30,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use openhuman_core::openhuman::config::Config; -use openhuman_core::openhuman::memory_tree::store::with_connection; +use openhuman_core::openhuman::memory_store::chunks::store::with_connection; fn main() -> ExitCode { env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")) diff --git a/src/bin/slack_backfill.rs b/src/bin/slack_backfill.rs index 1ec525be3..2292d516d 100644 --- a/src/bin/slack_backfill.rs +++ b/src/bin/slack_backfill.rs @@ -211,8 +211,8 @@ async fn main() -> Result<()> { if cli.seal_probe { use chrono::{Duration, Utc}; - use openhuman_core::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage}; - use openhuman_core::openhuman::memory_tree::ingest::ingest_chat; + use openhuman_core::openhuman::memory::ingest_pipeline::ingest_chat; + use openhuman_core::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage}; let connection_id = cli.connection_id.clone().ok_or_else(|| { anyhow::anyhow!( diff --git a/src/core/cli.rs b/src/core/cli.rs index dd2d53ae1..52bc5d564 100644 --- a/src/core/cli.rs +++ b/src/core/cli.rs @@ -68,7 +68,9 @@ pub fn run_from_cli_args(args: &[String]) -> Result<()> { } "text-input" => crate::openhuman::text_input::cli::run_text_input_command(&args[1..]), "tree-summarizer" => { - crate::openhuman::memory_tree::summarizer::cli::run_tree_summarizer_command(&args[1..]) + crate::openhuman::memory_tree::tree_runtime::cli::run_tree_summarizer_command( + &args[1..], + ) } "memory" => crate::core::memory_cli::run_memory_command(&args[1..]), "agent" => { diff --git a/src/core/jsonrpc.rs b/src/core/jsonrpc.rs index 6834bb1de..7f547bed9 100644 --- a/src/core/jsonrpc.rs +++ b/src/core/jsonrpc.rs @@ -1558,7 +1558,7 @@ fn register_domain_subscribers( ); } - crate::openhuman::memory_tree::jobs::start(config.clone()); + crate::openhuman::memory::jobs::start(config.clone()); // Restart requests go through a subscriber so every trigger path shares // the same respawn logic. diff --git a/src/openhuman/agent/harness/archivist.rs b/src/openhuman/agent/harness/archivist.rs index f453f1be3..3c53f4297 100644 --- a/src/openhuman/agent/harness/archivist.rs +++ b/src/openhuman/agent/harness/archivist.rs @@ -18,23 +18,18 @@ use crate::openhuman::agent::hooks::{PostTurnHook, TurnContext}; use crate::openhuman::config::Config; -use crate::openhuman::memory::store::events::{self, EventRecord, EventType}; -use crate::openhuman::memory::store::fts5::{self, EpisodicEntry}; -use crate::openhuman::memory::store::profile::{self, FacetType}; -use crate::openhuman::memory::store::segments::{ +use crate::openhuman::memory::chat::ChatProvider; +use crate::openhuman::memory::ingest_pipeline; +use crate::openhuman::memory::score::embed::{build_embedder_from_config, Embedder}; +use crate::openhuman::memory_store::events::{self, EventRecord, EventType}; +use crate::openhuman::memory_store::fts5::{self, EpisodicEntry}; +use crate::openhuman::memory_store::profile::{self, FacetType}; +use crate::openhuman::memory_store::segments::{ self, BoundaryConfig, BoundaryDecision, ConversationSegment, }; -use crate::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage}; -use crate::openhuman::memory_tree::chat::{ChatConsumer, ChatProvider}; -use crate::openhuman::memory_tree::ingest; -use crate::openhuman::memory_tree::score::embed::{build_embedder_from_config, Embedder}; -use crate::openhuman::memory_tree::tree_source::summariser::llm::{ - LlmSummariser, LlmSummariserConfig, -}; -use crate::openhuman::memory_tree::tree_source::summariser::{ - Summariser, SummaryContext, SummaryInput, -}; -use crate::openhuman::memory_tree::tree_source::types::TreeKind; +use crate::openhuman::memory_store::trees::types::TreeKind; +use crate::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage}; +use crate::openhuman::memory_tree::summarise::{summarise, SummaryContext, SummaryInput}; use async_trait::async_trait; use parking_lot::Mutex; use rusqlite::Connection; @@ -98,10 +93,7 @@ impl ArchivistHook { pub fn with_config(mut self, config: Config) -> Self { // Build the LLM chat provider for segment recap. let chat_provider: Option> = - match crate::openhuman::memory_tree::chat::build_chat_provider( - &config, - ChatConsumer::Summarise, - ) { + match crate::openhuman::memory::chat::build_chat_provider(&config) { Ok(p) => { tracing::debug!("[archivist] segment recap provider={} registered", p.name()); Some(p) @@ -207,6 +199,7 @@ impl ArchivistHook { timestamp: f64, user_message: &str, current_episodic_id: i64, + current_seq: Option, ) -> Option { let now = Self::now_timestamp(); @@ -241,6 +234,7 @@ impl ArchivistHook { conn, &segment.segment_id, current_episodic_id, + current_seq, timestamp, now, ) { @@ -269,6 +263,7 @@ impl ArchivistHook { session_id, "global", current_episodic_id, + current_seq, timestamp, now, ) { @@ -293,6 +288,7 @@ impl ArchivistHook { session_id, "global", current_episodic_id, + current_seq, timestamp, now, ) { @@ -318,8 +314,10 @@ impl ArchivistHook { session_id: &str, now: f64, ) { - // Gather the conversation text for this segment from episodic entries. - let entries = fts5::episodic_session_entries(conn, session_id).unwrap_or_default(); + // Gather the conversation text for this segment. Prefer the + // md-backed memory_archivist read when config is available; fall + // back to FTS5 in test paths or when config isn't wired. + let entries = self.read_session_entries(conn, session_id); // Filter entries that fall within the segment's time window. // Use <= for end_timestamp (entries at the boundary are part of this @@ -583,6 +581,56 @@ impl PostTurnHook for ArchivistHook { tracing::debug!("[archivist] episodic rows written: session={session_id}"); + // Dual-write into memory_archivist::store (md-backed) so we can + // validate the FTS5 → md migration before flipping the read side. + // Best-effort: a write failure here must not break the turn. The + // user turn's assigned seq is captured into `current_seq` so the + // segment ops can store it alongside the FTS5 episodic id. + let mut current_seq: Option = None; + if let Some(cfg) = self.config.as_ref() { + let ts_ms = (timestamp * 1000.0) as i64; + let user_turn = crate::openhuman::memory_archivist::ArchivedTurn { + session_id: session_id.to_string(), + seq: 0, // assigned by record_turn + timestamp_ms: ts_ms, + role: "user".to_string(), + content: ctx.user_message.clone(), + lesson: None, + tool_calls_json: None, + cost_microdollars: 0, + }; + match crate::openhuman::memory_archivist::store::record_turn(cfg, user_turn) { + Ok(stored) => current_seq = Some(stored.seq), + Err(e) => { + tracing::warn!("[archivist] memory_archivist user dual-write failed: {e}"); + } + } + // Assistant turn carries the tool_calls_json + lesson the FTS5 + // insert just wrote. Re-derive locally so we don't depend on + // FTS5 having returned. + let assistant_lesson = extract_lesson_from_tools(&ctx.tool_calls); + let assistant_tool_calls = if ctx.tool_calls.is_empty() { + None + } else { + Some(serde_json::to_string(&ctx.tool_calls).unwrap_or_default()) + }; + let assistant_turn = crate::openhuman::memory_archivist::ArchivedTurn { + session_id: session_id.to_string(), + seq: 0, + timestamp_ms: ts_ms + 1, + role: "assistant".to_string(), + content: ctx.assistant_response.clone(), + lesson: assistant_lesson, + tool_calls_json: assistant_tool_calls, + cost_microdollars: 0, + }; + if let Err(e) = + crate::openhuman::memory_archivist::store::record_turn(cfg, assistant_turn) + { + tracing::warn!("[archivist] memory_archivist assistant dual-write failed: {e}"); + } + } + // Manage conversation segmentation (sync boundary detection + SQLite // operations). Returns the just-closed segment when a boundary fired. let closed_segment = self.manage_segment_sync( @@ -591,6 +639,7 @@ impl PostTurnHook for ArchivistHook { timestamp, &ctx.user_message, current_episodic_id, + current_seq, ); // Run async recap + embed + segment-tree ingest on the closed segment @@ -607,6 +656,47 @@ impl PostTurnHook for ArchivistHook { } impl ArchivistHook { + /// Read every entry recorded for `session_id`, preferring the + /// md-backed `memory_archivist::store` when `self.config` is set and + /// falling back to the legacy FTS5 episodic table otherwise. + /// + /// Returns `EpisodicEntry` so the existing call sites (segment + /// gathering, recap rendering, tree push) keep their shape unchanged + /// during the FTS5 retirement migration. + fn read_session_entries( + &self, + conn: &Arc>, + session_id: &str, + ) -> Vec { + if let Some(cfg) = self.config.as_ref() { + match crate::openhuman::memory_archivist::store::session_entries(cfg, session_id) { + Ok(turns) => { + return turns + .into_iter() + .map(|t| EpisodicEntry { + id: None, + session_id: t.session_id, + // ArchivedTurn stores epoch-ms; EpisodicEntry + // takes epoch-seconds as f64. + timestamp: (t.timestamp_ms as f64) / 1000.0, + role: t.role, + content: t.content, + lesson: t.lesson, + tool_calls_json: t.tool_calls_json, + cost_microdollars: t.cost_microdollars, + }) + .collect(); + } + Err(e) => { + tracing::warn!( + "[archivist] memory_archivist read failed (falling back to FTS5): {e}" + ); + } + } + } + fts5::episodic_session_entries(conn, session_id).unwrap_or_default() + } + /// Shared summarize helper — the **single LLM summarizer** used by both /// the finalize path (`on_segment_closed`) and the rolling-recap path /// (`rolling_segment_recap`). @@ -650,7 +740,7 @@ impl ArchivistHook { .iter() .filter(|e| !e.content.trim().is_empty()) .map(|e| { - use crate::openhuman::memory_tree::types::approx_token_count; + use crate::openhuman::memory_store::chunks::types::approx_token_count; let content = e.content.clone(); let token_count = approx_token_count(&content); let ts = chrono::DateTime::from_timestamp(e.timestamp as i64, 0) @@ -678,53 +768,60 @@ impl ArchivistHook { let first = entries.first().map(|e| e.content.as_str()).unwrap_or(""); let last = entries.last().map(|e| e.content.as_str()).unwrap_or(first); - if let Some(ref provider) = self.chat_provider { - let cfg = LlmSummariserConfig { - model: provider.name().to_string(), - structured_facet_extraction: false, - output_language: self - .config - .as_ref() - .and_then(|cfg| cfg.output_language.clone()), - }; - let summariser = LlmSummariser::new(cfg, Arc::clone(provider)); - tracing::debug!( - "[archivist] summarize_entries: LLM recap segment={segment_id} \ - provider={} entries={}", - provider.name(), - entries.len() - ); - match summariser.summarise(&corpus_inputs, &summary_ctx).await { - Ok(output) if !output.content.is_empty() => { - tracing::debug!( - "[archivist] summarize_entries: LLM recap ok segment={segment_id} \ - chars={}", - output.content.len() - ); - (output.content, true) - } - Ok(_) => { - tracing::debug!( - "[archivist] summarize_entries: LLM returned empty — \ - heuristic fallback segment={segment_id}" - ); - (segments::fallback_summary(first, last, turn_count), false) - } - Err(e) => { - tracing::warn!( - "[archivist] summarize_entries: LLM recap failed (non-fatal) \ - segment={segment_id}: {e} — heuristic fallback" - ); - (segments::fallback_summary(first, last, turn_count), false) + if self.chat_provider.is_some() { + if let Some(ref config) = self.config { + tracing::debug!( + "[archivist] summarize_entries: LLM recap segment={segment_id} entries={}", + entries.len() + ); + #[cfg(test)] + let summary_result = if let Some(provider) = self.chat_provider.as_ref() { + crate::openhuman::memory::chat::test_override::with_provider( + Arc::clone(provider), + summarise(config, &corpus_inputs, &summary_ctx), + ) + .await + } else { + summarise(config, &corpus_inputs, &summary_ctx).await + }; + #[cfg(not(test))] + let summary_result = summarise(config, &corpus_inputs, &summary_ctx).await; + + match summary_result { + Ok(output) if !output.content.is_empty() => { + tracing::debug!( + "[archivist] summarize_entries: LLM recap ok segment={segment_id} \ + chars={}", + output.content.len() + ); + return (output.content, true); + } + Ok(_) => { + tracing::debug!( + "[archivist] summarize_entries: LLM returned empty — \ + heuristic fallback segment={segment_id}" + ); + } + Err(e) => { + tracing::warn!( + "[archivist] summarize_entries: LLM recap failed (non-fatal) \ + segment={segment_id}: {e} — heuristic fallback" + ); + } } + } else { + tracing::debug!( + "[archivist] summarize_entries: no config — \ + heuristic fallback segment={segment_id}" + ); } } else { tracing::debug!( "[archivist] summarize_entries: no chat provider — \ heuristic fallback segment={segment_id}" ); - (segments::fallback_summary(first, last, turn_count), false) } + (segments::fallback_summary(first, last, turn_count), false) } /// Produce a rolling recap of the **currently-open** segment for @@ -782,7 +879,7 @@ impl ArchivistHook { }; // Gather the episodic entries for this session so far. - let all_entries = fts5::episodic_session_entries(conn, session_id).unwrap_or_default(); + let all_entries = self.read_session_entries(conn, session_id); // Keep only entries within the open segment's time window (start → // now, inclusive). An open segment has `end_timestamp = None`. @@ -867,7 +964,7 @@ impl ArchivistHook { async fn pipe_segment_to_tree( &self, config: &Config, - segment: &crate::openhuman::memory::store::segments::ConversationSegment, + segment: &crate::openhuman::memory_store::segments::ConversationSegment, session_id: &str, entries: &[&fts5::EpisodicEntry], ) { @@ -945,7 +1042,7 @@ impl ArchivistHook { segment={segment_id} ep_span={start_ep}-{end_ep} provenance={provenance}" ); - match ingest::ingest_chat(config, source_id, owner, tags, batch).await { + match ingest_pipeline::ingest_chat(config, source_id, owner, tags, batch).await { Ok(result) => { tracing::debug!( "[archivist] tree ingest ok: source_id={source_id} \ @@ -1118,7 +1215,7 @@ impl ArchivistHook { conn: Some(conn), enabled: true, boundary_config: BoundaryConfig::default(), - config: None, + config: Some(Config::default()), chat_provider: Some(chat_provider), embedder: Some(embedder), } diff --git a/src/openhuman/agent/harness/archivist_tests.rs b/src/openhuman/agent/harness/archivist_tests.rs index 9ae3349a1..1f56364a6 100644 --- a/src/openhuman/agent/harness/archivist_tests.rs +++ b/src/openhuman/agent/harness/archivist_tests.rs @@ -1,7 +1,7 @@ use super::*; use crate::openhuman::agent::hooks::{ToolCallRecord, TurnContext}; -use crate::openhuman::memory::store::{events as ev, fts5, segments as seg}; -use crate::openhuman::memory_tree::chat::ChatPrompt; +use crate::openhuman::memory::chat::ChatPrompt; +use crate::openhuman::memory_store::{events as ev, fts5, segments as seg}; fn setup_conn() -> Arc> { let conn = Connection::open_in_memory().unwrap(); @@ -343,7 +343,7 @@ async fn phase0_episodic_rows_and_segment_without_learning_enabled() { struct StubChatProvider; #[async_trait::async_trait] -impl crate::openhuman::memory_tree::chat::ChatProvider for StubChatProvider { +impl crate::openhuman::memory::chat::ChatProvider for StubChatProvider { fn name(&self) -> &str { "stub:test" } @@ -361,7 +361,7 @@ impl crate::openhuman::memory_tree::chat::ChatProvider for StubChatProvider { struct StubEmbedder; #[async_trait::async_trait] -impl crate::openhuman::memory_tree::score::embed::Embedder for StubEmbedder { +impl crate::openhuman::memory::score::embed::Embedder for StubEmbedder { fn name(&self) -> &'static str { "stub-embedder-v1" } @@ -546,7 +546,7 @@ async fn phase1_flush_open_segment_finalizes_trailing_segment() { // g) flush_open_segment also triggers tree ingest. use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::store::{count_chunks, list_chunks, ListChunksQuery}; +use crate::openhuman::memory_store::chunks::store::{count_chunks, list_chunks, ListChunksQuery}; use tempfile::TempDir; /// Build a Config that points at a temp workspace, suitable for tree-ingest tests. @@ -736,7 +736,7 @@ async fn phase2_provenance_stamped_on_leaf_and_source_id_is_constant() { .iter() .find(|s| { s.session_id == session - && s.status != crate::openhuman::memory::store::segments::SegmentStatus::Open + && s.status != crate::openhuman::memory_store::segments::SegmentStatus::Open }) .expect("Expected a closed segment after flush"); @@ -836,7 +836,9 @@ async fn phase2_ingested_content_is_raw_prose_not_recap() { } // The raw prose text MUST appear in at least one chunk. - let has_user_prose = chunks.iter().any(|c| c.content.contains("lifetimes")); + let has_user_prose = chunks + .iter() + .any(|c| c.content.to_ascii_lowercase().contains("lifetimes")); assert!( has_user_prose, "Expected at least one chunk body to contain raw prose from the turn \ diff --git a/src/openhuman/agent/harness/payload_summarizer.rs b/src/openhuman/agent/harness/payload_summarizer.rs index 21870059c..065953f8c 100644 --- a/src/openhuman/agent/harness/payload_summarizer.rs +++ b/src/openhuman/agent/harness/payload_summarizer.rs @@ -309,7 +309,7 @@ impl PayloadSummarizer for SubagentPayloadSummarizer { } /// Rough token estimate: ~4 characters per token. Mirrors -/// [`crate::openhuman::memory_tree::summarizer::types::estimate_tokens`] but +/// [`crate::openhuman::memory_tree::tree_runtime::types::estimate_tokens`] but /// returns `usize` (not `u32`) and lives here to avoid a cross-module /// dependency from the agent harness on the tree summarizer. fn estimate_tokens(text: &str) -> usize { diff --git a/src/openhuman/agent/harness/session/turn.rs b/src/openhuman/agent/harness/session/turn.rs index 89d5a8a93..286d1e415 100644 --- a/src/openhuman/agent/harness/session/turn.rs +++ b/src/openhuman/agent/harness/session/turn.rs @@ -2150,7 +2150,7 @@ impl Agent { } /// Wrapper around -/// [`crate::openhuman::memory_tree::summarizer::store::collect_root_summaries_with_caps`] +/// [`crate::openhuman::memory_tree::tree_runtime::store::collect_root_summaries_with_caps`] /// that takes user-resolved per-namespace and total caps. The actual /// limits are derived from the active /// [`crate::openhuman::config::schema::agent::MemoryContextWindow`] @@ -2160,7 +2160,7 @@ fn collect_tree_root_summaries( per_namespace_cap: usize, total_cap: usize, ) -> Vec<(String, String)> { - crate::openhuman::memory_tree::summarizer::store::collect_root_summaries_with_caps( + crate::openhuman::memory_tree::tree_runtime::store::collect_root_summaries_with_caps( workspace_dir, per_namespace_cap, total_cap, diff --git a/src/openhuman/agent/harness/subagent_runner/handoff.rs b/src/openhuman/agent/harness/subagent_runner/handoff.rs index b423f0dae..da6b7d117 100644 --- a/src/openhuman/agent/harness/subagent_runner/handoff.rs +++ b/src/openhuman/agent/harness/subagent_runner/handoff.rs @@ -29,7 +29,7 @@ use std::sync::Mutex as StdMutex; /// cache instead of being pushed into history raw. Token count is /// estimated at ~4 chars/token (mirrors /// `crate::openhuman::agent::harness::payload_summarizer` and -/// `crate::openhuman::memory_tree::summarizer::types::estimate_tokens`). +/// `crate::openhuman::memory_tree::tree_runtime::types::estimate_tokens`). /// /// Set at `50_000` so the clean Gmail / Notion envelopes emitted by provider /// post-processing fit through unchanged for normal workloads — only diff --git a/src/openhuman/agent/tree_loader.rs b/src/openhuman/agent/tree_loader.rs index 2ec5a87cc..23ab52847 100644 --- a/src/openhuman/agent/tree_loader.rs +++ b/src/openhuman/agent/tree_loader.rs @@ -21,7 +21,7 @@ //! concatenate without branching. use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::retrieval::query_global; +use crate::openhuman::memory::retrieval::query_global; /// Default lookback window for the eager digest. Mirrors the language in /// the orchestrator prompt ("7-day digest pre-loaded into session context"). diff --git a/src/openhuman/approval/store.rs b/src/openhuman/approval/store.rs index 031369f02..a8bb0ddc6 100644 --- a/src/openhuman/approval/store.rs +++ b/src/openhuman/approval/store.rs @@ -25,7 +25,7 @@ use chrono::{DateTime, Utc}; use rusqlite::{params, types::Type, Connection}; use crate::openhuman::config::Config; -use crate::openhuman::memory::safety::sanitize_text; +use crate::openhuman::memory_store::safety::sanitize_text; use super::types::{ApprovalAuditEntry, ApprovalDecision, ExecutionOutcome, PendingApproval}; diff --git a/src/openhuman/channels/runtime/startup.rs b/src/openhuman/channels/runtime/startup.rs index 9138f9a5b..919cf1923 100644 --- a/src/openhuman/channels/runtime/startup.rs +++ b/src/openhuman/channels/runtime/startup.rs @@ -587,7 +587,7 @@ pub async fn start_channels(config: Config) -> Result<()> { }; // Register the tree summarizer event subscriber for observability logging. let _tree_summarizer_handle = bus.subscribe(Arc::new( - crate::openhuman::memory_tree::summarizer::bus::TreeSummarizerEventSubscriber::new(), + crate::openhuman::memory_tree::tree_runtime::bus::TreeSummarizerEventSubscriber::new(), )); let max_in_flight_messages = compute_max_in_flight_messages(channels.len()); diff --git a/src/openhuman/composio/ops_test.rs b/src/openhuman/composio/ops_test.rs index 661be857e..7337255fc 100644 --- a/src/openhuman/composio/ops_test.rs +++ b/src/openhuman/composio/ops_test.rs @@ -576,8 +576,8 @@ async fn composio_execute_via_mock_propagates_backend_error() { #[tokio::test] async fn composio_sync_gmail_via_mock_archives_raw_email_and_updates_outcome() { use crate::openhuman::config::TEST_ENV_LOCK; - use crate::openhuman::memory_tree::content_store::raw::{raw_rel_path, RawKind}; - use crate::openhuman::memory_tree::rpc::{list_chunks_rpc, ListChunksRequest}; + use crate::openhuman::memory::tree_rpc::{list_chunks_rpc, ListChunksRequest}; + use crate::openhuman::memory_store::content::raw::{raw_rel_path, RawKind}; let _cache_guard = CACHE_TEST_GUARD.lock().unwrap_or_else(|e| e.into_inner()); let _env_guard = TEST_ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); diff --git a/src/openhuman/composio/providers/gmail/ingest.rs b/src/openhuman/composio/providers/gmail/ingest.rs index 7c2ba94e2..b7949c4fc 100644 --- a/src/openhuman/composio/providers/gmail/ingest.rs +++ b/src/openhuman/composio/providers/gmail/ingest.rs @@ -21,14 +21,14 @@ use anyhow::Result; use serde_json::Value; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::canonicalize::email::{EmailMessage, EmailThread}; -use crate::openhuman::memory_tree::canonicalize::email_clean::{extract_email, parse_message_date}; -use crate::openhuman::memory_tree::content_store::raw::{ +use crate::openhuman::memory::ingest_pipeline::{ingest_email, IngestResult}; +use crate::openhuman::memory::util::redact::redact; +use crate::openhuman::memory_store::chunks::store::{set_chunk_raw_refs, RawRef}; +use crate::openhuman::memory_store::content::raw::{ self as raw_store, raw_rel_path, slug_account_email, RawItem, RawKind, }; -use crate::openhuman::memory_tree::ingest::{ingest_email, IngestResult}; -use crate::openhuman::memory_tree::store::{set_chunk_raw_refs, RawRef}; -use crate::openhuman::memory_tree::util::redact::redact; +use crate::openhuman::memory_sync::canonicalize::email::{EmailMessage, EmailThread}; +use crate::openhuman::memory_sync::canonicalize::email_clean::{extract_email, parse_message_date}; /// Provider name embedded in the canonical email-thread header. Matches /// the value `memory::tree::retrieval::source::PLATFORM_KINDS` expects. diff --git a/src/openhuman/composio/providers/profile.rs b/src/openhuman/composio/providers/profile.rs index 90c6e3cab..e3992150a 100644 --- a/src/openhuman/composio/providers/profile.rs +++ b/src/openhuman/composio/providers/profile.rs @@ -21,7 +21,7 @@ use super::ProviderUserProfile; use crate::openhuman::learning::candidate::{ self as learning_candidate, CueFamily, EvidenceRef, FacetClass, LearningCandidate, }; -use crate::openhuman::memory::store::profile::{self, FacetType}; +use crate::openhuman::memory_store::profile::{self, FacetType}; use rusqlite::params; use serde_json::Value; use std::collections::BTreeMap; @@ -32,7 +32,7 @@ use std::collections::BTreeMap; /// Shape of an identifier persisted against a connection. Mirrors the /// matching dimensions of the memory tree's -/// `crate::openhuman::memory_tree::score::extract::EntityKind` so the +/// `crate::openhuman::memory::score::extract::EntityKind` so the /// self-check is a direct `(toolkit, kind, value)` lookup. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum IdentityKind { @@ -534,7 +534,7 @@ fn now_secs() -> f64 { #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory::store::profile::{profile_load_all, PROFILE_INIT_SQL}; + use crate::openhuman::memory_store::profile::{profile_load_all, PROFILE_INIT_SQL}; use parking_lot::Mutex; use rusqlite::Connection; use serde_json::json; diff --git a/src/openhuman/composio/providers/slack/ingest.rs b/src/openhuman/composio/providers/slack/ingest.rs index 6dc8d0d36..1efc8de1d 100644 --- a/src/openhuman/composio/providers/slack/ingest.rs +++ b/src/openhuman/composio/providers/slack/ingest.rs @@ -2,7 +2,7 @@ //! //! Owns the conversion from a page of [`SlackMessage`]s (post-processed //! and enriched by [`super::sync`]) into per-channel [`ChatBatch`]es and -//! drives [`memory_tree::ingest::ingest_chat`] per message. +//! drives [`memory::ingest_pipeline::ingest_chat`] per message. //! //! ## Source-id scope //! @@ -30,16 +30,16 @@ use anyhow::Result; use super::types::SlackMessage; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage}; -use crate::openhuman::memory_tree::content_store::raw::{ +use crate::openhuman::memory::ingest_pipeline::ingest_chat; +use crate::openhuman::memory::util::redact::redact; +use crate::openhuman::memory_store::chunks::store::{set_chunk_raw_refs, RawRef}; +use crate::openhuman::memory_store::content::raw::{ self as raw_store, raw_rel_path, RawItem, RawKind, }; -use crate::openhuman::memory_tree::ingest::ingest_chat; -use crate::openhuman::memory_tree::store::{set_chunk_raw_refs, RawRef}; -use crate::openhuman::memory_tree::util::redact::redact; +use crate::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage}; /// Platform identifier embedded in the canonical chat transcript header. -/// Matches the value `memory_tree::retrieval::source::PLATFORM_KINDS` expects. +/// Matches the value `memory::retrieval::source::PLATFORM_KINDS` expects. pub const SLACK_PLATFORM: &str = "slack"; /// Tags attached to every Slack-ingested chunk. Stable list — retrieval diff --git a/src/openhuman/config/ops.rs b/src/openhuman/config/ops.rs index ef536097e..14f8b0c21 100644 --- a/src/openhuman/config/ops.rs +++ b/src/openhuman/config/ops.rs @@ -588,7 +588,7 @@ pub async fn apply_model_settings( // so a UI embedder switch recovers prior memory under the new // signature. Coverage-gated + non-fatal: if the active signature did // not actually change, this enqueues nothing. - crate::openhuman::memory_tree::jobs::ensure_reembed_backfill(config); + crate::openhuman::memory::jobs::ensure_reembed_backfill(config); let snapshot = snapshot_config_json(config)?; Ok(RpcOutcome::new( snapshot, @@ -638,7 +638,7 @@ pub async fn apply_memory_settings( // dark. Idempotent + non-fatal (covered space enqueues nothing; errors // are logged, never fail the settings save). §7's migration is // one-shot so it does not cover a later switch — this does. - crate::openhuman::memory_tree::jobs::ensure_reembed_backfill(config); + crate::openhuman::memory::jobs::ensure_reembed_backfill(config); let snapshot = snapshot_config_json(config)?; Ok(RpcOutcome::new( snapshot, diff --git a/src/openhuman/context/segment_recap_summarizer_tests.rs b/src/openhuman/context/segment_recap_summarizer_tests.rs index 476f8c1f6..dd7302cdf 100644 --- a/src/openhuman/context/segment_recap_summarizer_tests.rs +++ b/src/openhuman/context/segment_recap_summarizer_tests.rs @@ -15,8 +15,8 @@ use crate::openhuman::agent::harness::archivist::ArchivistHook; use crate::openhuman::agent::hooks::{PostTurnHook as _, TurnContext}; use crate::openhuman::context::summarizer::{Summarizer, SummaryStats}; use crate::openhuman::inference::provider::{ChatMessage, ConversationMessage}; -use crate::openhuman::memory::store::{fts5, segments as seg}; -use crate::openhuman::memory_tree::chat::ChatPrompt; +use crate::openhuman::memory::chat::ChatPrompt; +use crate::openhuman::memory_store::{fts5, segments as seg}; use anyhow::Result; use async_trait::async_trait; use parking_lot::Mutex; @@ -29,9 +29,9 @@ fn setup_conn() -> Arc> { let conn = Connection::open_in_memory().unwrap(); conn.execute_batch(fts5::EPISODIC_INIT_SQL).unwrap(); conn.execute_batch(seg::SEGMENTS_INIT_SQL).unwrap(); - conn.execute_batch(crate::openhuman::memory::store::events::EVENTS_INIT_SQL) + conn.execute_batch(crate::openhuman::memory_store::events::EVENTS_INIT_SQL) .unwrap(); - conn.execute_batch(crate::openhuman::memory::store::profile::PROFILE_INIT_SQL) + conn.execute_batch(crate::openhuman::memory_store::profile::PROFILE_INIT_SQL) .unwrap(); Arc::new(Mutex::new(conn)) } @@ -40,7 +40,7 @@ fn setup_conn() -> Arc> { struct StubChatProvider; #[async_trait] -impl crate::openhuman::memory_tree::chat::ChatProvider for StubChatProvider { +impl crate::openhuman::memory::chat::ChatProvider for StubChatProvider { fn name(&self) -> &str { "stub:test" } @@ -56,7 +56,7 @@ impl crate::openhuman::memory_tree::chat::ChatProvider for StubChatProvider { struct FailingChatProvider; #[async_trait] -impl crate::openhuman::memory_tree::chat::ChatProvider for FailingChatProvider { +impl crate::openhuman::memory::chat::ChatProvider for FailingChatProvider { fn name(&self) -> &str { "stub:failing" } @@ -72,7 +72,7 @@ impl crate::openhuman::memory_tree::chat::ChatProvider for FailingChatProvider { struct StubEmbedder; #[async_trait] -impl crate::openhuman::memory_tree::score::embed::Embedder for StubEmbedder { +impl crate::openhuman::memory::score::embed::Embedder for StubEmbedder { fn name(&self) -> &'static str { "stub-embedder-v1" } @@ -439,13 +439,13 @@ async fn failing_provider_yields_inert_clipped_recap_used_as_compaction() { .await .unwrap(); - // Provider present but failing → LlmSummariser inert fallback → real - // clipped content (not the bookend stub) → Some, treated as usable. + // Provider present but failing → summarize_entries returns a non-LLM + // fallback, which rolling_segment_recap must treat as unavailable for + // live compaction. let recap = hook.rolling_segment_recap(session).await; assert!( - recap.is_some(), - "Inert clipped-content recap (real text) is acceptable compaction \ - text — must be Some, not None" + recap.is_none(), + "Non-LLM fallback recap text must not be used as live compaction input" ); let inner = RecordingSummarizer::new(); @@ -464,9 +464,9 @@ async fn failing_provider_yields_inert_clipped_recap_used_as_compaction() { assert_eq!( inner.call_count(), - 0, - "Inner summarizer must NOT run when an inert clipped-content recap \ - is available (real content, better than no compaction)" + 1, + "Inner summarizer must run when rolling recap is unavailable after \ + provider failure" ); } diff --git a/src/openhuman/doctor/core.rs b/src/openhuman/doctor/core.rs index 8ed86ef35..cf05a9be8 100644 --- a/src/openhuman/doctor/core.rs +++ b/src/openhuman/doctor/core.rs @@ -817,7 +817,7 @@ fn check_memory_tree_db(config: &Config, items: &mut Vec) { } // ── Probe connection ───────────────────────────────────────────── - match crate::openhuman::memory_tree::store::with_connection(config, |conn| { + match crate::openhuman::memory_store::chunks::store::with_connection(config, |conn| { let n: i64 = conn.query_row("SELECT COUNT(*) FROM mem_tree_chunks", [], |r| r.get(0))?; Ok(n) }) { diff --git a/src/openhuman/doctor/core_tests.rs b/src/openhuman/doctor/core_tests.rs index 08faacd6e..0027aa313 100644 --- a/src/openhuman/doctor/core_tests.rs +++ b/src/openhuman/doctor/core_tests.rs @@ -89,7 +89,7 @@ fn check_memory_tree_db_ok_when_accessible() { let cfg = test_config_in(&tmp); // Trigger DB creation. - crate::openhuman::memory_tree::store::with_connection(&cfg, |_conn| Ok(())) + crate::openhuman::memory_store::chunks::store::with_connection(&cfg, |_conn| Ok(())) .expect("DB init must succeed"); let mut items = vec![]; diff --git a/src/openhuman/embeddings/mod.rs b/src/openhuman/embeddings/mod.rs index 90fbde0be..a31929347 100644 --- a/src/openhuman/embeddings/mod.rs +++ b/src/openhuman/embeddings/mod.rs @@ -18,8 +18,13 @@ pub mod ollama; pub mod openai; mod provider_trait; pub mod rate_limit; -pub mod store; +// VectorStore has moved to memory_store::vectors; re-exported for callers. +pub use crate::openhuman::memory_store::vectors::store; + +pub use crate::openhuman::memory_store::vectors::{ + bytes_to_vec, cosine_similarity, vec_to_bytes, SearchResult, VectorStore, +}; pub use cloud::{ OpenHumanCloudEmbedding, DEFAULT_CLOUD_EMBEDDING_DIMENSIONS, DEFAULT_CLOUD_EMBEDDING_MODEL, }; @@ -30,7 +35,6 @@ pub use noop::NoopEmbedding; pub use ollama::{OllamaEmbedding, DEFAULT_OLLAMA_DIMENSIONS, DEFAULT_OLLAMA_MODEL}; pub use openai::OpenAiEmbedding; pub use provider_trait::{format_embedding_signature, EmbeddingProvider}; -pub use store::{bytes_to_vec, cosine_similarity, vec_to_bytes, SearchResult, VectorStore}; #[cfg(test)] mod tests { diff --git a/src/openhuman/inference/local/model_requirements.rs b/src/openhuman/inference/local/model_requirements.rs index a23eb482e..d8345ca75 100644 --- a/src/openhuman/inference/local/model_requirements.rs +++ b/src/openhuman/inference/local/model_requirements.rs @@ -2,7 +2,7 @@ //! //! The memory tree's embedder (`bge-m3`) is requested with //! `num_ctx = 8192` (see -//! [`crate::openhuman::memory_tree::score::embed::ollama::EMBED_NUM_CTX`]) +//! [`crate::openhuman::memory::score::embed::ollama::EMBED_NUM_CTX`]) //! and the summariser hard-caps its output to fit that 8192-token embed //! ceiling. A local model whose native context window is below this floor //! silently truncates chunks/summaries and corrupts recall, so we refuse @@ -21,7 +21,7 @@ use serde::Serialize; /// time. Changing the embedder's context request automatically moves the /// acceptance floor with it. pub const MIN_CONTEXT_TOKENS: u64 = - crate::openhuman::memory_tree::score::embed::ollama::EMBED_NUM_CTX as u64; + crate::openhuman::memory::score::embed::ollama::EMBED_NUM_CTX as u64; /// Verdict for a single model's context window against /// [`MIN_CONTEXT_TOKENS`]. Serialized into the diagnostics payload so the @@ -79,7 +79,7 @@ mod tests { // requests; this guards against the two drifting apart. assert_eq!( MIN_CONTEXT_TOKENS, - crate::openhuman::memory_tree::score::embed::ollama::EMBED_NUM_CTX as u64 + crate::openhuman::memory::score::embed::ollama::EMBED_NUM_CTX as u64 ); assert_eq!(MIN_CONTEXT_TOKENS, 8_192); } diff --git a/src/openhuman/learning/cache.rs b/src/openhuman/learning/cache.rs index 416534aae..2321fad11 100644 --- a/src/openhuman/learning/cache.rs +++ b/src/openhuman/learning/cache.rs @@ -9,7 +9,7 @@ use rusqlite::Connection; use std::sync::Arc; use crate::openhuman::learning::candidate::FacetClass; -use crate::openhuman::memory::store::profile::{self, ProfileFacet, UserState}; +use crate::openhuman::memory_store::profile::{self, ProfileFacet, UserState}; /// Thin wrapper around the `user_profile` table. /// @@ -115,7 +115,7 @@ pub fn class_prefix(class: FacetClass) -> &'static str { // ── Facet state enum re-export (convenience for callers of this module) ─────── -pub use crate::openhuman::memory::store::profile::{ +pub use crate::openhuman::memory_store::profile::{ FacetState as CacheFacetState, UserState as CacheUserState, }; diff --git a/src/openhuman/learning/cache_tests.rs b/src/openhuman/learning/cache_tests.rs index 618fa7d75..5bf5f6bad 100644 --- a/src/openhuman/learning/cache_tests.rs +++ b/src/openhuman/learning/cache_tests.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use super::*; use crate::openhuman::learning::candidate::{EvidenceRef, FacetClass}; -use crate::openhuman::memory::store::profile::{ +use crate::openhuman::memory_store::profile::{ FacetState, FacetType, ProfileFacet, UserState, PROFILE_INIT_SQL, }; diff --git a/src/openhuman/learning/linkedin_enrichment.rs b/src/openhuman/learning/linkedin_enrichment.rs index 3d8dd6c06..150412403 100644 --- a/src/openhuman/learning/linkedin_enrichment.rs +++ b/src/openhuman/learning/linkedin_enrichment.rs @@ -680,15 +680,15 @@ pub async fn scrape_linkedin_profile( } /// Build a local memory client for profile persistence. -fn build_memory_client() -> anyhow::Result { - crate::openhuman::memory::store::MemoryClient::new_local() +fn build_memory_client() -> anyhow::Result { + crate::openhuman::memory_store::MemoryClient::new_local() .map_err(|e| anyhow::anyhow!("memory client unavailable: {e}")) } /// Persist the full scraped LinkedIn profile to the user-profile memory /// namespace so the agent has rich context about the user. async fn persist_linkedin_profile( - memory: &crate::openhuman::memory::store::MemoryClient, + memory: &crate::openhuman::memory_store::MemoryClient, url: &str, data: &serde_json::Value, ) -> anyhow::Result<()> { @@ -720,7 +720,7 @@ async fn persist_linkedin_profile( /// Fallback: persist just the LinkedIn URL when the full scrape fails. async fn persist_linkedin_url_only( - memory: &crate::openhuman::memory::store::MemoryClient, + memory: &crate::openhuman::memory_store::MemoryClient, url: &str, ) -> anyhow::Result<()> { memory diff --git a/src/openhuman/learning/profile_md_renderer.rs b/src/openhuman/learning/profile_md_renderer.rs index 484bb6ae9..e610815a0 100644 --- a/src/openhuman/learning/profile_md_renderer.rs +++ b/src/openhuman/learning/profile_md_renderer.rs @@ -42,7 +42,7 @@ use async_trait::async_trait; use crate::core::event_bus::{subscribe_global, DomainEvent, EventHandler, SubscriptionHandle}; use crate::openhuman::composio::providers::profile_md::replace_managed_block; use crate::openhuman::learning::cache::FacetCache; -use crate::openhuman::memory::store::profile::UserState; +use crate::openhuman::memory_store::profile::UserState; // ── Class → block metadata ──────────────────────────────────────────────────── @@ -215,7 +215,7 @@ impl EventHandler for RendererSubscriber { mod tests { use super::*; use crate::openhuman::composio::providers::profile_md::{block_end, block_start}; - use crate::openhuman::memory::store::profile::{ + use crate::openhuman::memory_store::profile::{ FacetState, FacetType, ProfileFacet, UserState, PROFILE_INIT_SQL, }; use parking_lot::Mutex; diff --git a/src/openhuman/learning/prompt_sections.rs b/src/openhuman/learning/prompt_sections.rs index fe97a1304..0bbb0026a 100644 --- a/src/openhuman/learning/prompt_sections.rs +++ b/src/openhuman/learning/prompt_sections.rs @@ -163,7 +163,7 @@ pub fn load_learned_from_cache( // Group by class prefix (portion before the first '/'), then sort within // each class by stability descending, then by key alphabetically. - use crate::openhuman::memory::store::profile::ProfileFacet; + use crate::openhuman::memory_store::profile::ProfileFacet; use std::collections::BTreeMap; let mut by_class: BTreeMap> = BTreeMap::new(); @@ -197,7 +197,7 @@ pub fn load_learned_from_cache( // agent can parse the source. Goal class keeps value-only (full // sentence, no key prefix). Pinned entries get a trailing suffix. let pinned = - if f.user_state == crate::openhuman::memory::store::profile::UserState::Pinned { + if f.user_state == crate::openhuman::memory_store::profile::UserState::Pinned { " *(pinned)*" } else { "" @@ -376,7 +376,7 @@ mod tests { #[test] fn load_learned_from_cache_formats_active_facets() { use crate::openhuman::learning::cache::FacetCache; - use crate::openhuman::memory::store::profile::{ + use crate::openhuman::memory_store::profile::{ FacetState, FacetType, ProfileFacet, UserState, PROFILE_INIT_SQL, }; use parking_lot::Mutex; @@ -456,7 +456,7 @@ mod tests { #[test] fn load_learned_from_cache_empty_when_no_active_facets() { use crate::openhuman::learning::cache::FacetCache; - use crate::openhuman::memory::store::profile::PROFILE_INIT_SQL; + use crate::openhuman::memory_store::profile::PROFILE_INIT_SQL; use parking_lot::Mutex; use rusqlite::Connection; diff --git a/src/openhuman/learning/prompt_sections_tests.rs b/src/openhuman/learning/prompt_sections_tests.rs index a76686572..e7bf9d782 100644 --- a/src/openhuman/learning/prompt_sections_tests.rs +++ b/src/openhuman/learning/prompt_sections_tests.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use super::load_learned_from_cache; use crate::openhuman::learning::cache::FacetCache; -use crate::openhuman::memory::store::profile::{ +use crate::openhuman::memory_store::profile::{ FacetState, FacetType, ProfileFacet, UserState, PROFILE_INIT_SQL, }; diff --git a/src/openhuman/learning/schemas.rs b/src/openhuman/learning/schemas.rs index 62594eeb1..02eac4c5f 100644 --- a/src/openhuman/learning/schemas.rs +++ b/src/openhuman/learning/schemas.rs @@ -644,7 +644,7 @@ fn handle_rebuild_cache(_params: Map) -> ControllerFuture { fn handle_cache_stats(_params: Map) -> ControllerFuture { Box::pin(async move { use crate::openhuman::learning::cache::FacetCache; - use crate::openhuman::memory::store::profile::FacetState; + use crate::openhuman::memory_store::profile::FacetState; tracing::debug!("[learning.cache_stats] cache stats requested via RPC"); @@ -723,7 +723,7 @@ fn full_key(class_str: &str, key_suffix: &str) -> String { } /// Serialize a [`ProfileFacet`] to a serde_json [`Value`] for RPC output. -fn facet_to_json(f: &crate::openhuman::memory::store::profile::ProfileFacet) -> serde_json::Value { +fn facet_to_json(f: &crate::openhuman::memory_store::profile::ProfileFacet) -> serde_json::Value { serde_json::json!({ "key": f.key, "value": f.value, @@ -742,7 +742,7 @@ fn facet_to_json(f: &crate::openhuman::memory::store::profile::ProfileFacet) -> fn handle_list_facets(params: Map) -> ControllerFuture { Box::pin(async move { - use crate::openhuman::memory::store::profile::FacetState; + use crate::openhuman::memory_store::profile::FacetState; tracing::debug!("[learning.list_facets] called"); @@ -822,7 +822,7 @@ fn handle_get_facet(params: Map) -> ControllerFuture { fn handle_update_facet(params: Map) -> ControllerFuture { Box::pin(async move { - use crate::openhuman::memory::store::profile::UserState; + use crate::openhuman::memory_store::profile::UserState; let class_str = params .get("class") @@ -875,7 +875,7 @@ fn handle_update_facet(params: Map) -> ControllerFuture { fn handle_pin_facet(params: Map) -> ControllerFuture { Box::pin(async move { - use crate::openhuman::memory::store::profile::UserState; + use crate::openhuman::memory_store::profile::UserState; let class_str = params .get("class") @@ -915,7 +915,7 @@ fn handle_pin_facet(params: Map) -> ControllerFuture { fn handle_unpin_facet(params: Map) -> ControllerFuture { Box::pin(async move { - use crate::openhuman::memory::store::profile::UserState; + use crate::openhuman::memory_store::profile::UserState; let class_str = params .get("class") @@ -955,7 +955,7 @@ fn handle_unpin_facet(params: Map) -> ControllerFuture { fn handle_forget_facet(params: Map) -> ControllerFuture { Box::pin(async move { - use crate::openhuman::memory::store::profile::{FacetState, UserState}; + use crate::openhuman::memory_store::profile::{FacetState, UserState}; let class_str = params .get("class") @@ -1003,7 +1003,7 @@ fn handle_forget_facet(params: Map) -> ControllerFuture { fn handle_reset_cache(_params: Map) -> ControllerFuture { Box::pin(async move { - use crate::openhuman::memory::store::profile::UserState; + use crate::openhuman::memory_store::profile::UserState; tracing::debug!("[learning.reset_cache] called"); diff --git a/src/openhuman/learning/stability_detector.rs b/src/openhuman/learning/stability_detector.rs index 1710ab56f..0d17d07a4 100644 --- a/src/openhuman/learning/stability_detector.rs +++ b/src/openhuman/learning/stability_detector.rs @@ -37,7 +37,7 @@ use crate::core::event_bus; use crate::core::event_bus::DomainEvent; use crate::openhuman::learning::cache::FacetCache; use crate::openhuman::learning::candidate::{self, CueFamily, FacetClass, LearningCandidate}; -use crate::openhuman::memory::store::profile::{FacetState, FacetType, ProfileFacet, UserState}; +use crate::openhuman::memory_store::profile::{FacetState, FacetType, ProfileFacet, UserState}; // ── Thresholds ──────────────────────────────────────────────────────────────── @@ -554,7 +554,7 @@ mod tests { use crate::openhuman::learning::candidate::{ Buffer, EvidenceRef, FacetClass, LearningCandidate, }; - use crate::openhuman::memory::store::profile::PROFILE_INIT_SQL; + use crate::openhuman::memory_store::profile::PROFILE_INIT_SQL; use parking_lot::Mutex; use rusqlite::Connection; use std::sync::Arc; @@ -788,7 +788,7 @@ mod tests { let now = 1_000_000.0; // Manually insert a Pinned row. - use crate::openhuman::memory::store::profile::{FacetState, FacetType, UserState}; + use crate::openhuman::memory_store::profile::{FacetState, FacetType, UserState}; let pinned = ProfileFacet { facet_id: "f-pinned".into(), facet_type: FacetType::Preference, diff --git a/src/openhuman/memory/README.md b/src/openhuman/memory/README.md index d48fc7cf1..19a77c129 100644 --- a/src/openhuman/memory/README.md +++ b/src/openhuman/memory/README.md @@ -1,82 +1,75 @@ -# Memory +# memory -Persistent knowledge layer. Owns the unified store (SQLite + FTS5 + vector embeddings + graph relations), document ingestion pipelines, namespace + KV operations, conversation history, and retrieval scoring. Does NOT own raw provider embedding APIs (`local_ai/`), agent prompt assembly (`agent/memory_loader.rs`), or per-channel ingestion adapters beyond the bundled Slack importer. +Orchestration layer over the memory stack. Owns: -## Architecture +- **Ingest pipeline** — orchestrates source → canonicalise → chunk → + score → persist → enqueue extract jobs. +- **Job handlers** — background workers that drain the queue (admit, + extract, seal, digest, topic-curate). +- **Scoring** — fast scorers, signal aggregation, score persistence. +- **Tree policy** — `tree_global` and `tree_topic` building rules. +- **RPC surface** — `read_rpc`, `tree_rpc`, controller schemas for the + memory_* RPC namespace. +- **Recall** — `stm_recall`, `retrieval` ranking, query orchestration. -The module is organised in concentric layers — the contract on the -inside, the persistent backend around it, the ingestion + retrieval -pipelines on top, and the per-domain glue at the edge: +Does **not** own any storage primitives — those live in +[`memory_store`](../memory_store/). See that module for raw md, chunks, +entities, trees, vectors, kv, and contacts. -```text - ┌──────────────────────────────────────┐ - │ conversations/ slack_ingestion/ │ per-domain plumbing - ├──────────────────────────────────────┤ - │ tree/ (bucket-seal LLD pipeline) │ new retrieval architecture - ├──────────────────────────────────────┤ - │ ingestion/ (extract chunks) │ document ingestion - ├──────────────────────────────────────┤ - │ store/ (UnifiedMemory backend) │ SQLite + FTS5 + vectors - ├──────────────────────────────────────┤ - │ traits.rs (Memory trait) │ contract - └──────────────────────────────────────┘ -``` +## Sibling memory_* modules -- **`traits.rs`** — `Memory`, `MemoryEntry`, `MemoryCategory`, - `RecallOpts`. The backend-agnostic contract every store implements. -- **`store/`** — `UnifiedMemory` is the production backend (SQLite - with FTS5 for keyword search, vector tables for embeddings, and - graph tables for entity/relation triples) plus the `MemoryClient` - handle used by the rest of the process. -- **`ingestion/`** — chunking + extraction pipeline (entities, - relations, embeddings) and the background `IngestionQueue` worker. -- **`tree/`** — the new bucket-seal retrieval architecture from - `docs/MEMORY_ARCHITECTURE_LLD.md`: `canonicalize` (normalise - inputs), `chunker` and `content_store` (durable chunks), - `score`/`retrieval` (ranking surface), - `tree_source`/`tree_topic`/`tree_global` (the three concentric - trees the LLD calls for), and `jobs` (background seals/summaries). -- **`conversations/`** — workspace-backed JSONL chat thread/message - history. See `conversations/README.md`. -- **`slack_ingestion/`** — Slack provider plumbing (bucketer + - ingest wrapper + RPC). See `slack_ingestion/README.md`. +The memory stack is split across several top-level modules so each has +one job. memory orchestrates and routes between them. -The legacy memory store (`store/` + `ingestion/`) and the new -`tree/` pipeline coexist for now — `tree/` is replacing the older -retrieval surface incrementally and both must remain wired into RPC -until the migration completes. +| Module | Role | +| --- | --- | +| [`memory_store`](../memory_store/) | Storage primitives: raw / chunks / entities / trees / vectors / kv / contacts. SQLite + on-disk md. | +| [`memory_tree`](../memory_tree/) | Generic tree mechanics: bucket-seal, flush, summarise, walk. Kind-agnostic. | +| [`memory_archivist`](../memory_archivist/) | Chat conversation → clip tool-calls → push to tree. | +| [`memory_entities`](../memory_entities/) | Md-backed entity registry (people + orgs + topics + …). Replacing `people/`. | +| [`memory_graph`](../memory_graph/) | Derived co-occurrence edges over the entity index. | +| [`memory_tools`](../memory_tools/) | Tool-scoped rules + agent read/write tools. | +| [`memory_sync`](../memory_sync/) | Composio + workspace + MCP sync pipelines. | -## Public surface +## What lives here -- `pub trait Memory` / `pub struct MemoryEntry` / `pub enum MemoryCategory` / `pub struct RecallOpts` — `traits.rs:11-100` — backend contract for any memory store. -- `pub struct UnifiedMemory` — `store/unified/` (re-exported `store/mod.rs:40`) — primary SQLite + FTS5 + vector implementation. -- `pub struct MemoryClient` / `pub struct MemoryClientRef` / `pub enum MemoryState` — `store/client.rs` — async client handle used by RPC handlers. -- `pub fn create_memory` / `pub fn create_memory_with_storage` / `pub fn create_memory_with_storage_and_routes` / `pub fn create_memory_for_migration` — `store/factories.rs` — bootstrap a memory instance. -- `pub struct MemoryIngestionRequest` / `pub struct MemoryIngestionResult` / `pub struct MemoryIngestionConfig` / `pub enum ExtractionMode` / `pub struct ExtractedEntity` / `pub struct ExtractedRelation` / `const DEFAULT_MEMORY_EXTRACTION_MODEL` — `ingestion.rs` (re-exported `mod.rs:22`). -- `pub struct IngestionQueue` / `pub struct IngestionJob` — `ingestion_queue.rs` — async background ingestion worker. -- `pub struct NamespaceDocumentInput` / `pub struct NamespaceMemoryHit` / `pub struct NamespaceQueryResult` / `pub struct NamespaceRetrievalContext` / `pub struct RetrievalScoreBreakdown` / `pub enum MemoryItemKind` — `store/types.rs`. -- RPC `memory.{init, list_documents, list_namespaces, delete_document, query_namespace, recall_context, recall_memories, list_files, read_file, write_file, namespace_list, doc_put, doc_ingest, doc_list, doc_delete, context_query, context_recall, kv_set, kv_get, kv_delete, kv_list_namespace, graph_upsert, graph_query, clear_namespace}` — `schemas.rs:29-55`. -- RPC tree `memory.tree.*` and retrieval — `tree/` (re-exported via `all_memory_tree_*` / `all_retrieval_*`). -- RPC slack ingestion — `slack_ingestion/` (re-exported via `all_slack_ingestion_*`). +| Path | Role | +| --- | --- | +| [`mod.rs`](mod.rs) | Module root + re-exports. | +| [`ingest_pipeline.rs`](ingest_pipeline.rs) | Source-agnostic ingest orchestration. Called by sync pipelines and tree_rpc. | +| [`jobs/`](../memory_queue/) | Background workers: extract, admit, seal, digest, topic curate. Re-exported here from `memory_queue`. | +| [`score/`](score/) | Fast scorer, signals, embeddings, entity extraction, entity-index persistence. `store.rs` will eventually split — entity-index pieces move to `memory_store::entities`. | +| [`retrieval/`](retrieval/) | Drill-down, fetch, query_source, query_global, query_topic, search; scoring + ranking on top of memory_store. | +| [`tree_global/`](../memory_tree/global/) | Global digest tree building policy: seal, digest, recap. Implemented in `memory_tree/global` and routed from here. | +| [`tree_topic/`](../memory_tree/topic/) | Topic tree building policy: hotness gating, routing, curator, backfill. Implemented in `memory_tree/topic` and routed from here. | +| [`summarizer/`](../memory_tree/summarise.rs) | LLM summarisation pipeline for sealed buckets. Implemented in `memory_tree/summarise.rs`. | +| [`stm_recall/`](stm_recall/) | Short-term recall: cross-session FTS5 lookup + ranking. | +| [`ingestion/`](ingestion/) | Document ingestion queue + extraction (entities, relations, embeddings) — feeds UnifiedMemory documents. | +| [`canonicalize/`](../memory_sync/canonicalize/) | Source → canonical markdown (chat / email / document). Implemented in `memory_sync/canonicalize` and used at ingest time. | +| [`chat/`](chat.rs) | Chat-source canonicalisation helpers. | +| [`conversations/`](../memory_conversations/) | Workspace-backed JSONL chat thread/message history. Re-exported here from `memory_conversations`. | +| [`read_rpc.rs`](read_rpc.rs) | RPC handlers for memory reads. | +| [`tree_rpc.rs`](tree_rpc.rs) | RPC handlers for tree ingest + introspection. | +| [`schemas/`](schemas/) + [`schema.rs`](schema.rs) | Controller schema definitions for the memory + memory_tree RPC namespaces. | +| [`sync_status/`](../memory_sync/sync_status/) | Sync freshness tracking + RPC. Re-exported here from `memory_sync::sync_status`. | +| [`ops/`](ops/) | RPC operation handlers + the shared `active_memory_client` helper. | +| [`preferences.rs`](preferences.rs) | User preference read/write helpers. | +| [`rpc_models.rs`](rpc_models.rs) | Shared RPC request/response shapes. | +| [`traits.rs`](traits.rs) | `Memory`, `MemoryEntry`, `MemoryCategory`, `NamespaceSummary`, `RecallOpts`. The backend-agnostic contract every store implements. | +| [`util/`](util/) | Small helpers (redact for log PII). | +| [`global.rs`](global.rs) | Global-namespace helpers. | -## Calls into +## Layer rules -- `src/openhuman/local_ai/` — embedding model, sentiment scoring, extraction LLM. -- `src/openhuman/embeddings/` — vector backend selection. -- `src/openhuman/config/` — memory backend choice + filesystem paths. -- `src/openhuman/encryption/` — at-rest secrets for KV namespaces. -- `src/core/event_bus/` — emits `DomainEvent::Memory(*)` on ingestion / mutation. - -## Called by - -- `src/openhuman/agent/` (`memory_loader.rs`, `harness/memory_context.rs`, `harness/archivist*.rs`, `harness/fork_context.rs`) — context injection and episodic indexing. -- `src/openhuman/learning/{reflection,tool_tracker,user_profile,prompt_sections}.rs` — long-term insight storage. -- `src/openhuman/screen_intelligence/{helpers,tests}.rs` — recall surfaces for visual context. -- `src/openhuman/autocomplete/history.rs` — query-history recall. -- `src/openhuman/tools/ops.rs` and `tools/impl/system/tool_stats.rs` — memory-backed tool stats. -- `src/core/all.rs` — registers `all_memory_*` controllers. - -## Tests - -- Unit: `ops_tests.rs`, `schemas_tests.rs`, `rpc_models_tests.rs`, `ingestion_tests.rs`, plus `*_tests.rs` files inside `store/`, `tree/`, `conversations/`, `slack_ingestion/`. -- Integration: `tests/autocomplete_memory_e2e.rs`, `tests/memory_graph_sync_e2e.rs`. +- **No storage in this module.** All persistence goes through + `memory_store::*`. If you're tempted to open a SQLite connection + here, the connection helper belongs one layer down. +- **No upward dependencies.** memory may import from memory_store / + memory_tree / memory_entities / memory_archivist / memory_graph / + memory_tools, but the inverse is a layer violation. (The two + documented exceptions today — `memory_store::retrieval::tree_walk` + calling `memory::retrieval::drill_down`, and `memory_store::trees::registry` + pulling `GLOBAL_SCOPE` from `memory::tree_global` — are tracked in + their respective READMEs.) +- **Surface high-level tool calls** that route to the right submodule; + don't expose internals at the call site. diff --git a/src/openhuman/memory/chat.rs b/src/openhuman/memory/chat.rs new file mode 100644 index 000000000..83c2edde4 --- /dev/null +++ b/src/openhuman/memory/chat.rs @@ -0,0 +1,257 @@ +//! Memory LLM adapter backed by the unified inference provider stack. +//! +//! Memory callers still want a tiny prompt surface: one system message, one +//! user message, and a string response. This module keeps that narrow contract +//! for the rest of the memory layer, but routes every production call through +//! `openhuman::inference::provider` so memory uses the same workload routing as +//! the rest of the app. + +use std::sync::Arc; + +use anyhow::Result; +use async_trait::async_trait; + +use crate::openhuman::config::{Config, DEFAULT_CLOUD_LLM_MODEL}; +use crate::openhuman::inference::provider::{ + create_chat_provider, provider_for_role, ChatMessage, Provider, +}; + +/// One pair of prompt messages handed to the memory LLM backend. +#[derive(Debug, Clone)] +pub struct ChatPrompt { + pub system: String, + pub user: String, + pub temperature: f64, + pub kind: &'static str, +} + +/// Pluggable LLM surface used by the memory layer. +#[async_trait] +pub trait ChatProvider: Send + Sync { + fn name(&self) -> &str; + + async fn chat_for_json(&self, prompt: &ChatPrompt) -> Result; + + async fn chat_for_text(&self, prompt: &ChatPrompt) -> Result { + self.chat_for_json(prompt).await + } +} + +struct InferenceChatProvider { + inner: Box, + model: String, + display: String, +} + +impl InferenceChatProvider { + fn new(inner: Box, model: String) -> Self { + let display = format!("inference:{model}"); + Self { + inner, + model, + display, + } + } + + async fn run(&self, prompt: &ChatPrompt) -> Result { + log::debug!( + "[memory::chat] provider={} kind={} model={} sys_chars={} user_chars={}", + self.display, + prompt.kind, + self.model, + prompt.system.len(), + prompt.user.len() + ); + + let messages = vec![ + ChatMessage::system(prompt.system.clone()), + ChatMessage::user(prompt.user.clone()), + ]; + + let text = self + .inner + .chat_with_history(&messages, &self.model, prompt.temperature) + .await?; + + log::debug!( + "[memory::chat] provider={} kind={} response_chars={}", + self.display, + prompt.kind, + text.len() + ); + + Ok(text) + } +} + +#[async_trait] +impl ChatProvider for InferenceChatProvider { + fn name(&self) -> &str { + &self.display + } + + async fn chat_for_json(&self, prompt: &ChatPrompt) -> Result { + self.run(prompt).await + } + + async fn chat_for_text(&self, prompt: &ChatPrompt) -> Result { + self.run(prompt).await + } +} + +fn routed_memory_config(config: &Config) -> Config { + let mut routed = config.clone(); + if !config.workload_uses_local("memory") { + routed.default_model = Some( + config + .memory_tree + .cloud_llm_model + .clone() + .unwrap_or_else(|| DEFAULT_CLOUD_LLM_MODEL.to_string()), + ); + } + routed +} + +#[cfg(test)] +fn test_override_runtime() -> Option<(Arc, String)> { + test_override::current().map(|provider| (provider, "test:override".to_string())) +} + +#[cfg(not(test))] +fn test_override_runtime() -> Option<(Arc, String)> { + None +} + +/// Build the memory LLM provider and return the resolved model id. +pub fn build_chat_runtime(config: &Config) -> Result<(Arc, String)> { + if let Some(runtime) = test_override_runtime() { + return Ok(runtime); + } + + let routed = routed_memory_config(config); + let resolved_provider = provider_for_role("summarization", &routed); + let (provider, model) = create_chat_provider("summarization", &routed)?; + + log::debug!( + "[memory::chat] built provider route={} model={}", + resolved_provider, + model + ); + + Ok(( + Arc::new(InferenceChatProvider::new(provider, model.clone())), + model, + )) +} + +/// Build the memory LLM provider dictated by the inference workload routing. +pub fn build_chat_provider(config: &Config) -> Result> { + Ok(build_chat_runtime(config)?.0) +} + +#[cfg(test)] +pub struct StaticChatProvider { + pub response: String, + pub calls: std::sync::atomic::AtomicUsize, +} + +#[cfg(test)] +impl StaticChatProvider { + pub fn new(response: impl Into) -> Self { + Self { + response: response.into(), + calls: std::sync::atomic::AtomicUsize::new(0), + } + } +} + +#[cfg(test)] +#[async_trait] +impl ChatProvider for StaticChatProvider { + fn name(&self) -> &str { + "test:static" + } + + async fn chat_for_json(&self, _prompt: &ChatPrompt) -> Result { + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(self.response.clone()) + } +} + +#[cfg(test)] +pub mod test_override { + use super::ChatProvider; + use std::sync::Arc; + + tokio::task_local! { + static OVERRIDE: Arc; + } + + pub fn current() -> Option> { + OVERRIDE.try_with(Arc::clone).ok() + } + + pub async fn with_provider(provider: Arc, fut: F) -> T + where + F: std::future::Future, + { + OVERRIDE.scope(provider, fut).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn build_provider_returns_inference_wrapper_when_default() { + let cfg = Config::default(); + let provider = build_chat_provider(&cfg).unwrap(); + assert!(provider.name().contains("inference:")); + } + + #[test] + fn build_chat_runtime_defaults_to_openhuman_resolved_model() { + let cfg = Config::default(); + let (_provider, model) = build_chat_runtime(&cfg).unwrap(); + assert_eq!(model, "reasoning-v1"); + } + + #[test] + fn build_chat_runtime_still_builds_when_cloud_memory_model_is_overridden() { + let mut cfg = Config::default(); + cfg.memory_tree.cloud_llm_model = Some("custom-summary-model".into()); + let (_provider, model) = build_chat_runtime(&cfg).unwrap(); + assert_eq!(model, "reasoning-v1"); + } + + #[test] + fn build_provider_returns_inference_wrapper_when_local_memory_is_configured() { + let mut cfg = Config::default(); + cfg.memory_provider = Some("ollama:qwen2.5:0.5b".into()); + let provider = build_chat_provider(&cfg).unwrap(); + assert!(provider.name().contains("qwen2.5:0.5b")); + } + + #[test] + fn build_chat_runtime_preserves_local_memory_model() { + let mut cfg = Config::default(); + cfg.memory_provider = Some("ollama:qwen2.5:0.5b".into()); + let (_provider, model) = build_chat_runtime(&cfg).unwrap(); + assert_eq!(model, "qwen2.5:0.5b"); + } + + #[tokio::test] + async fn static_chat_provider_returns_response_and_counts() { + let p = StaticChatProvider::new("hello"); + let prompt = ChatPrompt { + system: "sys".into(), + user: "u".into(), + temperature: 0.0, + kind: "test", + }; + assert_eq!(p.chat_for_json(&prompt).await.unwrap(), "hello"); + assert_eq!(p.calls.load(std::sync::atomic::Ordering::SeqCst), 1); + } +} diff --git a/src/openhuman/memory/conversations/types.rs b/src/openhuman/memory/conversations/types.rs deleted file mode 100644 index 8f7a7c657..000000000 --- a/src/openhuman/memory/conversations/types.rs +++ /dev/null @@ -1,72 +0,0 @@ -//! Wire/storage types for the workspace-backed conversation store: threads, -//! messages, create requests, and partial-update patches. - -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -/// A persisted conversation thread, mirroring one entry in `threads.jsonl`. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase")] -pub struct ConversationThread { - pub id: String, - pub title: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub chat_id: Option, - pub is_active: bool, - pub message_count: usize, - pub last_message_at: String, - pub created_at: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub parent_thread_id: Option, - #[serde(default)] - pub labels: Vec, -} - -/// A single message appended to a thread's JSONL log. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase")] -pub struct ConversationMessage { - pub id: String, - pub content: String, - #[serde(rename = "type")] - pub message_type: String, - #[serde(default)] - pub extra_metadata: Value, - pub sender: String, - pub created_at: String, -} - -/// Input payload to create-or-update a thread via [`super::ensure_thread`]. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CreateConversationThread { - pub id: String, - pub title: String, - pub created_at: String, - #[serde(default)] - pub parent_thread_id: Option, - #[serde(default)] - pub labels: Option>, -} - -/// Partial update to apply to a stored message (e.g. rewriting `extraMetadata`). -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -#[serde(rename_all = "camelCase")] -pub struct ConversationMessagePatch { - #[serde(default)] - pub extra_metadata: Option, -} - -/// A single match returned by -/// [`super::store::ConversationStore::search_cross_thread_messages`]. Carries -/// the source `thread_id` so the caller can render provenance into the -/// `[Cross-chat context]` block (issue #1505). -#[derive(Debug, Clone, PartialEq)] -pub struct CrossThreadHit { - pub thread_id: String, - pub message_id: String, - pub role: String, - pub content: String, - pub created_at: String, - pub score: f64, -} diff --git a/src/openhuman/memory_tree/ingest.rs b/src/openhuman/memory/ingest_pipeline.rs similarity index 92% rename from src/openhuman/memory_tree/ingest.rs rename to src/openhuman/memory/ingest_pipeline.rs index 49a0f52fb..26bcfd4c6 100644 --- a/src/openhuman/memory_tree/ingest.rs +++ b/src/openhuman/memory/ingest_pipeline.rs @@ -11,19 +11,19 @@ use serde::{Deserialize, Serialize}; use crate::core::event_bus::{publish_global, DomainEvent}; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::canonicalize::{ +use crate::openhuman::memory::jobs::{self, ExtractChunkPayload, NewJob}; +use crate::openhuman::memory::score::{self, ScoreResult, ScoringConfig}; +use crate::openhuman::memory::util::redact::redact; +use crate::openhuman::memory_store::chunks::store as chunk_store; +use crate::openhuman::memory_store::chunks::types::SourceKind; +use crate::openhuman::memory_store::chunks::{chunk_markdown, ChunkerInput, ChunkerOptions}; +use crate::openhuman::memory_store::content as content_store; +use crate::openhuman::memory_sync::canonicalize::{ chat::{self, ChatBatch}, document::{self, DocumentInput}, email::{self, EmailThread}, CanonicalisedSource, }; -use crate::openhuman::memory_tree::chunker::{chunk_markdown, ChunkerInput, ChunkerOptions}; -use crate::openhuman::memory_tree::content_store; -use crate::openhuman::memory_tree::jobs::{self, ExtractChunkPayload, NewJob}; -use crate::openhuman::memory_tree::score::{self, ScoreResult, ScoringConfig}; -use crate::openhuman::memory_tree::store; -use crate::openhuman::memory_tree::types::SourceKind; -use crate::openhuman::memory_tree::util::redact::redact; use std::time::{SystemTime, UNIX_EPOCH}; const BODY_PREVIEW_MAX_BYTES: usize = 2048; @@ -124,7 +124,7 @@ pub async fn ingest_document( ) -> Result { if already_ingested(config, SourceKind::Document, source_id).await? { log::debug!( - "[memory_tree::ingest] skip ingest_document — source_id_hash={} already ingested", + "[memory::ingest_pipeline] skip ingest_document — source_id_hash={} already ingested", redact(source_id) ); return Ok(IngestResult::already_ingested(source_id)); @@ -147,7 +147,7 @@ async fn already_ingested( ) -> Result { let cfg = config.clone(); let sid = source_id.to_string(); - tokio::task::spawn_blocking(move || store::is_source_ingested(&cfg, source_kind, &sid)) + tokio::task::spawn_blocking(move || chunk_store::is_source_ingested(&cfg, source_kind, &sid)) .await .map_err(|e| anyhow::anyhow!("already_ingested join error: {e}"))? } @@ -175,8 +175,8 @@ async fn persist( Ok(preview) => Some(preview), Err(_) => { log::error!( - "[memory_tree::ingest] markdown_body_preview panicked for source_id_hash={}; falling back to no preview", - crate::openhuman::memory_tree::util::redact::redact(source_id) + "[memory::ingest_pipeline] markdown_body_preview panicked for source_id_hash={}; falling back to no preview", + crate::openhuman::memory::util::redact::redact(source_id) ); None } @@ -201,13 +201,13 @@ async fn persist( // spawn_blocking so errors surface before the DB transaction opens. let content_root = config.memory_tree_content_root(); let staged = content_store::stage_chunks(&content_root, &chunks) - .map_err(|e| anyhow::anyhow!("[memory_tree::ingest] stage_chunks failed: {e}"))?; + .map_err(|e| anyhow::anyhow!("[memory::ingest_pipeline] stage_chunks failed: {e}"))?; let scoring_cfg = ScoringConfig::from_config(config); let scores = score::score_chunks_fast(&chunks, &scoring_cfg).await?; if scores.len() != chunks.len() { anyhow::bail!( - "[memory_tree::ingest] scorer length mismatch: chunks={} scores={}", + "[memory::ingest_pipeline] scorer length mismatch: chunks={} scores={}", chunks.len(), scores.len() ); @@ -226,7 +226,7 @@ async fn persist( let source_id_for_store = source_id.to_string(); let written = tokio::task::spawn_blocking(move || -> Result> { use std::collections::{HashMap, HashSet}; - store::with_connection(&config_owned, |conn| { + chunk_store::with_connection(&config_owned, |conn| { // IMMEDIATE, not the default DEFERRED: this transaction reads // (get_chunk_lifecycle_status_tx) before it writes // (upsert_staged_chunks_tx). A DEFERRED tx takes only a read @@ -262,7 +262,7 @@ async fn persist( // genuine replays without blocking legitimate appends. if source_kind_for_store == SourceKind::Document { let now_ms = chrono::Utc::now().timestamp_millis(); - let claimed = store::claim_source_ingest_tx( + let claimed = chunk_store::claim_source_ingest_tx( &tx, source_kind_for_store, &source_id_for_store, @@ -270,7 +270,7 @@ async fn persist( )?; if !claimed { log::debug!( - "[memory_tree::ingest] persist gate: document already ingested source_id_hash={}", + "[memory::ingest_pipeline] persist gate: document already ingested source_id_hash={}", redact(&source_id_for_store) ); // Drop the (empty) transaction implicitly; nothing to commit. @@ -287,11 +287,11 @@ async fn persist( // "already-admitted-from-prior-ingest". let mut prior: HashMap> = HashMap::new(); for s in &staged_for_store { - let status = store::get_chunk_lifecycle_status_tx(&tx, &s.chunk.id)?; + let status = chunk_store::get_chunk_lifecycle_status_tx(&tx, &s.chunk.id)?; prior.insert(s.chunk.id.clone(), status); } - let n = store::upsert_staged_chunks_tx(&tx, &staged_for_store)?; + let n = chunk_store::upsert_staged_chunks_tx(&tx, &staged_for_store)?; // Re-ingest of identical content (same chunk_id) must NOT // downgrade chunks that have already progressed through the @@ -312,13 +312,13 @@ async fn persist( let pre = prior.get(&s.chunk.id).cloned().flatten(); let needs_processing = matches!( pre.as_deref(), - None | Some(store::CHUNK_STATUS_PENDING_EXTRACTION), + None | Some(chunk_store::CHUNK_STATUS_PENDING_EXTRACTION), ); if needs_processing { - store::set_chunk_lifecycle_status_tx( + chunk_store::set_chunk_lifecycle_status_tx( &tx, &s.chunk.id, - store::CHUNK_STATUS_PENDING_EXTRACTION, + chunk_store::CHUNK_STATUS_PENDING_EXTRACTION, )?; to_schedule.insert(s.chunk.id.clone()); } @@ -407,7 +407,7 @@ fn markdown_body_preview(md: &str) -> String { // continuation byte; fall back to the full string rather than panicking. if start > len || !md.is_char_boundary(start) { log::error!( - "[memory_tree::ingest] ceil_char_boundary returned invalid boundary start={start} len={len}; returning full markdown" + "[memory::ingest_pipeline] ceil_char_boundary returned invalid boundary start={start} len={len}; returning full markdown" ); md.to_string() } else { @@ -419,14 +419,14 @@ fn markdown_body_preview(md: &str) -> String { #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory_tree::canonicalize::chat::ChatMessage; - use crate::openhuman::memory_tree::jobs::drain_until_idle; - use crate::openhuman::memory_tree::score::store::{count_scores, lookup_entity}; - use crate::openhuman::memory_tree::store::{ + use crate::openhuman::memory::jobs::drain_until_idle; + use crate::openhuman::memory::score::store::{count_scores, lookup_entity}; + use crate::openhuman::memory_store::chunks::store::{ count_chunks, count_chunks_by_lifecycle_status, get_chunk_embedding, list_chunks, ListChunksQuery, CHUNK_STATUS_BUFFERED, CHUNK_STATUS_DROPPED, }; - use crate::openhuman::memory_tree::types::SourceKind; + use crate::openhuman::memory_store::chunks::types::SourceKind; + use crate::openhuman::memory_sync::canonicalize::chat::ChatMessage; use chrono::{TimeZone, Utc}; use tempfile::TempDir; diff --git a/src/openhuman/memory/ingestion/mod.rs b/src/openhuman/memory/ingestion/mod.rs index b4724564c..0ed7f6238 100644 --- a/src/openhuman/memory/ingestion/mod.rs +++ b/src/openhuman/memory/ingestion/mod.rs @@ -30,8 +30,8 @@ use parse::{enrich_document_metadata, parse_document}; use serde_json::json; use types::ParsedIngestion; -use crate::openhuman::memory::store::types::NamespaceDocumentInput; use crate::openhuman::memory::UnifiedMemory; +use crate::openhuman::memory_store::types::NamespaceDocumentInput; impl UnifiedMemory { /// Run the full ingestion pipeline for a document: parse + chunk + extract diff --git a/src/openhuman/memory/ingestion/parse.rs b/src/openhuman/memory/ingestion/parse.rs index 9623cb399..f67249be9 100644 --- a/src/openhuman/memory/ingestion/parse.rs +++ b/src/openhuman/memory/ingestion/parse.rs @@ -14,8 +14,8 @@ use super::types::{ ExtractedEntity, ExtractedRelation, ExtractionAccumulator, ExtractionMode, ExtractionUnit, MemoryIngestionConfig, ParsedIngestion, RawEntity, RawRelation, DEFAULT_CHUNK_TOKENS, }; -use crate::openhuman::memory::store::types::NamespaceDocumentInput; use crate::openhuman::memory::UnifiedMemory; +use crate::openhuman::memory_store::types::NamespaceDocumentInput; // ── Chunking helpers ────────────────────────────────────────────────────────── @@ -933,3 +933,143 @@ pub(super) async fn parse_document( decision_count: accumulator.decisions.len(), } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::memory_store::types::NamespaceDocumentInput; + use serde_json::json; + + fn sample_input() -> NamespaceDocumentInput { + NamespaceDocumentInput { + namespace: "global".into(), + key: "doc-1".into(), + title: "OpenHuman roadmap".into(), + content: "Alice owns roadmap".into(), + source_type: "manual".into(), + priority: "normal".into(), + tags: vec!["existing".into()], + metadata: json!({"seed": true}), + category: "core".into(), + session_id: Some("session-1".into()), + document_id: Some("doc-id-1".into()), + } + } + + #[test] + fn split_sentences_breaks_on_punctuation_and_merges_tiny_fragments() { + let parts = split_sentences("Hello world. Ok.\nNext line?"); + assert_eq!(parts.len(), 2); + assert_eq!(parts[0], "Hello world Ok"); + assert_eq!(parts[1], "Next line?"); + } + + #[test] + fn build_units_respects_extraction_mode() { + let chunks = vec!["One. Two.".to_string(), "Three".to_string()]; + let sentence_units = build_units(&chunks, ExtractionMode::Sentence); + let chunk_units = build_units(&chunks, ExtractionMode::Chunk); + + assert_eq!(sentence_units.len(), 2); + assert_eq!(sentence_units[0].chunk_index, 0); + assert_eq!(sentence_units[0].text, "One Two"); + assert_eq!(sentence_units[1].chunk_index, 1); + assert_eq!(sentence_units[1].text, "Three"); + + assert_eq!(chunk_units.len(), 2); + assert_eq!(chunk_units[0].text, "One. Two"); + assert_eq!(chunk_units[1].text, "Three"); + } + + #[test] + fn find_chunk_index_prefers_hint_then_wraps() { + let chunks = vec![ + "alpha content".to_string(), + "beta needle".to_string(), + "gamma trailing".to_string(), + ]; + assert_eq!(find_chunk_index(&chunks, "needle", 1), 1); + assert_eq!(find_chunk_index(&chunks, "alpha", 2), 0); + assert_eq!(find_chunk_index(&chunks, "missing", 2), 2); + } + + #[test] + fn alias_map_builds_reverse_lookup() { + let mut entities = HashMap::new(); + entities.insert( + "ALICE".into(), + RawEntity { + name: "ALICE".into(), + entity_type: "PERSON".into(), + confidence: 0.8, + }, + ); + entities.insert( + "ALICE SMITH".into(), + RawEntity { + name: "ALICE SMITH".into(), + entity_type: "PERSON".into(), + confidence: 0.9, + }, + ); + let aliases = build_alias_map(&entities); + assert_eq!( + aliases.get("ALICE").map(String::as_str), + Some("ALICE SMITH") + ); + assert_eq!(resolve_alias("ALICE", &aliases), "ALICE SMITH"); + + let reverse = reverse_aliases(&aliases); + assert_eq!(reverse.get("ALICE SMITH"), Some(&vec!["ALICE".to_string()])); + } + + #[test] + fn enrich_document_metadata_merges_tags_and_ingestion_details() { + let input = sample_input(); + let parsed = ParsedIngestion { + tags: vec!["decision".into(), "existing".into()], + metadata: json!({"kind": "profile", "extra": 1}), + entities: vec![], + relations: vec![], + chunk_count: 3, + preference_count: 1, + decision_count: 2, + }; + let config = MemoryIngestionConfig::default(); + let (enriched, tags) = enrich_document_metadata(&input, &parsed, &config); + + assert_eq!(tags, vec!["decision".to_string(), "existing".to_string()]); + assert_eq!(enriched.tags, tags); + assert_eq!(enriched.metadata["seed"], json!(true)); + assert_eq!(enriched.metadata["extra"], json!(1)); + assert_eq!( + enriched.metadata["ingestion"]["model_name"], + config.model_name + ); + assert_eq!(enriched.metadata["ingestion"]["chunk_count"], json!(3)); + } + + #[test] + fn extract_people_from_header_collects_named_people() { + let mut acc = ExtractionAccumulator::default(); + let people = extract_people_from_header( + "Alice Smith , Bob Jones ", + &mut acc, + ); + assert_eq!( + people, + vec!["ALICE SMITH".to_string(), "BOB JONES".to_string()] + ); + assert!(acc.entities.contains_key("ALICE SMITH")); + assert!(acc.entities.contains_key("BOB JONES")); + } + + #[test] + fn detect_primary_subject_only_matches_openhuman() { + assert_eq!( + detect_primary_subject("OpenHuman desktop roadmap"), + Some("OPENHUMAN".to_string()) + ); + assert_eq!(detect_primary_subject("General roadmap"), None); + } +} diff --git a/src/openhuman/memory/ingestion/queue.rs b/src/openhuman/memory/ingestion/queue.rs index c34276444..32f22c4fb 100644 --- a/src/openhuman/memory/ingestion/queue.rs +++ b/src/openhuman/memory/ingestion/queue.rs @@ -19,7 +19,7 @@ use tokio::sync::mpsc; use super::state::IngestionState; use super::MemoryIngestionConfig; use crate::core::event_bus::{publish_global, DomainEvent}; -use crate::openhuman::memory::store::{NamespaceDocumentInput, UnifiedMemory}; +use crate::openhuman::memory_store::{NamespaceDocumentInput, UnifiedMemory}; /// Default capacity of the ingestion job channel. /// diff --git a/src/openhuman/memory/ingestion/regex.rs b/src/openhuman/memory/ingestion/regex.rs index ead2ae59d..1eea47771 100644 --- a/src/openhuman/memory/ingestion/regex.rs +++ b/src/openhuman/memory/ingestion/regex.rs @@ -187,3 +187,33 @@ pub(super) fn classify_entity(name: &str, known_people: &HashMap } "TOPIC" } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sanitize_entity_name_trims_punctuation_and_uppercases() { + assert_eq!(sanitize_entity_name(" Alice Smith. "), "ALICE SMITH"); + assert_eq!(sanitize_entity_name("\"openhuman\""), "OPENHUMAN"); + assert_eq!(sanitize_entity_name(""), ""); + } + + #[test] + fn sanitize_fact_text_collapses_whitespace_and_strips_edges() { + assert_eq!(sanitize_fact_text(" - Hello world. "), "Hello world"); + assert_eq!(sanitize_fact_text(":: spaced\ttext ;;"), "spaced text"); + } + + #[test] + fn classify_entity_detects_dates_people_and_products() { + let mut known_people = HashMap::new(); + known_people.insert("ALICE SMITH".to_string(), "ALICE SMITH".to_string()); + + assert_eq!(classify_entity("Jan 5, 2026", &known_people), "DATE"); + assert_eq!(classify_entity("Alice Smith", &known_people), "PERSON"); + assert_eq!(classify_entity("OpenHuman", &known_people), "PRODUCT"); + assert_eq!(classify_entity("Kitchen", &known_people), "ROOM"); + assert_eq!(classify_entity("phoenix-project", &known_people), "PROJECT"); + } +} diff --git a/src/openhuman/memory/ingestion/rules.rs b/src/openhuman/memory/ingestion/rules.rs index bcda067ca..0fc641752 100644 --- a/src/openhuman/memory/ingestion/rules.rs +++ b/src/openhuman/memory/ingestion/rules.rs @@ -220,3 +220,106 @@ impl ExtractionAccumulator { }); } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn relation_rule_normalizes_supported_predicates() { + let owns = relation_rule("works_on").unwrap(); + assert_eq!(owns.canonical, "OWNS"); + assert_eq!(owns.allowed_head, PERSON_TYPES); + + let prefers = relation_rule("prefers").unwrap(); + assert_eq!(prefers.canonical, "PREFERS"); + + let deadline = relation_rule("due_on").unwrap(); + assert_eq!(deadline.canonical, "HAS_DEADLINE"); + } + + #[test] + fn type_allowed_honors_allowlist_and_empty_list() { + assert!(type_allowed("PERSON", PERSON_TYPES)); + assert!(!type_allowed("PROJECT", PERSON_TYPES)); + assert!(type_allowed("ANYTHING", &[])); + } + + #[test] + fn resolve_person_alias_uses_known_people_map() { + let mut known = std::collections::HashMap::new(); + known.insert("ALICE".to_string(), "ALICE SMITH".to_string()); + assert_eq!(resolve_person_alias("ALICE", &known), "ALICE SMITH"); + assert_eq!(resolve_person_alias("BOB", &known), "BOB"); + } + + #[test] + fn add_entity_tracks_highest_confidence_and_person_aliases() { + let mut acc = ExtractionAccumulator::default(); + let first = acc.add_entity("Alice Smith", "PERSON", 0.6).unwrap(); + let second = acc.add_entity("Alice Smith", "PERSON", 0.9).unwrap(); + assert_eq!(first, "ALICE SMITH"); + assert_eq!(second, "ALICE SMITH"); + assert_eq!(acc.entities["ALICE SMITH"].confidence, 0.9); + assert_eq!( + acc.known_people.get("ALICE").map(String::as_str), + Some("ALICE SMITH") + ); + } + + #[test] + fn add_relation_rejects_invalid_or_self_relations() { + let mut acc = ExtractionAccumulator::default(); + acc.add_relation( + "Alice", + "PERSON", + "owns", + "Alice", + "PERSON", + 0.8, + 0, + 0, + Map::new(), + ); + assert!(acc.relations.is_empty(), "self relation should be dropped"); + + acc.add_relation( + "Alice", + "PERSON", + "unknown_predicate", + "Project X", + "PROJECT", + 0.8, + 0, + 0, + Map::new(), + ); + assert!( + acc.relations.is_empty(), + "unknown predicate should be ignored" + ); + } + + #[test] + fn add_relation_canonicalizes_predicate_and_collects_chunk_index() { + let mut acc = ExtractionAccumulator::default(); + acc.add_relation( + "Alice", + "PERSON", + "works_on", + "Phoenix", + "PROJECT", + 0.8, + 3, + 11, + Map::new(), + ); + assert_eq!(acc.relations.len(), 1); + let relation = &acc.relations[0]; + assert_eq!(relation.predicate, "OWNS"); + assert_eq!(relation.subject, "ALICE"); + assert_eq!(relation.object, "PHOENIX"); + assert!(relation.chunk_indexes.contains(&3)); + assert_eq!(relation.order_index, 11); + } +} diff --git a/src/openhuman/memory/ingestion/types.rs b/src/openhuman/memory/ingestion/types.rs index 1c1a79794..1f65b4622 100644 --- a/src/openhuman/memory/ingestion/types.rs +++ b/src/openhuman/memory/ingestion/types.rs @@ -5,7 +5,7 @@ use std::collections::{BTreeSet, HashMap}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; -use crate::openhuman::memory::store::types::NamespaceDocumentInput; +use crate::openhuman::memory_store::types::NamespaceDocumentInput; /// Default extraction backend label reported in ingestion metadata. pub const DEFAULT_MEMORY_EXTRACTION_MODEL: &str = "heuristic-only"; @@ -243,3 +243,45 @@ pub(super) struct ParsedIngestion { pub(super) preference_count: usize, pub(super) decision_count: usize, } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn extraction_mode_default_is_sentence() { + assert_eq!(ExtractionMode::default(), ExtractionMode::Sentence); + assert_eq!(ExtractionMode::Sentence.as_str(), "sentence"); + assert_eq!(ExtractionMode::Chunk.as_str(), "chunk"); + } + + #[test] + fn memory_ingestion_config_default_matches_expected_thresholds() { + let cfg = MemoryIngestionConfig::default(); + assert_eq!(cfg.model_name, DEFAULT_MEMORY_EXTRACTION_MODEL); + assert_eq!(cfg.extraction_mode, ExtractionMode::Sentence); + assert_eq!(cfg.entity_threshold, 0.45); + assert_eq!(cfg.relation_threshold, 0.30); + assert_eq!(cfg.adjacency_threshold, 0.50); + assert_eq!(cfg.batch_size, 16); + } + + #[test] + fn memory_ingestion_request_defaults_config_when_absent() { + let request: MemoryIngestionRequest = serde_json::from_value(json!({ + "document": { + "namespace": "global", + "key": "doc-1", + "title": "Doc", + "content": "Body", + "source_type": "manual", + "priority": "normal", + "category": "core" + } + })) + .unwrap(); + assert_eq!(request.config.model_name, DEFAULT_MEMORY_EXTRACTION_MODEL); + assert_eq!(request.config.extraction_mode, ExtractionMode::Sentence); + } +} diff --git a/src/openhuman/memory/mod.rs b/src/openhuman/memory/mod.rs index 64e73ea41..8cf55dab1 100644 --- a/src/openhuman/memory/mod.rs +++ b/src/openhuman/memory/mod.rs @@ -1,24 +1,56 @@ -//! Memory system for OpenHuman. +//! Memory orchestration. //! -//! This module provides the core abstractions and implementations for the memory system, -//! including semantic search, ingestion pipelines, document management, and knowledge graph -//! operations. It integrates vector search, keyword search, and relational data to provide -//! a unified memory interface for AI agents. +//! This module is the high-level routing + policy layer over the memory +//! stack. Owns the ingest pipeline, background job handlers, scoring, +//! tree-building policy (tree_global / tree_topic), recall ranking, and +//! the RPC surface. Storage primitives all live in sibling memory_* +//! modules — see [`README.md`](README.md) for the full map. +//! +//! No SQLite, no on-disk md, no vector tables here — those belong one +//! layer down in [`memory_store`](crate::openhuman::memory_store). -pub mod chunker; -pub mod conversations; +// Legacy memory modules pub mod global; pub mod ingestion; pub mod ops; pub mod preferences; pub mod rpc_models; -pub mod safety; pub mod schemas; pub mod stm_recall; -pub mod store; -pub mod sync_status; -pub mod tool_memory; pub mod traits; + +// Tool-scoped memory moved to top-level `memory_tools`. Re-exported here so +// existing `memory::tool_memory::*` paths still resolve during the migration. +pub use crate::openhuman::memory_tools as tool_memory; + +// Modules moved from memory_tree (Phase 3) +pub mod chat; +pub mod ingest_pipeline; +pub mod read_rpc; +pub mod retrieval; +pub mod schema; +pub mod score; +pub mod tree_rpc; +pub mod util; + +// Conversation storage moved to top-level `memory_conversations`. Re-exported +// here so existing `memory::conversations::*` paths still resolve during the +// migration. +pub use crate::openhuman::memory_conversations as conversations; +// Async memory job queue moved to top-level `memory_queue`. Re-exported here +// so existing `memory::jobs::*` paths still resolve during the migration. +pub use crate::openhuman::memory_queue as jobs; + +pub use crate::openhuman::memory_store::{ + create_memory, create_memory_for_migration, create_memory_with_local_ai, + effective_embedding_settings, effective_memory_backend_name, MemoryClient, MemoryClientRef, + MemoryItemKind, MemoryState, NamespaceDocumentInput, NamespaceMemoryHit, NamespaceQueryResult, + NamespaceRetrievalContext, RetrievalScoreBreakdown, UnifiedMemory, +}; +pub use crate::openhuman::memory_sync::sync_status::{ + all_memory_sync_status_controller_schemas, all_memory_sync_status_registered_controllers, + FreshnessLabel, MemorySyncStatus, +}; pub use ingestion::{ ExtractedEntity, ExtractedRelation, ExtractionMode, IngestionJob, IngestionQueue, IngestionState, IngestionStatusSnapshot, MemoryIngestionConfig, MemoryIngestionRequest, @@ -31,18 +63,6 @@ pub use schemas::{ all_controller_schemas as all_memory_controller_schemas, all_registered_controllers as all_memory_registered_controllers, }; -pub use store::{ - create_memory, create_memory_for_migration, create_memory_with_local_ai, - create_memory_with_storage, create_memory_with_storage_and_routes, - effective_embedding_settings, effective_embedding_settings_probed, - effective_memory_backend_name, MemoryClient, MemoryClientRef, MemoryItemKind, MemoryState, - NamespaceDocumentInput, NamespaceMemoryHit, NamespaceQueryResult, NamespaceRetrievalContext, - RetrievalScoreBreakdown, UnifiedMemory, -}; -pub use sync_status::{ - all_memory_sync_status_controller_schemas, all_memory_sync_status_registered_controllers, - FreshnessLabel, MemorySyncStatus, -}; pub use tool_memory::{ render_tool_memory_rules, tool_memory_namespace, ToolMemoryCaptureHook, ToolMemoryPriority, ToolMemoryRule, ToolMemoryRulesSection, ToolMemorySource, ToolMemoryStore, TOOL_MEMORY_HEADING, diff --git a/src/openhuman/memory/ops/documents.rs b/src/openhuman/memory/ops/documents.rs index 215eec804..88b461013 100644 --- a/src/openhuman/memory/ops/documents.rs +++ b/src/openhuman/memory/ops/documents.rs @@ -489,3 +489,222 @@ pub async fn memory_recall_memories( Err(message) => Ok(error_envelope("memory.recall_memories_failed", message)), } } + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + use std::sync::OnceLock; + + use serde_json::json; + use tempfile::TempDir; + + use super::*; + + fn ensure_memory_client() { + static WORKSPACE: OnceLock = OnceLock::new(); + let workspace = WORKSPACE.get_or_init(|| { + let tmp = TempDir::new().expect("tempdir"); + let path = tmp.path().join("workspace"); + std::fs::create_dir_all(&path).expect("workspace dir"); + std::mem::forget(tmp); + path + }); + let _ = crate::openhuman::memory::global::init(workspace.clone()); + } + + fn unique_namespace(prefix: &str) -> String { + format!("{prefix}-{}", uuid::Uuid::new_v4()) + } + + fn sample_put(namespace: String, key: String, title: &str, content: &str) -> PutDocParams { + PutDocParams { + namespace, + key, + title: title.into(), + content: content.into(), + source_type: default_source_type(), + priority: default_priority(), + tags: vec!["test".into()], + metadata: json!({"source": "test"}), + category: default_category(), + session_id: Some("session-docs".into()), + document_id: None, + } + } + + #[tokio::test] + async fn direct_document_handlers_roundtrip_through_namespace() { + ensure_memory_client(); + let namespace = unique_namespace("memory-docs-direct"); + let key = format!("note-{}", uuid::Uuid::new_v4()); + + let put = doc_put(sample_put( + namespace.clone(), + key.clone(), + "Rust ownership", + "Ownership and borrowing let Rust enforce memory safety.", + )) + .await + .expect("doc_put"); + let document_id = put.value.document_id.clone(); + assert!(!document_id.is_empty()); + + let listed = doc_list(Some(NamespaceOnlyParams { + namespace: namespace.clone(), + })) + .await + .expect("doc_list"); + let docs = listed + .value + .get("documents") + .and_then(|v| v.as_array()) + .expect("documents array"); + assert!(docs.iter().any(|doc| doc["key"] == key)); + + let queried = context_query(QueryNamespaceParams { + namespace: namespace.clone(), + query: "ownership".into(), + limit: Some(5), + }) + .await + .expect("context_query"); + assert!( + queried.value.to_lowercase().contains("ownership"), + "query result should mention the stored concept" + ); + + let recalled = context_recall(RecallNamespaceParams { + namespace: namespace.clone(), + limit: Some(5), + }) + .await + .expect("context_recall"); + assert!(recalled.value.is_some()); + + let deleted = doc_delete(DeleteDocParams { + namespace: namespace.clone(), + document_id: document_id.clone(), + }) + .await + .expect("doc_delete"); + assert_eq!(deleted.logs, vec!["memory document deleted".to_string()]); + + let deleted_flag = deleted + .value + .get("deleted") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + assert!(deleted_flag, "delete result should report success"); + + let after = doc_list(Some(NamespaceOnlyParams { namespace })) + .await + .expect("doc_list after delete"); + let after_docs = after + .value + .get("documents") + .and_then(|v| v.as_array()) + .expect("documents array after delete"); + assert!(after_docs.is_empty()); + } + + #[tokio::test] + async fn envelope_memory_handlers_report_counts_and_statuses() { + ensure_memory_client(); + let namespace = unique_namespace("memory-docs-envelope"); + let key = format!("env-{}", uuid::Uuid::new_v4()); + + let _ = memory_init(MemoryInitRequest { jwt_token: None }) + .await + .expect("memory_init"); + + let direct = doc_put(sample_put( + namespace.clone(), + key.clone(), + "Borrow checker", + "The borrow checker enforces aliasing and mutation rules.", + )) + .await + .expect("seed document"); + let document_id = direct.value.document_id; + + let listed = memory_list_documents(ListDocumentsRequest { + namespace: Some(namespace.clone()), + }) + .await + .expect("memory_list_documents"); + let listed_data = listed.value.data.expect("list envelope data"); + assert_eq!(listed_data.count, 1); + assert_eq!(listed_data.documents[0].key, key); + assert_eq!( + listed + .value + .meta + .counts + .as_ref() + .and_then(|m| m.get("num_documents")), + Some(&1) + ); + + let namespaces = memory_list_namespaces(EmptyRequest {}) + .await + .expect("memory_list_namespaces"); + let namespace_data = namespaces.value.data.expect("namespace data"); + assert!( + namespace_data.namespaces.iter().any(|ns| ns == &namespace), + "expected namespace list to include the seeded namespace" + ); + + let queried = memory_query_namespace(QueryNamespaceRequest { + namespace: namespace.clone(), + query: "borrow checker".into(), + limit: Some(5), + max_chunks: None, + include_references: Some(true), + document_ids: None, + }) + .await + .expect("memory_query_namespace"); + let query_data = queried.value.data.expect("query data"); + assert!(query_data.llm_context_message.is_some()); + assert!(query_data.context.is_some()); + + let recalled = memory_recall_memories(RecallMemoriesRequest { + namespace: namespace.clone(), + min_retention: None, + as_of: None, + limit: Some(5), + max_chunks: None, + top_k: None, + }) + .await + .expect("memory_recall_memories"); + let recall_data = recalled.value.data.expect("recall data"); + assert_eq!(recall_data.memories.len(), 1); + assert_eq!(recall_data.memories[0].kind, "document"); + + let deleted = memory_delete_document(DeleteDocumentRequest { + namespace: namespace.clone(), + document_id, + }) + .await + .expect("memory_delete_document"); + let deleted_data = deleted.value.data.expect("delete envelope data"); + assert_eq!(deleted_data.status, "completed"); + assert!(deleted_data.deleted); + + let cleared = clear_namespace(ClearNamespaceParams { + namespace: namespace.clone(), + }) + .await + .expect("clear_namespace"); + assert!(cleared.value.cleared); + + let listed_after = memory_list_documents(ListDocumentsRequest { + namespace: Some(namespace), + }) + .await + .expect("memory_list_documents after clear"); + let after_data = listed_after.value.data.expect("after clear data"); + assert_eq!(after_data.count, 0); + } +} diff --git a/src/openhuman/memory/ops/envelope.rs b/src/openhuman/memory/ops/envelope.rs index 265888853..7031483b1 100644 --- a/src/openhuman/memory/ops/envelope.rs +++ b/src/openhuman/memory/ops/envelope.rs @@ -80,3 +80,34 @@ pub(crate) fn error_envelope( vec![], ) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn memory_counts_collects_named_entries() { + let counts = memory_counts([("documents", 2), ("entities", 5)]); + assert_eq!(counts.get("documents"), Some(&2)); + assert_eq!(counts.get("entities"), Some(&5)); + } + + #[test] + fn envelope_wraps_data_with_meta() { + let outcome = envelope(serde_json::json!({"ok": true}), None, None); + let value = outcome.into_cli_compatible_json().unwrap(); + assert_eq!(value["data"]["ok"], true); + assert!(value["meta"]["request_id"].as_str().is_some()); + assert!(value["error"].is_null()); + } + + #[test] + fn error_envelope_wraps_code_and_message() { + let outcome: RpcOutcome> = + error_envelope("bad_request", "boom".to_string()); + let value = outcome.into_cli_compatible_json().unwrap(); + assert_eq!(value["error"]["code"], "bad_request"); + assert_eq!(value["error"]["message"], "boom"); + assert!(value["data"].is_null()); + } +} diff --git a/src/openhuman/memory/ops/files.rs b/src/openhuman/memory/ops/files.rs index 91452d086..f268c5dd5 100644 --- a/src/openhuman/memory/ops/files.rs +++ b/src/openhuman/memory/ops/files.rs @@ -102,3 +102,280 @@ pub async fn ai_write_memory_file( None, )) } + +#[cfg(test)] +mod tests { + use std::ffi::OsString; + use std::sync::{Mutex, OnceLock}; + + use tempfile::TempDir; + + use super::*; + + fn env_mutex() -> &'static Mutex<()> { + static ENV_MUTEX: OnceLock> = OnceLock::new(); + ENV_MUTEX.get_or_init(|| Mutex::new(())) + } + + struct WorkspaceEnvGuard { + previous: Option, + } + + impl WorkspaceEnvGuard { + fn set(path: &std::path::Path) -> Self { + let previous = std::env::var_os("OPENHUMAN_WORKSPACE"); + unsafe { + std::env::set_var("OPENHUMAN_WORKSPACE", path); + } + Self { previous } + } + } + + impl Drop for WorkspaceEnvGuard { + fn drop(&mut self) { + unsafe { + if let Some(previous) = self.previous.take() { + std::env::set_var("OPENHUMAN_WORKSPACE", previous); + } else { + std::env::remove_var("OPENHUMAN_WORKSPACE"); + } + } + } + } + + #[tokio::test] + async fn write_read_and_list_memory_files_roundtrip() { + let _guard = env_mutex() + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let tmp = TempDir::new().expect("tempdir"); + let _workspace = WorkspaceEnvGuard::set(tmp.path()); + + let write = ai_write_memory_file(WriteMemoryFileRequest { + relative_path: "notes/today.md".to_string(), + content: "remember this".to_string(), + }) + .await + .expect("write should succeed"); + let write_data = write.value.data.expect("write data"); + assert_eq!(write_data.relative_path, "notes/today.md"); + assert!(write_data.written); + assert_eq!(write_data.bytes_written, "remember this".len()); + + let read = ai_read_memory_file(ReadMemoryFileRequest { + relative_path: "notes/today.md".to_string(), + }) + .await + .expect("read should succeed"); + let read_data = read.value.data.expect("read data"); + assert_eq!(read_data.relative_path, "notes/today.md"); + assert_eq!(read_data.content, "remember this"); + + let memory_root = super::super::helpers::resolve_existing_memory_path("") + .await + .expect("resolve memory root"); + tokio::fs::write(memory_root.join("b.md"), "b") + .await + .expect("write b"); + tokio::fs::write(memory_root.join("a.md"), "a") + .await + .expect("write a"); + tokio::fs::create_dir_all(memory_root.join("nested")) + .await + .expect("create nested dir"); + tokio::fs::write(memory_root.join("nested").join("hidden.md"), "hidden") + .await + .expect("write nested file"); + + let listed = ai_list_memory_files(ListMemoryFilesRequest { + relative_dir: String::new(), + }) + .await + .expect("list should succeed"); + let listed_data = listed.value.data.expect("list data"); + assert_eq!(listed_data.relative_dir, ""); + assert_eq!(listed_data.files, vec!["a.md", "b.md"]); + assert_eq!(listed_data.count, 2); + } + + #[tokio::test] + async fn list_memory_files_rejects_non_directory_target() { + let _guard = env_mutex() + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let tmp = TempDir::new().expect("tempdir"); + let _workspace = WorkspaceEnvGuard::set(tmp.path()); + + tokio::fs::create_dir_all(tmp.path().join("memory")) + .await + .expect("create memory root"); + tokio::fs::write(tmp.path().join("memory").join("single.md"), "hello") + .await + .expect("write file"); + + let err = ai_list_memory_files(ListMemoryFilesRequest { + relative_dir: "single.md".to_string(), + }) + .await + .expect_err("listing a file path should fail"); + assert!( + err.contains("memory directory not found") || err.contains("resolve memory path"), + "unexpected error: {err}" + ); + } + + #[tokio::test] + async fn read_and_write_memory_files_reject_path_traversal() { + let _guard = env_mutex() + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let tmp = TempDir::new().expect("tempdir"); + let _workspace = WorkspaceEnvGuard::set(tmp.path()); + + let write_err = ai_write_memory_file(WriteMemoryFileRequest { + relative_path: "../secrets.txt".to_string(), + content: "nope".to_string(), + }) + .await + .expect_err("path traversal should fail for writes"); + assert!(write_err.contains("path traversal is not allowed")); + + let read_err = ai_read_memory_file(ReadMemoryFileRequest { + relative_path: "../secrets.txt".to_string(), + }) + .await + .expect_err("path traversal should fail for reads"); + assert!(read_err.contains("path traversal is not allowed")); + } + + #[tokio::test] + async fn list_and_read_memory_files_reject_absolute_paths() { + let _guard = env_mutex() + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let tmp = TempDir::new().expect("tempdir"); + let _workspace = WorkspaceEnvGuard::set(tmp.path()); + + let list_err = ai_list_memory_files(ListMemoryFilesRequest { + relative_dir: "/tmp".to_string(), + }) + .await + .expect_err("absolute list path should fail"); + assert!(list_err.contains("absolute paths are not allowed")); + + let read_err = ai_read_memory_file(ReadMemoryFileRequest { + relative_path: "/tmp/secret.txt".to_string(), + }) + .await + .expect_err("absolute read path should fail"); + assert!(read_err.contains("absolute paths are not allowed")); + + let write_err = ai_write_memory_file(WriteMemoryFileRequest { + relative_path: "/tmp/secret.txt".to_string(), + content: "nope".to_string(), + }) + .await + .expect_err("absolute write path should fail"); + assert!(write_err.contains("absolute paths are not allowed")); + } + + #[tokio::test] + async fn read_memory_file_surfaces_missing_file_error() { + let _guard = env_mutex() + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let tmp = TempDir::new().expect("tempdir"); + let _workspace = WorkspaceEnvGuard::set(tmp.path()); + + let err = ai_read_memory_file(ReadMemoryFileRequest { + relative_path: "missing.md".to_string(), + }) + .await + .expect_err("missing file should fail"); + assert!( + err.contains("resolve memory path") || err.contains("read memory file"), + "unexpected error: {err}" + ); + } + + #[tokio::test] + async fn read_memory_file_surfaces_invalid_utf8_error() { + let _guard = env_mutex() + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let tmp = TempDir::new().expect("tempdir"); + let _workspace = WorkspaceEnvGuard::set(tmp.path()); + + let memory_root = super::super::helpers::resolve_existing_memory_path("") + .await + .expect("resolve memory root"); + tokio::fs::write(memory_root.join("binary.bin"), [0xff, 0xfe, 0xfd]) + .await + .expect("write invalid utf8 file"); + + let err = ai_read_memory_file(ReadMemoryFileRequest { + relative_path: "binary.bin".to_string(), + }) + .await + .expect_err("invalid utf8 file should fail"); + assert!(err.contains("read memory file")); + } + + #[cfg(unix)] + #[tokio::test] + async fn write_memory_file_rejects_symlink_targets() { + use std::os::unix::fs::symlink; + + let _guard = env_mutex() + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let tmp = TempDir::new().expect("tempdir"); + let _workspace = WorkspaceEnvGuard::set(tmp.path()); + + let memory_root = super::super::helpers::resolve_existing_memory_path("") + .await + .expect("resolve memory root"); + let real = memory_root.join("real.md"); + tokio::fs::write(&real, "hello") + .await + .expect("write real file"); + symlink(&real, memory_root.join("alias.md")).expect("create symlink"); + + let err = ai_write_memory_file(WriteMemoryFileRequest { + relative_path: "alias.md".to_string(), + content: "mutate".to_string(), + }) + .await + .expect_err("writing through symlink should fail"); + assert!(err.contains("refusing to write through symlink")); + } + + #[cfg(unix)] + #[tokio::test] + async fn list_memory_files_skips_symlink_entries() { + use std::os::unix::fs::symlink; + + let _guard = env_mutex() + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let tmp = TempDir::new().expect("tempdir"); + let _workspace = WorkspaceEnvGuard::set(tmp.path()); + + let memory_root = super::super::helpers::resolve_existing_memory_path("") + .await + .expect("resolve memory root"); + let real = memory_root.join("real.md"); + tokio::fs::write(&real, "hello") + .await + .expect("write real file"); + symlink(&real, memory_root.join("alias.md")).expect("create symlink"); + + let listed = ai_list_memory_files(ListMemoryFilesRequest { + relative_dir: String::new(), + }) + .await + .expect("list should succeed"); + let listed_data = listed.value.data.expect("list data"); + assert_eq!(listed_data.files, vec!["real.md"]); + } +} diff --git a/src/openhuman/memory/ops/helpers.rs b/src/openhuman/memory/ops/helpers.rs index 45037a00b..61387dc07 100644 --- a/src/openhuman/memory/ops/helpers.rs +++ b/src/openhuman/memory/ops/helpers.rs @@ -9,12 +9,12 @@ use serde::Deserialize; use serde_json::{json, Value}; use crate::openhuman::config::Config; -use crate::openhuman::memory::store::GraphRelationRecord; use crate::openhuman::memory::{ MemoryClient, MemoryClientRef, MemoryDocumentSummary, MemoryItemKind, MemoryRetrievalChunk, MemoryRetrievalContext, MemoryRetrievalEntity, MemoryRetrievalRelation, NamespaceMemoryHit, QueryNamespaceRequest, }; +use crate::openhuman::memory_store::GraphRelationRecord; // --------------------------------------------------------------------------- // Formatting helpers @@ -225,6 +225,80 @@ pub(crate) fn format_llm_context_message( Some(parts.join("\n\n")) } +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::memory::RetrievalScoreBreakdown; + + fn sample_hit(kind: MemoryItemKind) -> NamespaceMemoryHit { + NamespaceMemoryHit { + id: "hit-1".into(), + kind, + namespace: "global".into(), + key: "note-1".into(), + title: Some("Title".into()), + content: "Body text".into(), + category: "core".into(), + source_type: Some("manual".into()), + updated_at: 1.5, + score: 0.7, + score_breakdown: RetrievalScoreBreakdown::default(), + document_id: Some("doc-1".into()), + chunk_id: Some("chunk-1".into()), + supporting_relations: vec![GraphRelationRecord { + namespace: Some("global".into()), + subject: "Alice".into(), + predicate: "OWNS".into(), + object: "OpenHuman".into(), + attrs: json!({"entity_types": {"subject": "PERSON", "object": "PRODUCT"}}), + updated_at: 2.0, + evidence_count: 1, + order_index: Some(0), + document_ids: vec!["doc-1".into()], + chunk_ids: vec!["chunk-1".into()], + }], + } + } + + #[test] + fn timestamp_to_rfc3339_rejects_invalid_values() { + assert!(timestamp_to_rfc3339(f64::NAN).is_none()); + assert!(timestamp_to_rfc3339(f64::INFINITY).is_none()); + assert!(timestamp_to_rfc3339(-1.0).is_none()); + assert!(timestamp_to_rfc3339(1.5).is_some()); + } + + #[test] + fn relation_identity_and_metadata_include_namespace_and_attrs() { + let relation = sample_hit(MemoryItemKind::Document) + .supporting_relations + .remove(0); + assert_eq!(relation_identity(&relation), "global|Alice|OWNS|OpenHuman"); + let meta = relation_metadata(&relation); + assert_eq!(meta["namespace"], "global"); + assert_eq!(meta["attrs"]["entity_types"]["subject"], "PERSON"); + } + + #[test] + fn build_retrieval_context_deduplicates_relations_and_entities() { + let hit = sample_hit(MemoryItemKind::Document); + let ctx = build_retrieval_context(&[hit.clone(), hit]); + assert_eq!(ctx.chunks.len(), 2); + assert_eq!(ctx.relations.len(), 1); + assert!(ctx.entities.iter().any(|e| e.name == "Alice")); + assert!(ctx.entities.iter().any(|e| e.name == "OpenHuman")); + } + + #[test] + fn format_llm_context_message_includes_query_and_relation_text() { + let hit = sample_hit(MemoryItemKind::Document); + let text = format_llm_context_message(Some("who owns it"), &[hit]).unwrap(); + assert!(text.contains("Query: who owns it")); + assert!(text.contains("Title: Body text")); + assert!(text.contains("Alice (PERSON) -[OWNS]-> OpenHuman (PRODUCT)")); + } +} + /// Filters memory hits to only include those matching specific document IDs. pub(crate) fn filter_hits_by_document_ids( hits: Vec, diff --git a/src/openhuman/memory/ops/kv_graph.rs b/src/openhuman/memory/ops/kv_graph.rs index 3992288fb..0551f6662 100644 --- a/src/openhuman/memory/ops/kv_graph.rs +++ b/src/openhuman/memory/ops/kv_graph.rs @@ -134,3 +134,111 @@ pub async fn graph_query( .await?; Ok(RpcOutcome::single_log(rows, "memory graph queried")) } + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + use std::sync::OnceLock; + + use tempfile::TempDir; + + use super::*; + + fn ensure_memory_client() { + static WORKSPACE: OnceLock = OnceLock::new(); + let workspace = WORKSPACE.get_or_init(|| { + let tmp = TempDir::new().expect("tempdir"); + let path = tmp.path().join("workspace"); + std::fs::create_dir_all(&path).expect("workspace dir"); + std::mem::forget(tmp); + path + }); + let _ = crate::openhuman::memory::global::init(workspace.clone()); + } + + fn unique_namespace(prefix: &str) -> String { + format!("{prefix}-{}", uuid::Uuid::new_v4()) + } + + #[tokio::test] + async fn kv_handlers_roundtrip_scoped_values() { + ensure_memory_client(); + let namespace = unique_namespace("kv-graph-kv"); + let key = format!("state-{}", uuid::Uuid::new_v4()); + + let set = kv_set(KvSetParams { + namespace: Some(namespace.clone()), + key: key.clone(), + value: serde_json::json!({"open": true}), + }) + .await + .expect("kv set"); + assert!(set.value); + + let get = kv_get(KvGetDeleteParams { + namespace: Some(namespace.clone()), + key: key.clone(), + }) + .await + .expect("kv get"); + assert_eq!(get.value, Some(serde_json::json!({"open": true}))); + + let listed = kv_list_namespace(super::super::documents::NamespaceOnlyParams { + namespace: namespace.clone(), + }) + .await + .expect("kv list namespace"); + assert!(listed + .value + .iter() + .any(|row| row["key"] == key && row["value"] == serde_json::json!({"open": true}))); + + let deleted = kv_delete(KvGetDeleteParams { + namespace: Some(namespace.clone()), + key: key.clone(), + }) + .await + .expect("kv delete"); + assert!(deleted.value); + + let after = kv_get(KvGetDeleteParams { + namespace: Some(namespace), + key, + }) + .await + .expect("kv get after delete"); + assert!(after.value.is_none()); + } + + #[tokio::test] + async fn graph_handlers_roundtrip_relation_rows() { + ensure_memory_client(); + let namespace = unique_namespace("kv-graph-rel"); + let subject = format!("alice-{}", uuid::Uuid::new_v4()); + + let upsert = graph_upsert(GraphUpsertParams { + namespace: Some(namespace.clone()), + subject: subject.clone(), + predicate: "OWNS".into(), + object: "Atlas".into(), + attrs: serde_json::json!({"source": "test", "confidence": 0.9}), + }) + .await + .expect("graph upsert"); + assert!(upsert.value); + + let queried = graph_query(GraphQueryParams { + namespace: Some(namespace), + subject: Some(subject.clone()), + predicate: Some("OWNS".into()), + }) + .await + .expect("graph query"); + + assert_eq!(queried.logs, vec!["memory graph queried".to_string()]); + assert_eq!(queried.value.len(), 1); + assert_eq!(queried.value[0]["subject"], subject.to_uppercase()); + assert_eq!(queried.value[0]["predicate"], "OWNS"); + assert_eq!(queried.value[0]["object"], "ATLAS"); + } +} diff --git a/src/openhuman/memory/ops/learn.rs b/src/openhuman/memory/ops/learn.rs index 5c188ba4b..79b470e95 100644 --- a/src/openhuman/memory/ops/learn.rs +++ b/src/openhuman/memory/ops/learn.rs @@ -111,9 +111,10 @@ pub async fn memory_learn_all( "[memory.learn] running summarization for namespace='{}'", namespace ); - let outcome = - crate::openhuman::memory_tree::summarizer::ops::tree_summarizer_run(&config, namespace) - .await; + let outcome = crate::openhuman::memory_tree::tree_runtime::ops::tree_summarizer_run( + &config, namespace, + ) + .await; match outcome { Ok(_) => { tracing::info!("[memory.learn] namespace='{}' ok", namespace); @@ -151,3 +152,192 @@ pub async fn memory_learn_all( vec![], )) } + +#[cfg(test)] +mod tests { + use std::ffi::OsString; + use std::path::PathBuf; + use std::sync::OnceLock; + + use serde_json::json; + use tempfile::TempDir; + + use super::*; + use crate::openhuman::memory_store::NamespaceDocumentInput; + + fn ensure_memory_client() { + static WORKSPACE: OnceLock = OnceLock::new(); + let workspace = WORKSPACE.get_or_init(|| { + let tmp = TempDir::new().expect("tempdir"); + let path = tmp.path().join("workspace"); + std::fs::create_dir_all(&path).expect("workspace dir"); + std::mem::forget(tmp); + path + }); + let _ = crate::openhuman::memory::global::init(workspace.clone()); + } + + struct WorkspaceEnvGuard { + _lock: std::sync::MutexGuard<'static, ()>, + previous: Option, + } + + impl WorkspaceEnvGuard { + fn set(path: &std::path::Path) -> Self { + let lock = crate::openhuman::config::TEST_ENV_LOCK.lock().unwrap(); + let previous = std::env::var_os("OPENHUMAN_WORKSPACE"); + std::env::set_var("OPENHUMAN_WORKSPACE", path); + Self { + _lock: lock, + previous, + } + } + } + + impl Drop for WorkspaceEnvGuard { + fn drop(&mut self) { + if let Some(previous) = self.previous.as_ref() { + std::env::set_var("OPENHUMAN_WORKSPACE", previous); + } else { + std::env::remove_var("OPENHUMAN_WORKSPACE"); + } + } + } + + async fn seed_namespace(prefix: &str) -> String { + ensure_memory_client(); + let namespace = format!("{prefix}-{}", uuid::Uuid::new_v4()); + let client = crate::openhuman::memory::global::client().expect("memory client"); + client + .put_doc_light(NamespaceDocumentInput { + namespace: namespace.clone(), + key: format!("key-{}", uuid::Uuid::new_v4()), + title: "Test".into(), + content: "Seed content".into(), + source_type: "doc".into(), + priority: "normal".into(), + tags: vec!["test".into()], + metadata: json!({"source": "test"}), + category: "core".into(), + session_id: None, + document_id: None, + }) + .await + .expect("seed namespace doc"); + namespace + } + + async fn write_config_with_runtime_enabled( + workspace_root: &std::path::Path, + runtime_enabled: bool, + ) { + let _guard = WorkspaceEnvGuard::set(workspace_root); + let mut config = crate::openhuman::config::Config::load_or_init() + .await + .expect("load config"); + config.local_ai.runtime_enabled = runtime_enabled; + config.save().await.expect("save config"); + } + + #[tokio::test] + async fn memory_learn_all_is_noop_for_explicit_empty_namespace_list() { + ensure_memory_client(); + let outcome = memory_learn_all(LearnAllParams { + namespaces: Some(vec![]), + }) + .await + .expect("empty list should early-return"); + assert_eq!(outcome.value.namespaces_processed, 0); + assert!(outcome.value.results.is_empty()); + assert!(outcome.logs.is_empty()); + } + + #[tokio::test] + async fn memory_learn_all_is_noop_when_requested_namespaces_do_not_exist() { + ensure_memory_client(); + let missing = format!("missing-{}", uuid::Uuid::new_v4()); + let outcome = memory_learn_all(LearnAllParams { + namespaces: Some(vec![missing]), + }) + .await + .expect("unknown namespaces should filter to no-op"); + assert_eq!(outcome.value.namespaces_processed, 0); + assert!(outcome.value.results.is_empty()); + } + + #[tokio::test] + async fn memory_learn_all_filters_missing_namespaces_and_dedupes_requested_order() { + let namespace_a = seed_namespace("memory-learn-a").await; + let namespace_b = seed_namespace("memory-learn-b").await; + let missing = format!("missing-{}", uuid::Uuid::new_v4()); + let tmp = TempDir::new().expect("tempdir"); + write_config_with_runtime_enabled(tmp.path(), true).await; + let _workspace = WorkspaceEnvGuard::set(tmp.path()); + + let outcome = memory_learn_all(LearnAllParams { + namespaces: Some(vec![ + missing, + namespace_b.clone(), + namespace_a.clone(), + namespace_b.clone(), + ]), + }) + .await + .expect("existing namespaces with runtime enabled should run"); + + assert_eq!(outcome.value.namespaces_processed, 2); + assert_eq!(outcome.value.results.len(), 2); + assert_eq!(outcome.value.results[0].namespace, namespace_b); + assert_eq!(outcome.value.results[1].namespace, namespace_a); + assert!(outcome.value.results.iter().all(|r| r.status == "ok")); + assert!(outcome.value.results.iter().all(|r| r.error.is_none())); + } + + #[tokio::test] + async fn memory_learn_all_requires_local_ai_once_existing_namespace_is_selected() { + let namespace = seed_namespace("memory-learn-runtime").await; + let tmp = TempDir::new().expect("tempdir"); + write_config_with_runtime_enabled(tmp.path(), false).await; + let _workspace = WorkspaceEnvGuard::set(tmp.path()); + + let err = memory_learn_all(LearnAllParams { + namespaces: Some(vec![namespace]), + }) + .await + .expect_err("runtime-disabled config should hard-fail"); + + assert!(err.contains("memory_learn_all requires local_ai.runtime_enabled=true")); + } + + #[tokio::test] + async fn memory_learn_all_uses_all_namespaces_when_none_is_requested() { + let namespace_a = seed_namespace("memory-learn-all-a").await; + let namespace_b = seed_namespace("memory-learn-all-b").await; + let tmp = TempDir::new().expect("tempdir"); + write_config_with_runtime_enabled(tmp.path(), true).await; + let _workspace = WorkspaceEnvGuard::set(tmp.path()); + + let outcome = memory_learn_all(LearnAllParams { namespaces: None }) + .await + .expect("runtime-enabled config should process all namespaces"); + + assert!( + outcome.value.namespaces_processed >= 2, + "expected at least the two seeded namespaces to be processed" + ); + let namespaces: std::collections::BTreeSet<_> = outcome + .value + .results + .iter() + .map(|r| r.namespace.as_str()) + .collect(); + assert!(namespaces.contains(namespace_a.as_str())); + assert!(namespaces.contains(namespace_b.as_str())); + assert!(outcome + .value + .results + .iter() + .filter(|r| r.namespace == namespace_a || r.namespace == namespace_b) + .all(|r| r.status == "ok" && r.error.is_none())); + } +} diff --git a/src/openhuman/memory/ops/sync.rs b/src/openhuman/memory/ops/sync.rs index c29542a60..65cdd88c1 100644 --- a/src/openhuman/memory/ops/sync.rs +++ b/src/openhuman/memory/ops/sync.rs @@ -110,3 +110,163 @@ pub async fn memory_ingestion_status() -> Result &'static std::sync::Mutex<()> { + static LOCK: OnceLock> = OnceLock::new(); + LOCK.get_or_init(|| std::sync::Mutex::new(())) + } + + fn ensure_memory_client() -> crate::openhuman::memory::MemoryClientRef { + static WORKSPACE: OnceLock = OnceLock::new(); + let workspace = WORKSPACE.get_or_init(|| { + let tmp = TempDir::new().expect("tempdir"); + let path = tmp.path().join("workspace"); + std::fs::create_dir_all(&path).expect("workspace dir"); + std::mem::forget(tmp); + path + }); + crate::openhuman::memory::global::init(workspace.clone()).expect("init memory client") + } + + struct ChannelCapture { + tx: mpsc::UnboundedSender>, + } + + #[async_trait] + impl EventHandler for ChannelCapture { + fn name(&self) -> &str { + "memory::ops::sync::tests::capture" + } + + fn domains(&self) -> Option<&[&str]> { + Some(&["memory"]) + } + + async fn handle(&self, event: &DomainEvent) { + if let DomainEvent::MemorySyncRequested { channel_id } = event { + let _ = self.tx.send(channel_id.clone()); + } + } + } + + #[test] + fn sync_channel_params_deserialize_channel_id() { + let params: SyncChannelParams = + serde_json::from_value(json!({"channel_id": "channel-1"})).unwrap(); + assert_eq!(params.channel_id, "channel-1"); + } + + #[test] + fn ingestion_status_result_default_is_idle() { + let status = IngestionStatusResult::default(); + assert!(!status.running); + assert!(status.current_document_id.is_none()); + assert!(status.current_title.is_none()); + assert!(status.current_namespace.is_none()); + assert_eq!(status.queue_depth, 0); + assert!(status.last_completed_at.is_none()); + assert!(status.last_document_id.is_none()); + assert!(status.last_success.is_none()); + } + + #[test] + fn sync_result_structs_serialize_expected_fields() { + let one = serde_json::to_value(SyncChannelResult { + requested: true, + channel_id: "abc".into(), + }) + .unwrap(); + assert_eq!(one, json!({"requested": true, "channel_id": "abc"})); + + let all = serde_json::to_value(SyncAllResult { requested: true }).unwrap(); + assert_eq!(all, json!({"requested": true})); + } + + #[tokio::test] + async fn memory_sync_channel_publishes_targeted_event() { + let _guard = test_mutex() + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + event_bus::init_global(event_bus::DEFAULT_CAPACITY); + let (tx, mut rx) = mpsc::unbounded_channel(); + let _subscription = event_bus::subscribe_global(Arc::new(ChannelCapture { tx })) + .expect("global bus should be initialized"); + + let outcome = memory_sync_channel(SyncChannelParams { + channel_id: "channel-123".into(), + }) + .await + .expect("memory_sync_channel"); + assert!(outcome.value.requested); + assert_eq!(outcome.value.channel_id, "channel-123"); + + let received = timeout(Duration::from_secs(1), rx.recv()) + .await + .expect("event should arrive before timeout") + .expect("sender should still be connected"); + assert_eq!(received.as_deref(), Some("channel-123")); + } + + #[tokio::test] + async fn memory_sync_all_publishes_broadcast_event() { + let _guard = test_mutex() + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + event_bus::init_global(event_bus::DEFAULT_CAPACITY); + let (tx, mut rx) = mpsc::unbounded_channel(); + let _subscription = event_bus::subscribe_global(Arc::new(ChannelCapture { tx })) + .expect("global bus should be initialized"); + + let outcome = memory_sync_all().await.expect("memory_sync_all"); + assert!(outcome.value.requested); + + let received = timeout(Duration::from_secs(1), rx.recv()) + .await + .expect("event should arrive before timeout") + .expect("sender should still be connected"); + assert!( + received.is_none(), + "sync-all should publish channel_id=None" + ); + } + + #[tokio::test] + async fn memory_ingestion_status_reflects_initialized_client_snapshot() { + let _guard = test_mutex() + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let client = ensure_memory_client(); + let state = client.ingestion_state(); + + state.enqueue(); + state.mark_running("doc-sync", "Sync Title", "sync-test"); + + let status = memory_ingestion_status() + .await + .expect("memory_ingestion_status") + .value; + + assert!(status.running); + assert_eq!(status.current_document_id.as_deref(), Some("doc-sync")); + assert_eq!(status.current_title.as_deref(), Some("Sync Title")); + assert_eq!(status.current_namespace.as_deref(), Some("sync-test")); + assert_eq!(status.queue_depth, 1); + + state.dequeue(); + state.mark_completed("doc-sync", true, 12345); + } +} diff --git a/src/openhuman/memory/ops/tool_memory.rs b/src/openhuman/memory/ops/tool_memory.rs index 495a2b0b9..e23feb706 100644 --- a/src/openhuman/memory/ops/tool_memory.rs +++ b/src/openhuman/memory/ops/tool_memory.rs @@ -172,3 +172,162 @@ pub async fn tool_rules_json(params: ToolRuleListParams) -> Result = OnceLock::new(); + let workspace = WORKSPACE.get_or_init(|| { + let tmp = TempDir::new().expect("tempdir"); + let path = tmp.path().join("workspace"); + std::fs::create_dir_all(&path).expect("workspace dir"); + std::mem::forget(tmp); + path + }); + let _ = crate::openhuman::memory::global::init(workspace.clone()); + } + + fn unique_tool_name() -> String { + format!("tool-memory-{}", uuid::Uuid::new_v4()) + } + + #[tokio::test] + async fn tool_rule_put_get_list_and_delete_roundtrip() { + ensure_memory_client(); + let tool_name = unique_tool_name(); + + let stored = tool_rule_put(ToolRulePutParams { + tool_name: tool_name.clone(), + rule: "Always ask before sending emails".into(), + priority: None, + source: None, + tags: vec!["safety".into()], + id: Some(" ".into()), + }) + .await + .expect("tool rule put") + .value; + + assert_eq!(stored.tool_name, tool_name); + assert_eq!(stored.priority, ToolMemoryPriority::Normal); + assert_eq!( + stored.source, + crate::openhuman::memory::ToolMemorySource::Programmatic + ); + assert_eq!(stored.tags, vec!["safety".to_string()]); + assert!( + !stored.id.trim().is_empty(), + "blank id should be regenerated" + ); + + let fetched = tool_rule_get(ToolRuleRefParams { + tool_name: stored.tool_name.clone(), + id: stored.id.clone(), + }) + .await + .expect("tool rule get") + .value + .expect("stored rule should exist"); + assert_eq!(fetched.rule, "Always ask before sending emails"); + + let listed = tool_rule_list(ToolRuleListParams { + tool_name: stored.tool_name.clone(), + }) + .await + .expect("tool rule list") + .value; + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].id, stored.id); + + let deleted = tool_rule_delete(ToolRuleRefParams { + tool_name: stored.tool_name.clone(), + id: stored.id.clone(), + }) + .await + .expect("tool rule delete") + .value; + assert!(deleted); + + let after = tool_rule_get(ToolRuleRefParams { + tool_name: stored.tool_name, + id: stored.id, + }) + .await + .expect("tool rule get after delete"); + assert!(after.value.is_none()); + } + + #[tokio::test] + async fn tool_rules_for_prompt_sorts_by_priority_and_tool_name() { + ensure_memory_client(); + let primary_tool = unique_tool_name(); + let secondary_tool = unique_tool_name(); + + let high = tool_rule_put(ToolRulePutParams { + tool_name: primary_tool.clone(), + rule: "Use the dry-run mode first".into(), + priority: Some(ToolMemoryPriority::High), + source: None, + tags: vec![], + id: None, + }) + .await + .expect("put high") + .value; + let normal = tool_rule_put(ToolRulePutParams { + tool_name: secondary_tool.clone(), + rule: "Log the final command".into(), + priority: Some(ToolMemoryPriority::Normal), + source: None, + tags: vec![], + id: None, + }) + .await + .expect("put normal") + .value; + + let prompt = tool_rules_for_prompt(ToolRulesForPromptParams { + tools: vec![secondary_tool.clone(), primary_tool.clone()], + }) + .await + .expect("rules for prompt") + .value; + + assert_eq!(prompt.rules.len(), 1, "only eager rules should be included"); + assert_eq!(prompt.rules[0].id, high.id); + assert!(prompt.rendered.contains(&primary_tool)); + assert!(prompt.rendered.contains("Use the dry-run mode first")); + + let json_rules = tool_rules_json(ToolRuleListParams { + tool_name: secondary_tool.clone(), + }) + .await + .expect("tool rules json") + .value; + assert!(json_rules.is_array(), "tool rules json should be an array"); + assert!(json_rules + .as_array() + .expect("array") + .iter() + .any(|row| row["rule"] == "Log the final command")); + + let _ = tool_rule_delete(ToolRuleRefParams { + tool_name: primary_tool, + id: high.id, + }) + .await; + let _ = tool_rule_delete(ToolRuleRefParams { + tool_name: secondary_tool, + id: normal.id, + }) + .await; + } +} diff --git a/src/openhuman/memory/ops_tests.rs b/src/openhuman/memory/ops_tests.rs index 75c47e6b7..464e00fe9 100644 --- a/src/openhuman/memory/ops_tests.rs +++ b/src/openhuman/memory/ops_tests.rs @@ -4,8 +4,8 @@ use serde_json::json; use super::{build_retrieval_context, filter_hits_by_document_ids, format_llm_context_message}; -use crate::openhuman::memory::store::GraphRelationRecord; use crate::openhuman::memory::{MemoryItemKind, NamespaceMemoryHit, RetrievalScoreBreakdown}; +use crate::openhuman::memory_store::GraphRelationRecord; fn sample_hit() -> NamespaceMemoryHit { NamespaceMemoryHit { diff --git a/src/openhuman/memory_tree/read_rpc.rs b/src/openhuman/memory/read_rpc.rs similarity index 87% rename from src/openhuman/memory_tree/read_rpc.rs rename to src/openhuman/memory/read_rpc.rs index 175a86cdd..42a2e37ba 100644 --- a/src/openhuman/memory_tree/read_rpc.rs +++ b/src/openhuman/memory/read_rpc.rs @@ -31,11 +31,11 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::content_store::read as content_read; -use crate::openhuman::memory_tree::retrieval::types::NodeKind; -use crate::openhuman::memory_tree::score::store as score_store; -use crate::openhuman::memory_tree::store::{self as chunk_store, with_connection}; -use crate::openhuman::memory_tree::types::SourceKind; +use crate::openhuman::memory::retrieval::types::NodeKind; +use crate::openhuman::memory::score::store as score_store; +use crate::openhuman::memory_store::chunks::store::{self as chunk_store, with_connection}; +use crate::openhuman::memory_store::chunks::types::SourceKind; +use crate::openhuman::memory_store::content::read as content_read; use crate::rpc::RpcOutcome; const PREVIEW_MAX_CHARS: usize = 500; @@ -46,7 +46,7 @@ const MAX_LIST_LIMIT: u32 = 1_000; /// Wire-shape chunk returned by the read RPCs. /// -/// Distinct from [`crate::openhuman::memory_tree::types::Chunk`] in two +/// Distinct from [`crate::openhuman::memory_store::chunks::types::Chunk`] in two /// ways: serialised timestamps are ms-since-epoch (matches the rest of the /// JSON-RPC surface) and the body is replaced with a `≤500-char preview` /// + a flag indicating whether the row has an embedding. UIs needing the @@ -462,7 +462,7 @@ pub async fn recall_rpc( // Reuse the source-tree retrieval path which already does cosine // rerank against query embeddings. We pull more summaries than `k` // because each summary expands into multiple leaves. - let resp = crate::openhuman::memory_tree::retrieval::query_source( + let resp = crate::openhuman::memory::retrieval::query_source( config, None, None, @@ -780,7 +780,7 @@ pub async fn chunk_score_rpc( ScoreBreakdown { signals, total: r.total, - threshold: crate::openhuman::memory_tree::score::DEFAULT_DROP_THRESHOLD, + threshold: crate::openhuman::memory::score::DEFAULT_DROP_THRESHOLD, kept: !r.dropped, llm_consulted, } @@ -843,7 +843,7 @@ pub async fn delete_chunk_rpc( if e.kind() != std::io::ErrorKind::NotFound { log::warn!( "[memory_tree::read::delete] failed to remove chunk file path_hash={}: {e}", - crate::openhuman::memory_tree::util::redact::redact(&rel), + crate::openhuman::memory::util::redact::redact(&rel), ); } } @@ -1001,7 +1001,7 @@ pub async fn graph_export_rpc( mode, resp.nodes.len(), resp.edges.len(), - crate::openhuman::memory_tree::util::redact::redact(&resp.content_root_abs), + crate::openhuman::memory::util::redact::redact(&resp.content_root_abs), ); Ok(RpcOutcome::single_log(resp, log)) } @@ -1371,7 +1371,7 @@ pub async fn wipe_all_rpc(config: &Config) -> Result /// keyed under [`crate::openhuman::composio::providers::sync_state::KV_NAMESPACE`]. /// /// We open the SQLite file directly rather than going through -/// [`crate::openhuman::memory::store::client::MemoryClientRef`] so +/// [`crate::openhuman::memory_store::client::MemoryClientRef`] so /// `wipe_all` stays a pure synchronous operation runnable from /// `spawn_blocking` without dragging in the full memory-store init /// path. The `kv_namespace` table is created up-front by @@ -1430,8 +1430,8 @@ pub struct ResetTreeResponse { /// outside `spawn_blocking`) so the on-disk removal can use /// async retry without blocking the worker thread. pub async fn reset_tree_rpc(config: &Config) -> Result, String> { - use crate::openhuman::memory_tree::jobs::store as jobs_store; - use crate::openhuman::memory_tree::jobs::types::{ExtractChunkPayload, NewJob}; + use crate::openhuman::memory::jobs::store as jobs_store; + use crate::openhuman::memory::jobs::types::{ExtractChunkPayload, NewJob}; let cfg = config.clone(); let (tree_rows_deleted, chunks_requeued, jobs_enqueued) = @@ -1535,7 +1535,7 @@ pub async fn reset_tree_rpc(config: &Config) -> Result` is the current 3-hour UTC block (0..=7), so /// spamming the button within the same window doesn't queue duplicates. pub async fn flush_now_rpc(config: &Config) -> Result, String> { - use crate::openhuman::memory_tree::jobs::store as jobs_store; - use crate::openhuman::memory_tree::jobs::types::{FlushStalePayload, NewJob}; - use crate::openhuman::memory_tree::tree_source::store as tree_store; + use crate::openhuman::memory::jobs::store as jobs_store; + use crate::openhuman::memory::jobs::types::{FlushStalePayload, NewJob}; + use crate::openhuman::memory_tree::tree::store as tree_store; let cfg = config.clone(); let resp = tokio::task::spawn_blocking(move || -> Result { @@ -1827,9 +1827,14 @@ fn parse_source_kind_str(s: &str) -> Option { #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage}; - use crate::openhuman::memory_tree::ingest::ingest_chat; + use crate::openhuman::composio::providers::sync_state::KV_NAMESPACE; + use crate::openhuman::embeddings::NoopEmbedding; + use crate::openhuman::memory::ingest_pipeline::ingest_chat; + use crate::openhuman::memory_store::unified::UnifiedMemory; + use crate::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage}; use chrono::{TimeZone, Utc}; + use rusqlite::params; + use std::sync::Arc; use tempfile::TempDir; fn test_config() -> (TempDir, Config) { @@ -1866,6 +1871,45 @@ mod tests { .unwrap(); } + fn update_chunk_timestamp(cfg: &Config, chunk_id: &str, timestamp_ms: i64) { + with_connection(cfg, |conn| { + conn.execute( + "UPDATE mem_tree_chunks + SET timestamp_ms = ?1, + time_range_start_ms = ?1, + time_range_end_ms = ?1 + WHERE id = ?2", + params![timestamp_ms, chunk_id], + )?; + Ok(()) + }) + .unwrap(); + } + + fn insert_raw_chunk( + cfg: &Config, + id: &str, + source_kind: &str, + source_id: &str, + timestamp_ms: i64, + tags_json: &str, + content: &str, + token_count: i64, + ) { + with_connection(cfg, |conn| { + conn.execute( + "INSERT INTO mem_tree_chunks ( + id, source_kind, source_id, source_ref, owner, timestamp_ms, + time_range_start_ms, time_range_end_ms, tags_json, content, + token_count, seq_in_source, created_at_ms, lifecycle_status, content_path + ) VALUES (?1, ?2, ?3, NULL, 'tester', ?4, ?4, ?4, ?5, ?6, ?7, 0, ?4, 'seeded', NULL)", + params![id, source_kind, source_id, timestamp_ms, tags_json, content, token_count], + )?; + Ok(()) + }) + .unwrap(); + } + #[tokio::test] async fn list_chunks_returns_seeded_chunk() { let (_tmp, cfg) = test_config(); @@ -1912,11 +1956,143 @@ mod tests { .await .unwrap() .value; - assert!(resp.chunks.iter().any(|c| c - .content_preview - .as_deref() - .unwrap_or("") - .contains("phoenix"))); + assert!(resp.chunks.iter().any(|c| { + c.content_preview + .as_deref() + .unwrap_or("") + .contains("phoenix") + })); + } + + #[tokio::test] + async fn list_chunks_filters_by_source_kind_and_applies_limit_offset() { + let (_tmp, cfg) = test_config(); + seed_chat_chunk(&cfg, "slack:#a", "first chat").await; + seed_chat_chunk(&cfg, "slack:#b", "second chat").await; + + let filtered = list_chunks_rpc( + &cfg, + ChunkFilter { + source_kinds: Some(vec!["chat".into()]), + limit: Some(1), + offset: Some(1), + ..ChunkFilter::default() + }, + ) + .await + .unwrap() + .value; + assert_eq!(filtered.chunks.len(), 1); + assert_eq!(filtered.total, 2); + assert!(filtered.chunks.iter().all(|c| c.source_kind == "chat")); + } + + #[tokio::test] + async fn list_chunks_filters_by_entity_id_and_time_window() { + let (_tmp, cfg) = test_config(); + seed_chat_chunk(&cfg, "slack:#eng", "alice@example.com handles phoenix").await; + seed_chat_chunk(&cfg, "slack:#eng", "bob@example.com handles atlas").await; + + let seeded = list_chunks_rpc(&cfg, ChunkFilter::default()) + .await + .unwrap() + .value + .chunks; + let alice = seeded + .iter() + .find(|chunk| { + chunk + .content_preview + .as_deref() + .unwrap_or("") + .contains("alice@example.com") + }) + .expect("alice chunk present"); + let bob = seeded + .iter() + .find(|chunk| { + chunk + .content_preview + .as_deref() + .unwrap_or("") + .contains("bob@example.com") + }) + .expect("bob chunk present"); + + update_chunk_timestamp(&cfg, &alice.id, 1_700_000_000_100); + update_chunk_timestamp(&cfg, &bob.id, 1_700_000_000_900); + + let filtered = list_chunks_rpc( + &cfg, + ChunkFilter { + entity_ids: Some(vec!["email:alice@example.com".into()]), + since_ms: Some(1_700_000_000_000), + until_ms: Some(1_700_000_000_500), + ..ChunkFilter::default() + }, + ) + .await + .unwrap() + .value; + + assert_eq!(filtered.total, 1); + assert_eq!(filtered.chunks.len(), 1); + assert_eq!(filtered.chunks[0].id, alice.id); + } + + #[tokio::test] + async fn list_chunks_ignores_empty_filter_lists_and_blank_query() { + let (_tmp, cfg) = test_config(); + seed_chat_chunk(&cfg, "slack:#a", "alpha").await; + seed_chat_chunk(&cfg, "slack:#b", "beta").await; + + let resp = list_chunks_rpc( + &cfg, + ChunkFilter { + source_kinds: Some(vec![]), + source_ids: Some(vec![]), + entity_ids: Some(vec![]), + query: Some(" ".into()), + limit: Some(10), + ..ChunkFilter::default() + }, + ) + .await + .unwrap() + .value; + + assert_eq!(resp.total, 2); + assert_eq!(resp.chunks.len(), 2); + } + + #[tokio::test] + async fn list_chunks_normalizes_invalid_tags_negative_tokens_and_empty_content() { + let (_tmp, cfg) = test_config(); + insert_raw_chunk( + &cfg, + "raw-empty", + "document", + "notion:page-1", + 1_700_000_000_123, + "not-json", + "", + -7, + ); + + let resp = list_chunks_rpc(&cfg, ChunkFilter::default()) + .await + .unwrap() + .value; + let row = resp + .chunks + .into_iter() + .find(|chunk| chunk.id == "raw-empty") + .expect("raw chunk listed"); + + assert_eq!(row.token_count, 0); + assert_eq!(row.tags, Vec::::new()); + assert_eq!(row.content_preview, None); + assert!(!row.has_embedding); } #[tokio::test] @@ -1938,6 +2114,33 @@ mod tests { assert_eq!(b.chunk_count, 1); } + #[tokio::test] + async fn list_sources_formats_email_threads_with_trimmed_user_hint() { + let (_tmp, cfg) = test_config(); + insert_raw_chunk( + &cfg, + "email-thread", + "email", + "gmail:Alice@Example.com|bob@example.com|carol@example.com", + 1_700_000_000_123, + "[]", + "thread body", + 12, + ); + + let sources = list_sources_rpc(&cfg, Some(" alice@example.com ".into())) + .await + .unwrap() + .value; + let source = sources + .iter() + .find(|row| { + row.source_id == "gmail:Alice@Example.com|bob@example.com|carol@example.com" + }) + .expect("email thread source present"); + assert_eq!(source.display_name, "bob@example.com, carol@example.com"); + } + #[tokio::test] async fn entity_index_for_returns_extracted_entities() { let (_tmp, cfg) = test_config(); @@ -1956,6 +2159,25 @@ mod tests { ); } + #[tokio::test] + async fn chunks_for_entity_returns_leaf_chunk_ids_only() { + let (_tmp, cfg) = test_config(); + seed_chat_chunk(&cfg, "slack:#eng", "alice@example.com owns it").await; + let chunk_id = list_chunks_rpc(&cfg, ChunkFilter::default()) + .await + .unwrap() + .value + .chunks[0] + .id + .clone(); + + let rows = chunks_for_entity_rpc(&cfg, "email:alice@example.com".into()) + .await + .unwrap() + .value; + assert_eq!(rows, vec![chunk_id]); + } + #[tokio::test] async fn top_entities_returns_most_frequent() { let (_tmp, cfg) = test_config(); @@ -2030,11 +2252,74 @@ mod tests { seed_chat_chunk(&cfg, "slack:#eng", "phoenix migration scheduled friday").await; seed_chat_chunk(&cfg, "slack:#eng", "different unrelated text").await; let hits = search_rpc(&cfg, "phoenix".into(), 10).await.unwrap().value; - assert!(hits.iter().any(|c| c + assert!(hits.iter().any(|c| { + c.content_preview + .as_deref() + .unwrap_or("") + .contains("phoenix") + })); + } + + #[tokio::test] + async fn read_chunk_row_returns_preview_and_metadata() { + let (_tmp, cfg) = test_config(); + seed_chat_chunk( + &cfg, + "slack:#eng", + "phoenix migration scheduled friday with context and source refs", + ) + .await; + let chunk = list_chunks_rpc(&cfg, ChunkFilter::default()) + .await + .unwrap() + .value + .chunks + .into_iter() + .next() + .expect("seeded chunk"); + + let row = read_chunk_row(&cfg, &chunk.id).unwrap().expect("chunk row"); + assert_eq!(row.id, chunk.id); + assert_eq!(row.source_kind, "chat"); + assert_eq!(row.source_id, "slack:#eng"); + assert_eq!(row.source_ref.as_deref(), Some("slack://x")); + assert_eq!(row.owner, "alice"); + assert_eq!(row.lifecycle_status, "pending_extraction"); + assert!(row.content_path.is_some()); + assert!(row .content_preview .as_deref() .unwrap_or("") - .contains("phoenix"))); + .contains("phoenix migration scheduled friday")); + } + + #[tokio::test] + async fn read_chunk_row_falls_back_to_sqlite_preview_when_file_missing() { + let (_tmp, cfg) = test_config(); + let body = "sqlite preview survives missing file"; + seed_chat_chunk(&cfg, "slack:#eng", body).await; + let chunk = list_chunks_rpc(&cfg, ChunkFilter::default()) + .await + .unwrap() + .value + .chunks + .into_iter() + .next() + .expect("seeded chunk"); + + let rel_path = chunk.content_path.clone().expect("content path present"); + let abs_path = cfg.memory_tree_content_root().join(rel_path); + std::fs::remove_file(&abs_path).expect("remove chunk file"); + + let row = read_chunk_row(&cfg, &chunk.id).unwrap().expect("chunk row"); + assert_eq!(row.content_path, chunk.content_path); + assert!(row.content_preview.as_deref().unwrap_or("").contains(body)); + } + + #[test] + fn read_chunk_row_returns_none_for_missing_chunk() { + let (_tmp, cfg) = test_config(); + assert!(read_chunk_row(&cfg, "missing-chunk").unwrap().is_none()); } #[tokio::test] @@ -2338,8 +2623,92 @@ mod tests { ); } + #[test] + fn display_name_handles_multiple_participants_and_trimmed_hint() { + let name = display_name_for_source( + "gmail:Alice@Example.com|bob@example.com|carol@example.com", + Some(" alice@example.com "), + ); + assert_eq!(name, "bob@example.com, carol@example.com"); + } + #[test] fn display_name_handles_no_prefix() { assert_eq!(display_name_for_source("loose-id", None), "loose-id"); } + + #[test] + fn sanitize_basename_replaces_windows_illegal_characters() { + assert_eq!( + sanitize_basename(r#"chat:slack/#eng\name*?"<>|"#), + "chat-slack-#eng-name------" + ); + assert_eq!(sanitize_basename("safe-name.md"), "safe-name.md"); + } + + #[test] + fn parse_source_kind_str_accepts_known_values_only() { + assert_eq!(parse_source_kind_str("chat"), Some(SourceKind::Chat)); + assert_eq!(parse_source_kind_str("email"), Some(SourceKind::Email)); + assert_eq!( + parse_source_kind_str("document"), + Some(SourceKind::Document) + ); + assert_eq!(parse_source_kind_str("unknown"), None); + } + + #[tokio::test] + async fn get_llm_rpc_includes_current_backend_in_value_and_log() { + let (_tmp, mut cfg) = test_config(); + cfg.memory_tree.llm_backend = crate::openhuman::config::LlmBackend::Local; + let outcome = get_llm_rpc(&cfg).await.unwrap(); + assert_eq!(outcome.value.current, "local"); + assert_eq!( + outcome.logs, + vec!["memory_tree::read: get_llm current=local".to_string()] + ); + } + + #[test] + fn clear_composio_sync_state_removes_only_target_namespace() { + let tmp = TempDir::new().unwrap(); + let _memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + let db_path = tmp.path().join("memory").join("memory.db"); + let conn = rusqlite::Connection::open(&db_path).unwrap(); + + conn.execute( + "INSERT INTO kv_namespace (namespace, key, value_json, updated_at) + VALUES (?1, 'cursor', '{}', 1.0)", + params![KV_NAMESPACE], + ) + .unwrap(); + conn.execute( + "INSERT INTO kv_namespace (namespace, key, value_json, updated_at) + VALUES ('other-namespace', 'cursor', '{}', 2.0)", + [], + ) + .unwrap(); + drop(conn); + + let removed = clear_composio_sync_state(&db_path).unwrap(); + assert_eq!(removed, 1); + + let conn = rusqlite::Connection::open(&db_path).unwrap(); + let composio_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM kv_namespace WHERE namespace = ?1", + params![KV_NAMESPACE], + |row| row.get(0), + ) + .unwrap(); + let other_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM kv_namespace WHERE namespace = 'other-namespace'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(composio_count, 0); + assert_eq!(other_count, 1); + } } diff --git a/src/openhuman/memory_tree/retrieval/README.md b/src/openhuman/memory/retrieval/README.md similarity index 100% rename from src/openhuman/memory_tree/retrieval/README.md rename to src/openhuman/memory/retrieval/README.md diff --git a/src/openhuman/memory_tree/retrieval/benchmarks.rs b/src/openhuman/memory/retrieval/benchmarks.rs similarity index 98% rename from src/openhuman/memory_tree/retrieval/benchmarks.rs rename to src/openhuman/memory/retrieval/benchmarks.rs index ca2ec8715..3b7bf26da 100644 --- a/src/openhuman/memory_tree/retrieval/benchmarks.rs +++ b/src/openhuman/memory/retrieval/benchmarks.rs @@ -21,13 +21,13 @@ use chrono::{TimeZone, Utc}; use tempfile::TempDir; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage}; -use crate::openhuman::memory_tree::ingest::ingest_chat; -use crate::openhuman::memory_tree::jobs::testing::drain_until_idle; -use crate::openhuman::memory_tree::retrieval::{ +use crate::openhuman::memory::ingest_pipeline::ingest_chat; +use crate::openhuman::memory::jobs::testing::drain_until_idle; +use crate::openhuman::memory::retrieval::{ fetch_leaves, query_source, query_topic, search_entities, }; -use crate::openhuman::memory_tree::types::SourceKind; +use crate::openhuman::memory_store::chunks::types::SourceKind; +use crate::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage}; /// Shared test config — disables embedding for deterministic inert behaviour. fn bench_config() -> (TempDir, Config) { diff --git a/src/openhuman/memory_tree/retrieval/drill_down.rs b/src/openhuman/memory/retrieval/drill_down.rs similarity index 89% rename from src/openhuman/memory_tree/retrieval/drill_down.rs rename to src/openhuman/memory/retrieval/drill_down.rs index 0063cb30e..e37470be8 100644 --- a/src/openhuman/memory_tree/retrieval/drill_down.rs +++ b/src/openhuman/memory/retrieval/drill_down.rs @@ -23,13 +23,11 @@ use std::collections::VecDeque; use anyhow::Result; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::content_store::read as content_read; -use crate::openhuman::memory_tree::retrieval::types::{ - hit_from_chunk, hit_from_summary, RetrievalHit, -}; -use crate::openhuman::memory_tree::score::embed::{build_embedder_from_config, cosine_similarity}; -use crate::openhuman::memory_tree::store::{get_chunk, get_chunk_embedding}; -use crate::openhuman::memory_tree::tree_source::store; +use crate::openhuman::memory::retrieval::types::{hit_from_chunk, hit_from_summary, RetrievalHit}; +use crate::openhuman::memory::score::embed::{build_embedder_from_config, cosine_similarity}; +use crate::openhuman::memory_store::chunks::store::{get_chunk, get_chunk_embedding}; +use crate::openhuman::memory_store::content::read as content_read; +use crate::openhuman::memory_tree::tree::store; /// Walk the summary hierarchy down one step (or more if `max_depth > 1`) /// and return the hydrated child hits. Children at level 1 are raw chunks; @@ -257,16 +255,17 @@ fn walk_with_embeddings( #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory_tree::content_store; - use crate::openhuman::memory_tree::store::upsert_chunks; - use crate::openhuman::memory_tree::tree_source::bucket_seal::{ - append_leaf, LabelStrategy, LeafRef, + use crate::openhuman::memory::chat::{test_override, ChatProvider, StaticChatProvider}; + use crate::openhuman::memory_store::chunks::store::upsert_chunks; + use crate::openhuman::memory_store::chunks::types::{ + chunk_id, Chunk, Metadata, SourceKind, SourceRef, }; - use crate::openhuman::memory_tree::tree_source::registry::get_or_create_source_tree; - use crate::openhuman::memory_tree::tree_source::summariser::inert::InertSummariser; - use crate::openhuman::memory_tree::tree_source::types::TreeKind; - use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; + use crate::openhuman::memory_store::content as content_store; + use crate::openhuman::memory_store::trees::types::TreeKind; + use crate::openhuman::memory_tree::sources::registry::get_or_create_source_tree; + use crate::openhuman::memory_tree::tree::bucket_seal::{append_leaf, LabelStrategy, LeafRef}; use chrono::Utc; + use std::sync::Arc; use tempfile::TempDir; fn test_config() -> (TempDir, Config) { @@ -284,7 +283,8 @@ mod tests { // Seed two 6k-token leaves so the L0 buffer seals into an L1 node. let ts = Utc::now(); let tree = get_or_create_source_tree(cfg, "slack:#eng").unwrap(); - let summariser = InertSummariser::new(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); let content_root = cfg.memory_tree_content_root(); std::fs::create_dir_all(&content_root).unwrap(); let mut leaf_ids: Vec = Vec::new(); @@ -301,8 +301,7 @@ mod tests { tags: vec![], source_ref: Some(SourceRef::new("slack://x")), }, - token_count: crate::openhuman::memory_tree::tree_source::types::INPUT_TOKEN_BUDGET - * 6 + token_count: crate::openhuman::memory_store::trees::types::INPUT_TOKEN_BUDGET * 6 / 10, seq_in_source: seq, created_at: ts, @@ -312,33 +311,32 @@ mod tests { // Stage to disk so `hydrate_leaf_inputs` can read the full body // via `read_chunk_body` during the seal triggered by `append_leaf`. let staged = content_store::stage_chunks(&content_root, &[c.clone()]).unwrap(); - crate::openhuman::memory_tree::store::with_connection(cfg, |conn| { + crate::openhuman::memory_store::chunks::store::with_connection(cfg, |conn| { let tx = conn.unchecked_transaction()?; - crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?; + crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx( + &tx, &staged, + )?; tx.commit()?; Ok(()) }) .unwrap(); leaf_ids.push(c.id.clone()); - append_leaf( - cfg, - &tree, - &LeafRef { - chunk_id: c.id.clone(), - token_count: - crate::openhuman::memory_tree::tree_source::types::INPUT_TOKEN_BUDGET * 6 - / 10, - timestamp: ts, - content: c.content.clone(), - entities: vec![], - topics: vec![], - score: 0.5, - }, - &summariser, - &LabelStrategy::Empty, - ) - .await - .unwrap(); + let leaf = LeafRef { + chunk_id: c.id.clone(), + token_count: crate::openhuman::memory_store::trees::types::INPUT_TOKEN_BUDGET * 6 + / 10, + timestamp: ts, + content: c.content.clone(), + entities: vec![], + topics: vec![], + score: 0.5, + }; + test_override::with_provider(Arc::clone(&provider), async { + append_leaf(cfg, &tree, &leaf, &LabelStrategy::Empty) + .await + .unwrap() + }) + .await; } // Fetch the sealed L1 summary id from the tree row. let refreshed = store::get_tree(cfg, &tree.id).unwrap().unwrap(); @@ -430,9 +428,9 @@ mod tests { // (or similar — the key invariant is that BFS returns all siblings at // one depth before any descendant at a deeper depth). - use crate::openhuman::memory_tree::store::with_connection; - use crate::openhuman::memory_tree::tree_source::store as tree_store; - use crate::openhuman::memory_tree::tree_source::types::{SummaryNode, Tree, TreeStatus}; + use crate::openhuman::memory_store::chunks::store::with_connection; + use crate::openhuman::memory_store::trees::types::{SummaryNode, Tree, TreeStatus}; + use crate::openhuman::memory_tree::tree::store as tree_store; /// Build a tiny 2-level tree directly via store inserts so we can /// assert BFS ordering without needing ~100 leaves to cascade L1→L2 @@ -499,9 +497,9 @@ mod tests { let content_root = cfg.memory_tree_content_root(); std::fs::create_dir_all(&content_root).unwrap(); let staged = content_store::stage_chunks(&content_root, &all_leaves).unwrap(); - crate::openhuman::memory_tree::store::with_connection(cfg, |conn| { + crate::openhuman::memory_store::chunks::store::with_connection(cfg, |conn| { let tx = conn.unchecked_transaction()?; - crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?; + crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(&tx, &staged)?; tx.commit()?; Ok(()) }) diff --git a/src/openhuman/memory_tree/retrieval/fetch.rs b/src/openhuman/memory/retrieval/fetch.rs similarity index 91% rename from src/openhuman/memory_tree/retrieval/fetch.rs rename to src/openhuman/memory/retrieval/fetch.rs index 96dd7a33e..599bcecea 100644 --- a/src/openhuman/memory_tree/retrieval/fetch.rs +++ b/src/openhuman/memory/retrieval/fetch.rs @@ -13,10 +13,10 @@ use anyhow::Result; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::content_store::read as content_read; -use crate::openhuman::memory_tree::retrieval::types::{hit_from_chunk, RetrievalHit}; -use crate::openhuman::memory_tree::score::store::get_score; -use crate::openhuman::memory_tree::store::get_chunk; +use crate::openhuman::memory::retrieval::types::{hit_from_chunk, RetrievalHit}; +use crate::openhuman::memory::score::store::get_score; +use crate::openhuman::memory_store::chunks::store::get_chunk; +use crate::openhuman::memory_store::content::read as content_read; /// Max batch size. Callers that pass more than this get truncated with a /// warn log — no error surface so the LLM sees a partial result. @@ -96,9 +96,11 @@ pub async fn fetch_leaves(config: &Config, chunk_ids: &[String]) -> Result Vec = + Arc::new(StaticChatProvider::new("test summary content")); let day = Utc::now().date_naive(); let ts = day.and_hms_opt(12, 0, 0).unwrap().and_utc(); seed_source_for_day(cfg, "slack:#eng", ts).await; - end_of_day_digest(cfg, day, &summariser).await.unwrap(); + test_override::with_provider(provider, async { + end_of_day_digest(cfg, day).await.unwrap() + }) + .await; } async fn seed_source_for_day(cfg: &Config, scope: &str, ts: DateTime) { let tree = get_or_create_source_tree(cfg, scope).unwrap(); - let summariser = InertSummariser::new(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); for seq in 0..2u32 { let c = Chunk { id: chunk_id(SourceKind::Chat, scope, seq, "test-content"), @@ -155,23 +161,21 @@ mod tests { }; upsert_chunks(cfg, &[c.clone()]).unwrap(); stage_test_chunks(cfg, &[c.clone()]); - append_leaf( - cfg, - &tree, - &LeafRef { - chunk_id: c.id.clone(), - token_count: 30_000, - timestamp: ts, - content: c.content.clone(), - entities: vec![], - topics: vec![], - score: 0.5, - }, - &summariser, - &LabelStrategy::Empty, - ) - .await - .unwrap(); + let leaf = LeafRef { + chunk_id: c.id.clone(), + token_count: 30_000, + timestamp: ts, + content: c.content.clone(), + entities: vec![], + topics: vec![], + score: 0.5, + }; + test_override::with_provider(Arc::clone(&provider), async { + append_leaf(cfg, &tree, &leaf, &LabelStrategy::Empty) + .await + .unwrap() + }) + .await; } } @@ -205,11 +209,15 @@ mod tests { // if this ever returned Skipped the rest of the suite would trivially // pass which would be misleading. let (_tmp, cfg) = test_config(); - let summariser = InertSummariser::new(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); let day = Utc::now().date_naive(); let ts = day.and_hms_opt(12, 0, 0).unwrap().and_utc(); seed_source_for_day(&cfg, "slack:#eng", ts).await; - let outcome = end_of_day_digest(&cfg, day, &summariser).await.unwrap(); + let outcome = test_override::with_provider(provider, async { + end_of_day_digest(&cfg, day).await.unwrap() + }) + .await; assert!(matches!(outcome, DigestOutcome::Emitted { .. })); } } diff --git a/src/openhuman/memory_tree/retrieval/integration_test.rs b/src/openhuman/memory/retrieval/integration_test.rs similarity index 85% rename from src/openhuman/memory_tree/retrieval/integration_test.rs rename to src/openhuman/memory/retrieval/integration_test.rs index 8432fe2cd..ce5f92c65 100644 --- a/src/openhuman/memory_tree/retrieval/integration_test.rs +++ b/src/openhuman/memory/retrieval/integration_test.rs @@ -14,12 +14,12 @@ use chrono::{TimeZone, Utc}; use tempfile::TempDir; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage}; -use crate::openhuman::memory_tree::ingest::ingest_chat; -use crate::openhuman::memory_tree::retrieval::{ +use crate::openhuman::memory::ingest_pipeline::ingest_chat; +use crate::openhuman::memory::retrieval::{ drill_down, fetch_leaves, query_global, query_source, query_topic, search_entities, }; -use crate::openhuman::memory_tree::types::SourceKind; +use crate::openhuman::memory_store::chunks::types::SourceKind; +use crate::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage}; fn test_config() -> (TempDir, Config) { let tmp = TempDir::new().unwrap(); @@ -124,7 +124,7 @@ async fn end_to_end_three_chat_batches() { // ── fetch_leaves: find a guaranteed leaf hit from alice's topic results // and assert that fetch_leaves hydrates it correctly. - use crate::openhuman::memory_tree::retrieval::types::NodeKind; + use crate::openhuman::memory::retrieval::types::NodeKind; let leaf_hit = by_email .hits .iter() @@ -164,9 +164,9 @@ async fn topic_entity_surfaces_after_ingest() { /// handler, so the test drains the queue before inspecting. #[tokio::test] async fn ingest_populates_chunk_embeddings() { - use crate::openhuman::memory_tree::jobs::drain_until_idle; - use crate::openhuman::memory_tree::score::embed::EMBEDDING_DIM; - use crate::openhuman::memory_tree::store::get_chunk_embedding; + use crate::openhuman::memory::jobs::drain_until_idle; + use crate::openhuman::memory::score::embed::EMBEDDING_DIM; + use crate::openhuman::memory_store::chunks::store::get_chunk_embedding; let (_tmp, cfg) = test_config(); let out = ingest_chat(&cfg, "slack:#eng", "alice", vec![], chat_about_phoenix(0)) @@ -192,20 +192,21 @@ async fn ingest_populates_chunk_embeddings() { /// the seal from firing on short batches. #[tokio::test] async fn seal_populates_summary_embedding() { - use crate::openhuman::memory_tree::content_store; - use crate::openhuman::memory_tree::score::embed::EMBEDDING_DIM; - use crate::openhuman::memory_tree::store::upsert_chunks; - use crate::openhuman::memory_tree::tree_source::bucket_seal::{ - append_leaf, LabelStrategy, LeafRef, + use crate::openhuman::memory::chat::{test_override, ChatProvider, StaticChatProvider}; + use crate::openhuman::memory::score::embed::EMBEDDING_DIM; + use crate::openhuman::memory_store::chunks::store::upsert_chunks; + use crate::openhuman::memory_store::chunks::types::{ + chunk_id, Chunk, Metadata, SourceKind, SourceRef, }; - use crate::openhuman::memory_tree::tree_source::registry::get_or_create_source_tree; - use crate::openhuman::memory_tree::tree_source::store as src_store; - use crate::openhuman::memory_tree::tree_source::summariser::inert::InertSummariser; - use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; + use crate::openhuman::memory_store::content as content_store; + use crate::openhuman::memory_tree::sources::registry::get_or_create_source_tree; + use crate::openhuman::memory_tree::tree::bucket_seal::{append_leaf, LabelStrategy, LeafRef}; + use crate::openhuman::memory_tree::tree::store as src_store; + use std::sync::Arc; let (_tmp, cfg) = test_config(); let tree = get_or_create_source_tree(&cfg, "slack:#seal-test").unwrap(); - let summariser = InertSummariser::new(); + let provider: Arc = Arc::new(StaticChatProvider::new("test summary content")); let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(); let mk_chunk = |seq: u32, tokens: u32| Chunk { @@ -233,9 +234,9 @@ async fn seal_populates_summary_embedding() { std::fs::create_dir_all(&content_root).expect("create content_root for test"); let staged = content_store::stage_chunks(&content_root, &[c1.clone(), c2.clone()]) .expect("stage_chunks for test chunks"); - crate::openhuman::memory_tree::store::with_connection(&cfg, |conn| { + crate::openhuman::memory_store::chunks::store::with_connection(&cfg, |conn| { let tx = conn.unchecked_transaction()?; - crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?; + crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(&tx, &staged)?; tx.commit()?; Ok(()) }) @@ -251,24 +252,18 @@ async fn seal_populates_summary_embedding() { topics: vec![], score: 0.5, }; - append_leaf( - &cfg, - &tree, - &leaf_of(&c1), - &summariser, - &LabelStrategy::Empty, - ) - .await - .unwrap(); - let sealed = append_leaf( - &cfg, - &tree, - &leaf_of(&c2), - &summariser, - &LabelStrategy::Empty, - ) - .await - .unwrap(); + test_override::with_provider(Arc::clone(&provider), async { + append_leaf(&cfg, &tree, &leaf_of(&c1), &LabelStrategy::Empty) + .await + .unwrap() + }) + .await; + let sealed = test_override::with_provider(Arc::clone(&provider), async { + append_leaf(&cfg, &tree, &leaf_of(&c2), &LabelStrategy::Empty) + .await + .unwrap() + }) + .await; assert_eq!(sealed.len(), 1, "expected one seal at the budget crossing"); // #1574 cutover: the seal path no longer writes the legacy diff --git a/src/openhuman/memory_tree/retrieval/mod.rs b/src/openhuman/memory/retrieval/mod.rs similarity index 100% rename from src/openhuman/memory_tree/retrieval/mod.rs rename to src/openhuman/memory/retrieval/mod.rs diff --git a/src/openhuman/memory_tree/retrieval/rpc.rs b/src/openhuman/memory/retrieval/rpc.rs similarity index 97% rename from src/openhuman/memory_tree/retrieval/rpc.rs rename to src/openhuman/memory/retrieval/rpc.rs index 24987bb2b..a062adb8e 100644 --- a/src/openhuman/memory_tree/retrieval/rpc.rs +++ b/src/openhuman/memory/retrieval/rpc.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::retrieval::{ +use crate::openhuman::memory::retrieval::{ drill_down::drill_down, fetch::fetch_leaves, global::query_global, @@ -17,8 +17,8 @@ use crate::openhuman::memory_tree::retrieval::{ topic::query_topic, types::{EntityMatch, QueryResponse, RetrievalHit}, }; -use crate::openhuman::memory_tree::score::extract::EntityKind; -use crate::openhuman::memory_tree::types::SourceKind; +use crate::openhuman::memory::score::extract::EntityKind; +use crate::openhuman::memory_store::chunks::types::SourceKind; use crate::rpc::RpcOutcome; // ── query_source ────────────────────────────────────────────────────── @@ -307,9 +307,9 @@ mod tests { //! initialises the schema idempotently on first access, so read-only //! calls return empty responses rather than erroring. use super::*; - use crate::openhuman::memory_tree::content_store; - use crate::openhuman::memory_tree::store::upsert_chunks; - use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceRef}; + use crate::openhuman::memory_store::chunks::store::upsert_chunks; + use crate::openhuman::memory_store::chunks::types::{chunk_id, Chunk, Metadata, SourceRef}; + use crate::openhuman::memory_store::content as content_store; use chrono::{TimeZone, Utc}; use tempfile::TempDir; @@ -318,9 +318,9 @@ mod tests { std::fs::create_dir_all(&content_root).expect("create content_root for test"); let staged = content_store::stage_chunks(&content_root, chunks) .expect("stage_chunks for test chunks"); - crate::openhuman::memory_tree::store::with_connection(cfg, |conn| { + crate::openhuman::memory_store::chunks::store::with_connection(cfg, |conn| { let tx = conn.unchecked_transaction()?; - crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?; + crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(&tx, &staged)?; tx.commit()?; Ok(()) }) diff --git a/src/openhuman/memory_tree/retrieval/schemas.rs b/src/openhuman/memory/retrieval/schemas.rs similarity index 91% rename from src/openhuman/memory_tree/retrieval/schemas.rs rename to src/openhuman/memory/retrieval/schemas.rs index 1503fe62b..fafb5661e 100644 --- a/src/openhuman/memory_tree/retrieval/schemas.rs +++ b/src/openhuman/memory/retrieval/schemas.rs @@ -18,7 +18,7 @@ use serde_json::{Map, Value}; use crate::core::all::{ControllerFuture, RegisteredController}; use crate::core::{ControllerSchema, FieldSchema, TypeSchema}; use crate::openhuman::config::rpc as config_rpc; -use crate::openhuman::memory_tree::retrieval::rpc as retrieval_rpc; +use crate::openhuman::memory::retrieval::rpc as retrieval_rpc; use crate::rpc::RpcOutcome; const NAMESPACE: &str = "memory_tree"; @@ -385,3 +385,49 @@ fn parse_value(v: Value) -> Result { fn to_json(outcome: RpcOutcome) -> Result { outcome.into_cli_compatible_json() } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn all_controller_schemas_cover_every_registered_retrieval_function() { + let schemas = all_controller_schemas(); + let functions: Vec<&str> = schemas.iter().map(|s| s.function).collect(); + assert_eq!( + functions, + vec![ + "query_source", + "query_global", + "query_topic", + "search_entities", + "drill_down", + "fetch_leaves", + ] + ); + } + + #[test] + fn registered_controllers_use_memory_tree_namespace() { + let controllers = all_registered_controllers(); + assert_eq!(controllers.len(), 6); + assert!(controllers.iter().all(|c| c.schema.namespace == NAMESPACE)); + } + + #[test] + fn unknown_schema_returns_error_output() { + let schema = schemas("not_a_real_function"); + assert_eq!(schema.namespace, NAMESPACE); + assert_eq!(schema.function, "unknown"); + assert_eq!(schema.outputs.len(), 1); + assert_eq!(schema.outputs[0].name, "error"); + } + + #[test] + fn query_global_schema_requires_time_window_days() { + let schema = schemas("query_global"); + assert_eq!(schema.inputs.len(), 1); + assert_eq!(schema.inputs[0].name, "time_window_days"); + assert!(schema.inputs[0].required); + } +} diff --git a/src/openhuman/memory_tree/retrieval/search.rs b/src/openhuman/memory/retrieval/search.rs similarity index 97% rename from src/openhuman/memory_tree/retrieval/search.rs rename to src/openhuman/memory/retrieval/search.rs index e86d28d33..0cc605429 100644 --- a/src/openhuman/memory_tree/retrieval/search.rs +++ b/src/openhuman/memory/retrieval/search.rs @@ -19,9 +19,9 @@ use anyhow::{Context, Result}; use rusqlite::params_from_iter; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::retrieval::types::EntityMatch; -use crate::openhuman::memory_tree::score::extract::EntityKind; -use crate::openhuman::memory_tree::store::with_connection; +use crate::openhuman::memory::retrieval::types::EntityMatch; +use crate::openhuman::memory::score::extract::EntityKind; +use crate::openhuman::memory_store::chunks::store::with_connection; const DEFAULT_LIMIT: usize = 5; const MAX_LIMIT: usize = 100; @@ -156,8 +156,8 @@ fn row_to_match(row: &rusqlite::Row<'_>) -> rusqlite::Result { #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage}; - use crate::openhuman::memory_tree::ingest::ingest_chat; + use crate::openhuman::memory::ingest_pipeline::ingest_chat; + use crate::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage}; use chrono::{TimeZone, Utc}; use tempfile::TempDir; diff --git a/src/openhuman/memory_tree/retrieval/source.rs b/src/openhuman/memory/retrieval/source.rs similarity index 89% rename from src/openhuman/memory_tree/retrieval/source.rs rename to src/openhuman/memory/retrieval/source.rs index 558e36324..c2794e497 100644 --- a/src/openhuman/memory_tree/retrieval/source.rs +++ b/src/openhuman/memory/retrieval/source.rs @@ -21,14 +21,12 @@ use anyhow::Result; use chrono::{Duration, Utc}; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::content_store::read as content_read; -use crate::openhuman::memory_tree::retrieval::types::{ - hit_from_summary, QueryResponse, RetrievalHit, -}; -use crate::openhuman::memory_tree::score::embed::{build_embedder_from_config, cosine_similarity}; -use crate::openhuman::memory_tree::tree_source::store; -use crate::openhuman::memory_tree::tree_source::types::{SummaryNode, Tree, TreeKind}; -use crate::openhuman::memory_tree::types::SourceKind; +use crate::openhuman::memory::retrieval::types::{hit_from_summary, QueryResponse, RetrievalHit}; +use crate::openhuman::memory::score::embed::{build_embedder_from_config, cosine_similarity}; +use crate::openhuman::memory_store::chunks::types::SourceKind; +use crate::openhuman::memory_store::content::read as content_read; +use crate::openhuman::memory_store::trees::types::{SummaryNode, Tree, TreeKind}; +use crate::openhuman::memory_tree::tree::store; const DEFAULT_LIMIT: usize = 10; @@ -306,15 +304,16 @@ fn filter_by_window(hits: Vec, window_days: u32) -> Vec (TempDir, Config) { @@ -330,7 +329,8 @@ mod tests { async fn seed_source(cfg: &Config, scope: &str, ts: DateTime) { let tree = get_or_create_source_tree(cfg, scope).unwrap(); - let summariser = InertSummariser::new(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); let content_root = cfg.memory_tree_content_root(); std::fs::create_dir_all(&content_root).unwrap(); for seq in 0..2u32 { @@ -346,8 +346,7 @@ mod tests { tags: vec!["eng".into()], source_ref: Some(SourceRef::new(format!("slack://{scope}/{seq}"))), }, - token_count: crate::openhuman::memory_tree::tree_source::types::INPUT_TOKEN_BUDGET - * 6 + token_count: crate::openhuman::memory_store::trees::types::INPUT_TOKEN_BUDGET * 6 / 10, seq_in_source: seq, created_at: ts, @@ -358,32 +357,31 @@ mod tests { // via `read_chunk_body` during the seal triggered by `append_leaf`, // and `collect_hits_and_nodes` can read summary bodies for the API. let staged = content_store::stage_chunks(&content_root, &[c.clone()]).unwrap(); - crate::openhuman::memory_tree::store::with_connection(cfg, |conn| { + crate::openhuman::memory_store::chunks::store::with_connection(cfg, |conn| { let tx = conn.unchecked_transaction()?; - crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?; + crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx( + &tx, &staged, + )?; tx.commit()?; Ok(()) }) .unwrap(); - append_leaf( - cfg, - &tree, - &LeafRef { - chunk_id: c.id.clone(), - token_count: - crate::openhuman::memory_tree::tree_source::types::INPUT_TOKEN_BUDGET * 6 - / 10, - timestamp: ts, - content: c.content.clone(), - entities: vec![], - topics: vec![], - score: 0.5, - }, - &summariser, - &LabelStrategy::Empty, - ) - .await - .unwrap(); + let leaf = LeafRef { + chunk_id: c.id.clone(), + token_count: crate::openhuman::memory_store::trees::types::INPUT_TOKEN_BUDGET * 6 + / 10, + timestamp: ts, + content: c.content.clone(), + entities: vec![], + topics: vec![], + score: 0.5, + }; + test_override::with_provider(Arc::clone(&provider), async { + append_leaf(cfg, &tree, &leaf, &LabelStrategy::Empty) + .await + .unwrap() + }) + .await; } } @@ -527,8 +525,8 @@ mod tests { /// the ingest path writes by default. #[tokio::test] async fn query_reranks_by_cosine_similarity() { - use crate::openhuman::memory_tree::score::embed::{pack_embedding, EMBEDDING_DIM}; - use crate::openhuman::memory_tree::tree_source::store as src_store; + use crate::openhuman::memory::score::embed::{pack_embedding, EMBEDDING_DIM}; + use crate::openhuman::memory_tree::tree::store as src_store; let (_tmp, cfg) = test_config(); let ts = Utc::now(); @@ -548,17 +546,17 @@ mod tests { // Write directly via raw UPDATE so we replace whatever the // seal-time inert embedder wrote. - use crate::openhuman::memory_tree::store::with_connection; + use crate::openhuman::memory_store::chunks::store::with_connection; let phoenix_tree = src_store::get_tree_by_scope( &cfg, - crate::openhuman::memory_tree::tree_source::types::TreeKind::Source, + crate::openhuman::memory_store::trees::types::TreeKind::Source, "slack:#phoenix", ) .unwrap() .unwrap(); let unrelated_tree = src_store::get_tree_by_scope( &cfg, - crate::openhuman::memory_tree::tree_source::types::TreeKind::Source, + crate::openhuman::memory_store::trees::types::TreeKind::Source, "slack:#unrelated", ) .unwrap() @@ -597,7 +595,7 @@ mod tests { // The practical test here: construct a hypothetical query // vector equal to phoenix_vec, then verify that running the // rerank helper with that vector places phoenix first. - use crate::openhuman::memory_tree::score::embed::cosine_similarity; + use crate::openhuman::memory::score::embed::cosine_similarity; let query_vec = phoenix_vec.clone(); let phoenix_sim = cosine_similarity(&query_vec, &phoenix_vec); let unrelated_sim = cosine_similarity(&query_vec, &unrelated_vec); @@ -629,9 +627,9 @@ mod tests { /// summaries that do have embeddings when a `query` is supplied. #[tokio::test] async fn legacy_null_embedding_rows_sort_last() { - use crate::openhuman::memory_tree::score::embed::{pack_embedding, EMBEDDING_DIM}; - use crate::openhuman::memory_tree::tree_source::store as src_store; - use crate::openhuman::memory_tree::tree_source::types::TreeKind; + use crate::openhuman::memory::score::embed::{pack_embedding, EMBEDDING_DIM}; + use crate::openhuman::memory_store::trees::types::TreeKind; + use crate::openhuman::memory_tree::tree::store as src_store; let (_tmp, cfg) = test_config(); let ts = Utc::now(); @@ -655,7 +653,7 @@ mod tests { v[0] = 1.0; let blob = pack_embedding(&v); - use crate::openhuman::memory_tree::store::with_connection; + use crate::openhuman::memory_store::chunks::store::with_connection; with_connection(&cfg, |conn| { conn.execute( "UPDATE mem_tree_summaries SET embedding = ?1 WHERE id = ?2", diff --git a/src/openhuman/memory_tree/retrieval/topic.rs b/src/openhuman/memory/retrieval/topic.rs similarity index 72% rename from src/openhuman/memory_tree/retrieval/topic.rs rename to src/openhuman/memory/retrieval/topic.rs index b154dbcf2..6a8c3da58 100644 --- a/src/openhuman/memory_tree/retrieval/topic.rs +++ b/src/openhuman/memory/retrieval/topic.rs @@ -17,14 +17,12 @@ use anyhow::Result; use chrono::{Duration, TimeZone, Utc}; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::content_store::read as content_read; -use crate::openhuman::memory_tree::retrieval::types::{ - hit_from_summary, QueryResponse, RetrievalHit, -}; -use crate::openhuman::memory_tree::score::embed::{build_embedder_from_config, cosine_similarity}; -use crate::openhuman::memory_tree::score::store::{lookup_entity, EntityHit}; -use crate::openhuman::memory_tree::tree_source::store; -use crate::openhuman::memory_tree::tree_source::types::{Tree, TreeKind}; +use crate::openhuman::memory::retrieval::types::{hit_from_summary, QueryResponse, RetrievalHit}; +use crate::openhuman::memory::score::embed::{build_embedder_from_config, cosine_similarity}; +use crate::openhuman::memory::score::store::{lookup_entity, EntityHit}; +use crate::openhuman::memory_store::content::read as content_read; +use crate::openhuman::memory_store::trees::types::{Tree, TreeKind}; +use crate::openhuman::memory_tree::tree::store; const DEFAULT_LIMIT: usize = 10; /// How many rows we pull from the entity index before filtering. We give @@ -175,9 +173,9 @@ async fn rerank_by_semantic_similarity( query: &str, hits: Vec, ) -> Result> { - use crate::openhuman::memory_tree::retrieval::types::NodeKind; - use crate::openhuman::memory_tree::store::get_chunk_embedding; - use crate::openhuman::memory_tree::tree_source::store as src_store; + use crate::openhuman::memory::retrieval::types::NodeKind; + use crate::openhuman::memory_store::chunks::store::get_chunk_embedding; + use crate::openhuman::memory_tree::tree::store as src_store; let embedder = build_embedder_from_config(config)?; let query_vec = embedder.embed(query).await?; @@ -318,8 +316,8 @@ async fn entity_hit_to_retrieval_hit( return Ok(Some(h)); } // Leaf: fetch chunk and hydrate. - use crate::openhuman::memory_tree::retrieval::types::hit_from_chunk; - use crate::openhuman::memory_tree::store::get_chunk; + use crate::openhuman::memory::retrieval::types::hit_from_chunk; + use crate::openhuman::memory_store::chunks::store::get_chunk; let mut chunk = match get_chunk(&config_owned, &node_id)? { Some(c) => c, None => { @@ -368,8 +366,14 @@ fn filter_by_window(hits: Vec, window_days: u32) -> Vec DEFAULT_LIMIT { + assert!(resp.truncated); + } + } + + #[tokio::test] + async fn topic_tree_root_missing_summary_row_is_ignored() { + use crate::openhuman::memory_store::chunks::store::with_connection; + use crate::openhuman::memory_store::trees::types::{Tree, TreeKind, TreeStatus}; + use crate::openhuman::memory_tree::tree::store as tree_store; + + let (_tmp, cfg) = test_config(); + let ts = Utc::now(); + let entity_id = "topic:phoenix"; + let tree = Tree { + id: "test:phoenix-missing-root".into(), + kind: TreeKind::Topic, + scope: entity_id.into(), + root_id: Some("summary:missing".into()), + max_level: 1, + status: TreeStatus::Active, + created_at: ts, + last_sealed_at: Some(ts), + }; + + with_connection(&cfg, |conn| { + let tx = conn.unchecked_transaction()?; + tree_store::insert_tree_conn(&tx, &tree)?; + tx.commit()?; + Ok(()) + }) + .unwrap(); + + let resp = query_topic(&cfg, entity_id, None, None, 10).await.unwrap(); + assert!(resp.hits.is_empty()); + assert_eq!(resp.total, 0); + } + // Regression: the same node_id must only appear once in `hits`, even // when the topic-tree root overlaps with its own entity-index row. // Flagged on PR #831 CodeRabbit review — see the HashMap-based merge @@ -529,14 +598,13 @@ mod tests { // caller would see two rows for the same summary. #[tokio::test] async fn duplicate_node_is_deduplicated_across_index_and_topic_tree_root() { - use crate::openhuman::memory_tree::score::extract::EntityKind; - use crate::openhuman::memory_tree::score::resolver::CanonicalEntity; - use crate::openhuman::memory_tree::score::store as score_store; - use crate::openhuman::memory_tree::store::with_connection; - use crate::openhuman::memory_tree::tree_source::store as tree_store; - use crate::openhuman::memory_tree::tree_source::types::{ + use crate::openhuman::memory::score::resolver::CanonicalEntity; + use crate::openhuman::memory::score::store as score_store; + use crate::openhuman::memory_store::chunks::store::with_connection; + use crate::openhuman::memory_store::trees::types::{ SummaryNode, Tree, TreeKind, TreeStatus, }; + use crate::openhuman::memory_tree::tree::store as tree_store; let (_tmp, cfg) = test_config(); let ts = Utc::now(); @@ -625,4 +693,144 @@ mod tests { "total should count distinct nodes, not raw row occurrences" ); } + + #[tokio::test] + async fn stale_leaf_entity_index_row_is_skipped() { + let (_tmp, cfg) = test_config(); + let hit = EntityHit { + entity_id: "email:alice@example.com".into(), + node_id: "chunk:missing".into(), + node_kind: "leaf".into(), + entity_kind: EntityKind::Email, + surface: "alice@example.com".into(), + score: 0.7, + timestamp_ms: Utc::now().timestamp_millis(), + tree_id: Some("tree:missing".into()), + is_user: false, + }; + + let converted = entity_hit_to_retrieval_hit(&cfg, &hit).await.unwrap(); + assert!(converted.is_none()); + } + + #[tokio::test] + async fn leaf_hit_falls_back_to_source_scope_when_tree_lookup_misses() { + let (_tmp, cfg) = test_config(); + let ts = Utc.timestamp_millis_opt(1_700_123_456_789).unwrap(); + let chunk = Chunk { + id: chunk_id( + SourceKind::Chat, + "slack:#eng", + 0, + "Phoenix owner is alice@example.com", + ), + content: "Phoenix owner is alice@example.com".into(), + metadata: Metadata { + source_kind: SourceKind::Chat, + source_id: "slack:#eng".into(), + owner: "alice".into(), + timestamp: ts, + time_range: (ts, ts), + tags: vec!["phoenix".into()], + source_ref: Some(SourceRef::new("slack://eng/1")), + }, + token_count: 8, + seq_in_source: 0, + created_at: ts, + partial_message: false, + }; + upsert_chunks(&cfg, &[chunk.clone()]).unwrap(); + + let hit = EntityHit { + entity_id: "email:alice@example.com".into(), + node_id: chunk.id.clone(), + node_kind: "leaf".into(), + entity_kind: EntityKind::Email, + surface: "alice@example.com".into(), + score: 0.91, + timestamp_ms: ts.timestamp_millis(), + tree_id: Some("tree:missing".into()), + is_user: false, + }; + + let converted = entity_hit_to_retrieval_hit(&cfg, &hit) + .await + .unwrap() + .unwrap(); + assert_eq!(converted.tree_scope, "slack:#eng"); + assert_eq!(converted.tree_id, "tree:missing"); + assert_eq!(converted.score, 0.91); + assert_eq!(converted.source_ref.as_deref(), Some("slack://eng/1")); + assert_eq!(converted.topics, vec!["phoenix"]); + } + + #[tokio::test] + async fn summary_hit_without_tree_id_uses_empty_scope_and_index_score() { + use crate::openhuman::memory_store::chunks::store::with_connection; + use crate::openhuman::memory_store::trees::types::{ + SummaryNode, Tree, TreeKind, TreeStatus, + }; + use crate::openhuman::memory_tree::tree::store as tree_store; + + let (_tmp, cfg) = test_config(); + let ts = Utc::now(); + let tree = Tree { + id: "tree:topic:phoenix".into(), + kind: TreeKind::Topic, + scope: "topic:phoenix".into(), + root_id: Some("summary:l1:phoenix".into()), + max_level: 1, + status: TreeStatus::Active, + created_at: ts, + last_sealed_at: Some(ts), + }; + let summary = SummaryNode { + id: "summary:l1:phoenix".into(), + tree_id: tree.id.clone(), + tree_kind: TreeKind::Topic, + level: 1, + parent_id: None, + child_ids: vec!["chunk:a".into()], + content: "Phoenix recap preview".into(), + token_count: 16, + entities: vec!["topic:phoenix".into()], + topics: vec!["phoenix".into()], + time_range_start: ts, + time_range_end: ts, + score: 0.12, + sealed_at: ts, + deleted: false, + embedding: None, + }; + + with_connection(&cfg, |conn| { + let tx = conn.unchecked_transaction()?; + tree_store::insert_tree_conn(&tx, &tree)?; + tree_store::insert_summary_tx(&tx, &summary, None, "test")?; + tx.commit()?; + Ok(()) + }) + .unwrap(); + + let hit = EntityHit { + entity_id: "topic:phoenix".into(), + node_id: summary.id.clone(), + node_kind: "summary".into(), + entity_kind: EntityKind::Topic, + surface: "phoenix".into(), + score: 0.88, + timestamp_ms: ts.timestamp_millis(), + tree_id: None, + is_user: false, + }; + + let converted = entity_hit_to_retrieval_hit(&cfg, &hit) + .await + .unwrap() + .unwrap(); + assert_eq!(converted.tree_scope, ""); + assert_eq!(converted.tree_id, tree.id); + assert_eq!(converted.score, 0.88); + assert_eq!(converted.content, "Phoenix recap preview"); + } } diff --git a/src/openhuman/memory_tree/retrieval/types.rs b/src/openhuman/memory/retrieval/types.rs similarity index 97% rename from src/openhuman/memory_tree/retrieval/types.rs rename to src/openhuman/memory/retrieval/types.rs index d84ef6f79..2db643945 100644 --- a/src/openhuman/memory_tree/retrieval/types.rs +++ b/src/openhuman/memory/retrieval/types.rs @@ -18,9 +18,9 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use crate::openhuman::memory_tree::score::extract::EntityKind; -use crate::openhuman::memory_tree::tree_source::types::{SummaryNode, Tree, TreeKind}; -use crate::openhuman::memory_tree::types::{Chunk, SourceKind}; +use crate::openhuman::memory::score::extract::EntityKind; +use crate::openhuman::memory_store::chunks::types::{Chunk, SourceKind}; +use crate::openhuman::memory_store::trees::types::{SummaryNode, Tree, TreeKind}; /// Whether a hit represents a leaf (raw chunk) or a summary node. #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] diff --git a/src/openhuman/memory_tree/schemas.rs b/src/openhuman/memory/schema.rs similarity index 92% rename from src/openhuman/memory_tree/schemas.rs rename to src/openhuman/memory/schema.rs index d18744a72..d20f63b88 100644 --- a/src/openhuman/memory_tree/schemas.rs +++ b/src/openhuman/memory/schema.rs @@ -16,8 +16,8 @@ use serde_json::{Map, Value}; use crate::core::all::{ControllerFuture, RegisteredController}; use crate::core::{ControllerSchema, FieldSchema, TypeSchema}; use crate::openhuman::config::rpc as config_rpc; -use crate::openhuman::memory_tree::read_rpc; -use crate::openhuman::memory_tree::rpc as tree_rpc; +use crate::openhuman::memory::read_rpc; +use crate::openhuman::memory::tree_rpc; use crate::rpc::RpcOutcome; const NAMESPACE: &str = "memory_tree"; @@ -207,26 +207,31 @@ pub fn schemas(function: &str) -> ControllerSchema { "list_chunks" => ControllerSchema { namespace: NAMESPACE, function: "list_chunks", - description: - "Paginated list of chunks with optional filters by source kind / source id / \ + description: "Paginated list of chunks with optional filters by source kind / source id / \ entity ids / time window / keyword. Returns chunks plus total match count for \ pagination.", inputs: vec![ FieldSchema { name: "source_kinds", - ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new(TypeSchema::String)))), + ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new( + TypeSchema::String, + )))), comment: "Restrict to one or more source kinds (chat / email / document).", required: false, }, FieldSchema { name: "source_ids", - ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new(TypeSchema::String)))), + ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new( + TypeSchema::String, + )))), comment: "Restrict to one or more logical source ids.", required: false, }, FieldSchema { name: "entity_ids", - ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new(TypeSchema::String)))), + ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new( + TypeSchema::String, + )))), comment: "Restrict to chunks indexed against any of these canonical entity ids.", required: false, }, @@ -296,8 +301,7 @@ pub fn schemas(function: &str) -> ControllerSchema { "list_sources" => ControllerSchema { namespace: NAMESPACE, function: "list_sources", - description: - "Distinct (source_kind, source_id) pairs with chunk counts and most-recent timestamps. \ + description: "Distinct (source_kind, source_id) pairs with chunk counts and most-recent timestamps. \ `display_name` is computed from the source_id (un-slug + strip user email when known).", inputs: vec![FieldSchema { name: "user_email_hint", @@ -316,8 +320,7 @@ pub fn schemas(function: &str) -> ControllerSchema { "search" => ControllerSchema { namespace: NAMESPACE, function: "search", - description: - "Keyword LIKE-search over chunk bodies. Cheap, deterministic; useful as a \ + description: "Keyword LIKE-search over chunk bodies. Cheap, deterministic; useful as a \ fallback when semantic recall is unavailable.", inputs: vec![ FieldSchema { @@ -343,8 +346,7 @@ pub fn schemas(function: &str) -> ControllerSchema { "recall" => ControllerSchema { namespace: NAMESPACE, function: "recall", - description: - "Semantic recall — runs the Phase 4 cosine rerank against the query embedding \ + description: "Semantic recall — runs the Phase 4 cosine rerank against the query embedding \ and returns leaf chunks (not summaries) for UI display.", inputs: vec![ FieldSchema { @@ -395,14 +397,12 @@ pub fn schemas(function: &str) -> ControllerSchema { "chunks_for_entity" => ControllerSchema { namespace: NAMESPACE, function: "chunks_for_entity", - description: - "Return chunk IDs that reference an entity_id (inverse of entity_index_for). \ + description: "Return chunk IDs that reference an entity_id (inverse of entity_index_for). \ Used by the Memory tab's People/Topics lenses to filter the chunk list.", inputs: vec![FieldSchema { name: "entity_id", ty: TypeSchema::String, - comment: - "Canonical entity id (e.g. `person:Steven Enamakel`, \ + comment: "Canonical entity id (e.g. `person:Steven Enamakel`, \ `email:alice@example.com`).", required: true, }], @@ -416,8 +416,7 @@ pub fn schemas(function: &str) -> ControllerSchema { "top_entities" => ControllerSchema { namespace: NAMESPACE, function: "top_entities", - description: - "Most-frequent canonical entities across the workspace, optionally narrowed by kind.", + description: "Most-frequent canonical entities across the workspace, optionally narrowed by kind.", inputs: vec![ FieldSchema { name: "kind", @@ -442,8 +441,7 @@ pub fn schemas(function: &str) -> ControllerSchema { "chunk_score" => ControllerSchema { namespace: NAMESPACE, function: "chunk_score", - description: - "Score breakdown stored in `mem_tree_score` for one chunk — used by the Memory \ + description: "Score breakdown stored in `mem_tree_score` for one chunk — used by the Memory \ tab's 'why was this kept / dropped' panel.", inputs: vec![FieldSchema { name: "chunk_id", @@ -461,8 +459,7 @@ pub fn schemas(function: &str) -> ControllerSchema { "delete_chunk" => ControllerSchema { namespace: NAMESPACE, function: "delete_chunk", - description: - "Purge one chunk plus its score row, entity-index rows, and on-disk .md file. \ + description: "Purge one chunk plus its score row, entity-index rows, and on-disk .md file. \ Idempotent — missing chunk returns deleted=false. Does NOT cascade through \ sealed summaries; UIs warn the user.", inputs: vec![FieldSchema { @@ -509,8 +506,7 @@ pub fn schemas(function: &str) -> ControllerSchema { "set_llm" => ControllerSchema { namespace: NAMESPACE, function: "set_llm", - description: - "Update the LLM backend selector and (optionally) per-role model choices \ + description: "Update the LLM backend selector and (optionally) per-role model choices \ (`cloud_model`, `extract_model`, `summariser_model`) and persist the \ result to config.toml in a single atomic write. Absent model fields \ leave the corresponding config key unchanged so a caller flipping just \ @@ -961,3 +957,51 @@ fn parse_value(v: Value) -> Result { fn to_json(outcome: RpcOutcome) -> Result { outcome.into_cli_compatible_json() } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn all_controller_schemas_and_registered_controllers_stay_in_sync() { + let schemas = all_controller_schemas(); + let controllers = all_registered_controllers(); + assert_eq!(schemas.len(), controllers.len()); + assert!(schemas.iter().all(|s| s.namespace == NAMESPACE)); + assert!(controllers.iter().all(|c| c.schema.namespace == NAMESPACE)); + } + + #[test] + fn unknown_function_schema_returns_error_output() { + let schema = schemas("not_real"); + assert_eq!(schema.namespace, NAMESPACE); + assert_eq!(schema.function, "unknown"); + assert_eq!(schema.outputs.len(), 1); + assert_eq!(schema.outputs[0].name, "error"); + } + + #[test] + fn ingest_schema_requires_source_kind_source_id_and_payload() { + let schema = schemas("ingest"); + assert_eq!(schema.function, "ingest"); + let required: Vec<&str> = schema + .inputs + .iter() + .filter(|f| f.required) + .map(|f| f.name) + .collect(); + assert!(required.contains(&"source_kind")); + assert!(required.contains(&"source_id")); + assert!(required.contains(&"payload")); + } + + #[test] + fn set_llm_schema_exposes_backend_update_fields() { + let schema = schemas("set_llm"); + let names: Vec<&str> = schema.inputs.iter().map(|f| f.name).collect(); + assert!(names.contains(&"backend")); + assert!(names.contains(&"cloud_model")); + assert!(names.contains(&"extract_model")); + assert!(names.contains(&"summariser_model")); + } +} diff --git a/src/openhuman/memory/schemas/documents.rs b/src/openhuman/memory/schemas/documents.rs index 6399bb22c..ce121ce26 100644 --- a/src/openhuman/memory/schemas/documents.rs +++ b/src/openhuman/memory/schemas/documents.rs @@ -535,3 +535,42 @@ fn handle_clear_namespace(params: Map) -> ControllerFuture { to_json(rpc::clear_namespace(payload).await?) }) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn documents_schema_exposes_all_functions() { + assert_eq!(controllers().len(), FUNCTIONS.len()); + assert!(FUNCTIONS.contains(&"init")); + assert!(FUNCTIONS.contains(&"doc_ingest")); + assert!(FUNCTIONS.contains(&"clear_namespace")); + } + + #[test] + fn unknown_document_schema_returns_none() { + assert!(schema("not_real").is_none()); + } + + #[test] + fn query_namespace_schema_requires_namespace_and_query() { + let schema = schema("query_namespace").unwrap(); + let required: Vec<&str> = schema + .inputs + .iter() + .filter(|f| f.required) + .map(|f| f.name) + .collect(); + assert!(required.contains(&"namespace")); + assert!(required.contains(&"query")); + } + + #[test] + fn clear_namespace_schema_requires_namespace() { + let schema = schema("clear_namespace").unwrap(); + assert_eq!(schema.inputs.len(), 1); + assert_eq!(schema.inputs[0].name, "namespace"); + assert!(schema.inputs[0].required); + } +} diff --git a/src/openhuman/memory/schemas/files.rs b/src/openhuman/memory/schemas/files.rs index 53525e7b2..721ed3a81 100644 --- a/src/openhuman/memory/schemas/files.rs +++ b/src/openhuman/memory/schemas/files.rs @@ -126,3 +126,32 @@ fn handle_write_file(params: Map) -> ControllerFuture { to_json(rpc::ai_write_memory_file(payload).await?) }) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn file_schema_exposes_all_functions() { + assert_eq!(FUNCTIONS, &["list_files", "read_file", "write_file"]); + assert_eq!(controllers().len(), FUNCTIONS.len()); + } + + #[test] + fn unknown_file_schema_returns_none() { + assert!(schema("not_real").is_none()); + } + + #[test] + fn write_file_schema_requires_path_and_content() { + let schema = schema("write_file").unwrap(); + let required: Vec<&str> = schema + .inputs + .iter() + .filter(|f| f.required) + .map(|f| f.name) + .collect(); + assert!(required.contains(&"relative_path")); + assert!(required.contains(&"content")); + } +} diff --git a/src/openhuman/memory/schemas/kv_graph.rs b/src/openhuman/memory/schemas/kv_graph.rs index 0fa77eb05..9c040ac7e 100644 --- a/src/openhuman/memory/schemas/kv_graph.rs +++ b/src/openhuman/memory/schemas/kv_graph.rs @@ -267,3 +267,43 @@ fn handle_graph_query(params: Map) -> ControllerFuture { to_json(rpc::graph_query(payload).await?) }) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn kv_graph_schema_exposes_all_functions() { + assert_eq!( + FUNCTIONS, + &[ + "kv_set", + "kv_get", + "kv_delete", + "kv_list_namespace", + "graph_upsert", + "graph_query", + ] + ); + assert_eq!(controllers().len(), FUNCTIONS.len()); + } + + #[test] + fn unknown_kv_graph_schema_returns_none() { + assert!(schema("not_real").is_none()); + } + + #[test] + fn graph_upsert_schema_requires_subject_predicate_and_object() { + let schema = schema("graph_upsert").unwrap(); + let required: Vec<&str> = schema + .inputs + .iter() + .filter(|f| f.required) + .map(|f| f.name) + .collect(); + assert!(required.contains(&"subject")); + assert!(required.contains(&"predicate")); + assert!(required.contains(&"object")); + } +} diff --git a/src/openhuman/memory/schemas/learn.rs b/src/openhuman/memory/schemas/learn.rs index 666f8356c..2746b7af0 100644 --- a/src/openhuman/memory/schemas/learn.rs +++ b/src/openhuman/memory/schemas/learn.rs @@ -54,3 +54,27 @@ fn handle_learn_all(params: Map) -> ControllerFuture { to_json(rpc::memory_learn_all(payload).await?) }) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn learn_schema_only_exposes_learn_all() { + assert_eq!(FUNCTIONS, &["learn_all"]); + assert_eq!(controllers().len(), 1); + } + + #[test] + fn unknown_learn_schema_returns_none() { + assert!(schema("not_real").is_none()); + } + + #[test] + fn learn_all_schema_has_optional_namespaces_input() { + let schema = schema("learn_all").unwrap(); + assert_eq!(schema.inputs.len(), 1); + assert_eq!(schema.inputs[0].name, "namespaces"); + assert!(!schema.inputs[0].required); + } +} diff --git a/src/openhuman/memory/schemas/sync.rs b/src/openhuman/memory/schemas/sync.rs index 8ba74c7f3..4778f18af 100644 --- a/src/openhuman/memory/schemas/sync.rs +++ b/src/openhuman/memory/schemas/sync.rs @@ -85,3 +85,27 @@ fn handle_sync_all(_params: Map) -> ControllerFuture { fn handle_ingestion_status(_params: Map) -> ControllerFuture { Box::pin(async move { to_json(rpc::memory_ingestion_status().await?) }) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sync_schema_exposes_all_functions() { + assert_eq!(FUNCTIONS, &["sync_channel", "sync_all", "ingestion_status"]); + assert_eq!(controllers().len(), FUNCTIONS.len()); + } + + #[test] + fn unknown_sync_schema_returns_none() { + assert!(schema("not_real").is_none()); + } + + #[test] + fn sync_channel_schema_requires_channel_id() { + let schema = schema("sync_channel").unwrap(); + assert_eq!(schema.inputs.len(), 1); + assert_eq!(schema.inputs[0].name, "channel_id"); + assert!(schema.inputs[0].required); + } +} diff --git a/src/openhuman/memory/schemas/tool_memory.rs b/src/openhuman/memory/schemas/tool_memory.rs index 832e038cc..548e1815d 100644 --- a/src/openhuman/memory/schemas/tool_memory.rs +++ b/src/openhuman/memory/schemas/tool_memory.rs @@ -226,6 +226,52 @@ pub(super) fn schema(function: &str) -> Option { }) } +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn schema_exposes_all_tool_memory_functions() { + let functions: Vec<&str> = FUNCTIONS.to_vec(); + assert_eq!( + functions, + vec![ + "tool_rule_put", + "tool_rule_get", + "tool_rule_list", + "tool_rule_delete", + "tool_rules_for_prompt", + "tool_rules_json", + ] + ); + } + + #[test] + fn controllers_match_function_count() { + let registered = controllers(); + assert_eq!(registered.len(), FUNCTIONS.len()); + assert!(registered.iter().all(|c| c.schema.namespace == "memory")); + } + + #[test] + fn schema_returns_none_for_unknown_function() { + assert!(schema("not_real").is_none()); + } + + #[test] + fn tool_rule_put_schema_requires_tool_name_and_rule() { + let schema = schema("tool_rule_put").unwrap(); + let input_names: Vec<&str> = schema.inputs.iter().map(|f| f.name).collect(); + assert!(input_names.contains(&"tool_name")); + assert!(input_names.contains(&"rule")); + assert!(schema + .inputs + .iter() + .any(|f| f.name == "tool_name" && f.required)); + assert!(schema.inputs.iter().any(|f| f.name == "rule" && f.required)); + } +} + fn handle_tool_rule_put(params: Map) -> ControllerFuture { Box::pin(async move { let payload = parse_params::(params)?; diff --git a/src/openhuman/memory_tree/score/README.md b/src/openhuman/memory/score/README.md similarity index 100% rename from src/openhuman/memory_tree/score/README.md rename to src/openhuman/memory/score/README.md diff --git a/src/openhuman/memory_tree/score/embed/README.md b/src/openhuman/memory/score/embed/README.md similarity index 100% rename from src/openhuman/memory_tree/score/embed/README.md rename to src/openhuman/memory/score/embed/README.md diff --git a/src/openhuman/memory_tree/score/embed/cloud.rs b/src/openhuman/memory/score/embed/cloud.rs similarity index 100% rename from src/openhuman/memory_tree/score/embed/cloud.rs rename to src/openhuman/memory/score/embed/cloud.rs diff --git a/src/openhuman/memory_tree/score/embed/factory.rs b/src/openhuman/memory/score/embed/factory.rs similarity index 100% rename from src/openhuman/memory_tree/score/embed/factory.rs rename to src/openhuman/memory/score/embed/factory.rs diff --git a/src/openhuman/memory_tree/score/embed/inert.rs b/src/openhuman/memory/score/embed/inert.rs similarity index 100% rename from src/openhuman/memory_tree/score/embed/inert.rs rename to src/openhuman/memory/score/embed/inert.rs diff --git a/src/openhuman/memory_tree/score/embed/mod.rs b/src/openhuman/memory/score/embed/mod.rs similarity index 100% rename from src/openhuman/memory_tree/score/embed/mod.rs rename to src/openhuman/memory/score/embed/mod.rs diff --git a/src/openhuman/memory_tree/score/embed/ollama.rs b/src/openhuman/memory/score/embed/ollama.rs similarity index 100% rename from src/openhuman/memory_tree/score/embed/ollama.rs rename to src/openhuman/memory/score/embed/ollama.rs diff --git a/src/openhuman/memory_tree/score/extract/README.md b/src/openhuman/memory/score/extract/README.md similarity index 100% rename from src/openhuman/memory_tree/score/extract/README.md rename to src/openhuman/memory/score/extract/README.md diff --git a/src/openhuman/memory_tree/score/extract/extractor.rs b/src/openhuman/memory/score/extract/extractor.rs similarity index 98% rename from src/openhuman/memory_tree/score/extract/extractor.rs rename to src/openhuman/memory/score/extract/extractor.rs index 2e05e6f3b..9c1536262 100644 --- a/src/openhuman/memory_tree/score/extract/extractor.rs +++ b/src/openhuman/memory/score/extract/extractor.rs @@ -77,7 +77,7 @@ impl EntityExtractor for CompositeExtractor { #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory_tree::score::extract::EntityKind; + use crate::openhuman::memory::score::extract::EntityKind; #[tokio::test] async fn regex_only_extractor_works() { diff --git a/src/openhuman/memory_tree/score/extract/llm.rs b/src/openhuman/memory/score/extract/llm.rs similarity index 99% rename from src/openhuman/memory_tree/score/extract/llm.rs rename to src/openhuman/memory/score/extract/llm.rs index 1442854ac..51d075053 100644 --- a/src/openhuman/memory_tree/score/extract/llm.rs +++ b/src/openhuman/memory/score/extract/llm.rs @@ -35,7 +35,7 @@ use serde::Deserialize; use super::types::{EntityKind, ExtractedEntities, ExtractedEntity, ExtractedTopic}; use super::EntityExtractor; -use crate::openhuman::memory_tree::chat::{ChatPrompt, ChatProvider}; +use crate::openhuman::memory::chat::{ChatPrompt, ChatProvider}; // ── Configuration ──────────────────────────────────────────────────────── @@ -101,7 +101,7 @@ impl Default for LlmExtractorConfig { /// Holds an `Arc` rather than a per-instance HTTP /// client. The provider abstraction lets a single workspace choose /// cloud vs local at runtime (see -/// [`crate::openhuman::memory_tree::chat::build_chat_provider`]). Tests +/// [`crate::openhuman::memory::chat::build_chat_provider`]). Tests /// can mock the provider to assert the prompt / parse behaviour without /// a real Ollama or backend. pub struct LlmEntityExtractor { diff --git a/src/openhuman/memory_tree/score/extract/llm_tests.rs b/src/openhuman/memory/score/extract/llm_tests.rs similarity index 97% rename from src/openhuman/memory_tree/score/extract/llm_tests.rs rename to src/openhuman/memory/score/extract/llm_tests.rs index c6a7ae5d8..b72ac3da1 100644 --- a/src/openhuman/memory_tree/score/extract/llm_tests.rs +++ b/src/openhuman/memory/score/extract/llm_tests.rs @@ -193,7 +193,7 @@ async fn extract_soft_fallback_on_provider_failure() { // Provider always errors. extract() must NOT return Err — it must // return an empty ExtractedEntities with a warn log after retry // exhaustion. - use crate::openhuman::memory_tree::chat::{ChatPrompt, ChatProvider}; + use crate::openhuman::memory::chat::{ChatPrompt, ChatProvider}; use async_trait::async_trait; use std::sync::Arc; @@ -220,7 +220,7 @@ async fn extract_routes_through_chat_provider_and_parses_response() { // Mock provider returns canned NER+importance JSON. Verify the // extractor parses it, recovers spans by string search, and emits the // expected entities + importance signal. - use crate::openhuman::memory_tree::chat::{ChatPrompt, ChatProvider}; + use crate::openhuman::memory::chat::{ChatPrompt, ChatProvider}; use async_trait::async_trait; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; @@ -266,7 +266,7 @@ async fn extract_returns_empty_on_malformed_provider_response() { // Provider returns garbage. Caller must NOT see an Err — the parse // failure path returns empty entities (retrying the same input would // yield the same garbage, so we don't burn retries). - use crate::openhuman::memory_tree::chat::{ChatPrompt, ChatProvider}; + use crate::openhuman::memory::chat::{ChatPrompt, ChatProvider}; use async_trait::async_trait; use std::sync::Arc; @@ -386,7 +386,7 @@ fn into_extracted_entities_disallowed_known_kind_falls_back_to_misc() { #[test] fn build_prompt_carries_user_text_and_kind_tag() { - use crate::openhuman::memory_tree::chat::{ChatPrompt, ChatProvider}; + use crate::openhuman::memory::chat::{ChatPrompt, ChatProvider}; use async_trait::async_trait; use std::sync::Arc; diff --git a/src/openhuman/memory/score/extract/mod.rs b/src/openhuman/memory/score/extract/mod.rs new file mode 100644 index 000000000..01f90d7ae --- /dev/null +++ b/src/openhuman/memory/score/extract/mod.rs @@ -0,0 +1,63 @@ +//! Entity extraction (Phase 2 / #708). +//! +//! Exposes [`EntityExtractor`] as a pluggable interface and a default +//! [`CompositeExtractor`] that runs a chain of extractors and merges their +//! output. Phase 2 ships with the mechanical regex extractor only; semantic +//! NER (GLiNER / LLM) plugs in later without changing any call sites. + +mod extractor; +pub mod llm; +pub mod regex; +pub mod types; + +use std::sync::Arc; + +use crate::openhuman::config::Config; +use crate::openhuman::memory::chat::build_chat_runtime; + +pub use extractor::{CompositeExtractor, EntityExtractor, RegexEntityExtractor}; +pub use llm::{LlmEntityExtractor, LlmExtractorConfig}; +pub use types::{EntityKind, ExtractedEntities, ExtractedEntity, ExtractedTopic}; + +/// Build the extractor used by seal handlers to label new summary nodes. +/// +/// Composition: +/// - regex extractor — always on, mechanical, near-zero cost +/// - LLM extractor with `emit_topics: true` — added when the unified +/// summarization workload can be built from inference routing. +/// +/// Differs from [`super::ScoringConfig::from_config`] (the chunk-admission +/// builder) in two ways: returns *just* an extractor (no thresholds / +/// weights / drop logic — none of which apply at seal time), and flips +/// `emit_topics` on so summaries surface thematic labels alongside +/// entities. Leaf-side scoring is unchanged. +pub fn build_summary_extractor(config: &Config) -> Arc { + let (provider, model) = match build_chat_runtime(config) { + Ok(runtime) => runtime, + Err(err) => { + log::warn!( + "[memory_tree::extract] summary extractor: build_chat_runtime failed: \ + {err:#} — falling back to regex-only" + ); + return Arc::new(CompositeExtractor::regex_only()); + } + }; + + let cfg = LlmExtractorConfig { + model: model.clone(), + emit_topics: true, + output_language: config.output_language.clone(), + ..LlmExtractorConfig::default() + }; + + log::debug!( + "[memory_tree::extract] summary extractor: regex + LLM provider={} model={} \ + emit_topics=true", + provider.name(), + model + ); + Arc::new(CompositeExtractor::new(vec![ + Box::new(RegexEntityExtractor), + Box::new(LlmEntityExtractor::new(cfg, provider)), + ])) +} diff --git a/src/openhuman/memory_tree/score/extract/regex.rs b/src/openhuman/memory/score/extract/regex.rs similarity index 100% rename from src/openhuman/memory_tree/score/extract/regex.rs rename to src/openhuman/memory/score/extract/regex.rs diff --git a/src/openhuman/memory_tree/score/extract/types.rs b/src/openhuman/memory/score/extract/types.rs similarity index 100% rename from src/openhuman/memory_tree/score/extract/types.rs rename to src/openhuman/memory/score/extract/types.rs diff --git a/src/openhuman/memory_tree/score/mod.rs b/src/openhuman/memory/score/mod.rs similarity index 86% rename from src/openhuman/memory_tree/score/mod.rs rename to src/openhuman/memory/score/mod.rs index 8f2965252..0b8895a8e 100644 --- a/src/openhuman/memory_tree/score/mod.rs +++ b/src/openhuman/memory/score/mod.rs @@ -22,7 +22,7 @@ use serde::{Deserialize, Serialize}; use self::extract::{EntityExtractor, ExtractedEntities}; use self::resolver::{canonicalise, CanonicalEntity}; use self::signals::{ScoreSignals, SignalWeights}; -use crate::openhuman::memory_tree::types::{approx_token_count, Chunk, SourceKind}; +use crate::openhuman::memory_store::chunks::types::{approx_token_count, Chunk, SourceKind}; /// Default drop threshold. Chunks with `total < DEFAULT_DROP_THRESHOLD` /// are tombstoned and never reach the chunk store. @@ -105,29 +105,20 @@ impl ScoringConfig { } } - /// Build a [`ScoringConfig`] from the workspace [`Config`]. The - /// resolution rules match `build_summary_extractor`: + /// Build a [`ScoringConfig`] from the workspace [`Config`]. /// - /// - `llm_backend = "cloud"` (default): always wires the LLM extractor - /// against the cloud provider, using the configured - /// `cloud_llm_model` (defaulting to `summarization-v1`). - /// - `llm_backend = "local"`: wires the LLM extractor only when both - /// `llm_extractor_endpoint` and `llm_extractor_model` are set; - /// otherwise falls back to [`Self::default_regex_only`]. - /// - /// Construction errors in the chat provider (rare — only client-builder - /// failures) fall back to regex-only with a warn log; scoring never - /// blocks on LLM availability. + /// The LLM extractor follows the unified summarization workload routing. + /// Construction errors fall back to regex-only with a warn log; scoring + /// never blocks on LLM availability. pub fn from_config(config: &crate::openhuman::config::Config) -> Self { - use crate::openhuman::memory_tree::chat::{build_chat_provider, ChatConsumer}; + use crate::openhuman::memory::chat::build_chat_runtime; - let model = match extract::resolve_extractor_model(config) { - Some(m) => m, - None => { - log::debug!( - "[memory_tree::score] llm_extractor not resolvable for memory_provider={:?} \ - — using regex-only", - config.memory_provider.as_deref().unwrap_or("cloud") + let (provider, model) = match build_chat_runtime(config) { + Ok(runtime) => runtime, + Err(err) => { + log::warn!( + "[memory::score] build_chat_runtime failed: {err:#} — \ + falling back to regex-only" ); return Self::default_regex_only(); } @@ -139,23 +130,12 @@ impl ScoringConfig { ..extract::LlmExtractorConfig::default() }; - match build_chat_provider(config, ChatConsumer::Extract) { - Ok(provider) => { - log::info!( - "[memory_tree::score] using LlmEntityExtractor provider={} model={}", - provider.name(), - model - ); - Self::with_llm_extractor(Arc::new(extract::LlmEntityExtractor::new(cfg, provider))) - } - Err(err) => { - log::warn!( - "[memory_tree::score] build_chat_provider failed: {err:#} — \ - falling back to regex-only" - ); - Self::default_regex_only() - } - } + log::info!( + "[memory::score] using LlmEntityExtractor provider={} model={}", + provider.name(), + model + ); + Self::with_llm_extractor(Arc::new(extract::LlmEntityExtractor::new(cfg, provider))) } } @@ -175,7 +155,7 @@ impl ScoringConfig { /// 4. Apply final admission gate against `drop_threshold`. pub async fn score_chunk(chunk: &Chunk, cfg: &ScoringConfig) -> Result { log::debug!( - "[memory_tree::score] score_chunk chunk_id={} tokens={}", + "[memory::score] score_chunk chunk_id={} tokens={}", chunk.id, chunk.token_count ); @@ -201,7 +181,7 @@ pub async fn score_chunk(chunk: &Chunk, cfg: &ScoringConfig) -> Result Result { log::warn!( - "[memory_tree::score] LLM extractor `{}` failed: {e} — \ + "[memory::score] LLM extractor `{}` failed: {e} — \ falling back to cheap signals only", llm.name() ); @@ -231,7 +211,7 @@ pub async fn score_chunk(chunk: &Chunk, cfg: &ScoringConfig) -> Result Result Chunk { @@ -7,7 +7,7 @@ fn test_chunk(content: &str) -> Chunk { Chunk { id: chunk_id(SourceKind::Email, "t1", 0, "test-content"), content: content.to_string(), - token_count: crate::openhuman::memory_tree::types::approx_token_count(content), + token_count: crate::openhuman::memory_store::chunks::types::approx_token_count(content), metadata: meta, seq_in_source: 0, created_at: Utc::now(), diff --git a/src/openhuman/memory_tree/score/resolver.rs b/src/openhuman/memory/score/resolver.rs similarity index 97% rename from src/openhuman/memory_tree/score/resolver.rs rename to src/openhuman/memory/score/resolver.rs index 845ad0c31..4cf463d91 100644 --- a/src/openhuman/memory_tree/score/resolver.rs +++ b/src/openhuman/memory/score/resolver.rs @@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize}; -use crate::openhuman::memory_tree::score::extract::{EntityKind, ExtractedEntities}; +use crate::openhuman::memory::score::extract::{EntityKind, ExtractedEntities}; /// Canonicalised entity — same shape as [`ExtractedEntity`] plus a stable /// `canonical_id` suitable for indexing. @@ -103,7 +103,7 @@ pub fn canonical_id_for(kind: EntityKind, surface: &str) -> String { #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory_tree::score::extract::ExtractedEntity; + use crate::openhuman::memory::score::extract::ExtractedEntity; fn entity(kind: EntityKind, text: &str) -> ExtractedEntity { ExtractedEntity { @@ -173,7 +173,7 @@ mod tests { // ── Topic canonicalisation (#709 / Phase 3c topic-tree scope) ──── - use crate::openhuman::memory_tree::score::extract::ExtractedTopic; + use crate::openhuman::memory::score::extract::ExtractedTopic; fn topic(label: &str, score: f32) -> ExtractedTopic { ExtractedTopic { diff --git a/src/openhuman/memory_tree/score/signals/README.md b/src/openhuman/memory/score/signals/README.md similarity index 100% rename from src/openhuman/memory_tree/score/signals/README.md rename to src/openhuman/memory/score/signals/README.md diff --git a/src/openhuman/memory_tree/score/signals/interaction.rs b/src/openhuman/memory/score/signals/interaction.rs similarity index 96% rename from src/openhuman/memory_tree/score/signals/interaction.rs rename to src/openhuman/memory/score/signals/interaction.rs index b254cdd83..622714860 100644 --- a/src/openhuman/memory_tree/score/signals/interaction.rs +++ b/src/openhuman/memory/score/signals/interaction.rs @@ -13,7 +13,7 @@ //! Ingest adapters can attach these tags during canonicalisation when the //! upstream source supports the distinction. Absent tags → neutral score. -use crate::openhuman::memory_tree::types::Metadata; +use crate::openhuman::memory_store::chunks::types::Metadata; /// Tag set when the user replied to this message/thread. pub const TAG_REPLY: &str = "reply"; @@ -67,7 +67,7 @@ pub fn score(meta: &Metadata) -> f32 { #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory_tree::types::SourceKind; + use crate::openhuman::memory_store::chunks::types::SourceKind; use chrono::Utc; fn meta(tags: &[&str]) -> Metadata { diff --git a/src/openhuman/memory_tree/score/signals/metadata_weight.rs b/src/openhuman/memory/score/signals/metadata_weight.rs similarity index 95% rename from src/openhuman/memory_tree/score/signals/metadata_weight.rs rename to src/openhuman/memory/score/signals/metadata_weight.rs index 1c88a710d..880eb5c25 100644 --- a/src/openhuman/memory_tree/score/signals/metadata_weight.rs +++ b/src/openhuman/memory/score/signals/metadata_weight.rs @@ -8,7 +8,7 @@ //! context (e.g., channel size, thread participant count) is a future //! refinement when we actually have that metadata at ingest. -use crate::openhuman::memory_tree::types::{Metadata, SourceKind}; +use crate::openhuman::memory_store::chunks::types::{Metadata, SourceKind}; /// Base weight for each source kind. /// diff --git a/src/openhuman/memory_tree/score/signals/mod.rs b/src/openhuman/memory/score/signals/mod.rs similarity index 100% rename from src/openhuman/memory_tree/score/signals/mod.rs rename to src/openhuman/memory/score/signals/mod.rs diff --git a/src/openhuman/memory_tree/score/signals/ops.rs b/src/openhuman/memory/score/signals/ops.rs similarity index 96% rename from src/openhuman/memory_tree/score/signals/ops.rs rename to src/openhuman/memory/score/signals/ops.rs index 76407b063..7f1e1a8fe 100644 --- a/src/openhuman/memory_tree/score/signals/ops.rs +++ b/src/openhuman/memory/score/signals/ops.rs @@ -3,8 +3,8 @@ use super::{interaction, metadata_weight, source_weight, token_count, unique_words}; use super::{ScoreSignals, SignalWeights}; -use crate::openhuman::memory_tree::score::extract::ExtractedEntities; -use crate::openhuman::memory_tree::types::Metadata; +use crate::openhuman::memory::score::extract::ExtractedEntities; +use crate::openhuman::memory_store::chunks::types::Metadata; /// Compute all signals for a chunk. /// @@ -95,10 +95,10 @@ pub fn combine_cheap_only(signals: &ScoreSignals, w: &SignalWeights) -> f32 { #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory_tree::score::extract::{ + use crate::openhuman::memory::score::extract::{ EntityKind, ExtractedEntities, ExtractedEntity, }; - use crate::openhuman::memory_tree::types::SourceKind; + use crate::openhuman::memory_store::chunks::types::SourceKind; use chrono::Utc; fn meta(tags: &[&str], kind: SourceKind) -> Metadata { diff --git a/src/openhuman/memory_tree/score/signals/source_weight.rs b/src/openhuman/memory/score/signals/source_weight.rs similarity index 97% rename from src/openhuman/memory_tree/score/signals/source_weight.rs rename to src/openhuman/memory/score/signals/source_weight.rs index b65390728..b9180fa67 100644 --- a/src/openhuman/memory_tree/score/signals/source_weight.rs +++ b/src/openhuman/memory/score/signals/source_weight.rs @@ -10,7 +10,7 @@ //! Finer distinction (DM vs channel on Slack specifically) requires richer //! ingest-time metadata and is deferred. -use crate::openhuman::memory_tree::types::{DataSource, Metadata, SourceKind}; +use crate::openhuman::memory_store::chunks::types::{DataSource, Metadata, SourceKind}; const PROVIDER_PREFIX: &str = "provider:"; diff --git a/src/openhuman/memory_tree/score/signals/token_count.rs b/src/openhuman/memory/score/signals/token_count.rs similarity index 100% rename from src/openhuman/memory_tree/score/signals/token_count.rs rename to src/openhuman/memory/score/signals/token_count.rs diff --git a/src/openhuman/memory_tree/score/signals/types.rs b/src/openhuman/memory/score/signals/types.rs similarity index 58% rename from src/openhuman/memory_tree/score/signals/types.rs rename to src/openhuman/memory/score/signals/types.rs index 529136850..2aa40115a 100644 --- a/src/openhuman/memory_tree/score/signals/types.rs +++ b/src/openhuman/memory/score/signals/types.rs @@ -63,3 +63,45 @@ impl SignalWeights { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn score_signals_default_to_zero() { + let signals = ScoreSignals::default(); + assert_eq!(signals.token_count, 0.0); + assert_eq!(signals.unique_words, 0.0); + assert_eq!(signals.metadata_weight, 0.0); + assert_eq!(signals.source_weight, 0.0); + assert_eq!(signals.interaction, 0.0); + assert_eq!(signals.entity_density, 0.0); + assert_eq!(signals.llm_importance, 0.0); + } + + #[test] + fn signal_weights_default_match_expected_priorities() { + let weights = SignalWeights::default(); + assert_eq!(weights.token_count, 1.0); + assert_eq!(weights.unique_words, 1.0); + assert_eq!(weights.metadata_weight, 1.5); + assert_eq!(weights.source_weight, 1.5); + assert_eq!(weights.interaction, 3.0); + assert_eq!(weights.entity_density, 1.0); + assert_eq!(weights.llm_importance, 0.0); + } + + #[test] + fn with_llm_enabled_only_changes_llm_weight() { + let default = SignalWeights::default(); + let enabled = SignalWeights::with_llm_enabled(); + assert_eq!(enabled.token_count, default.token_count); + assert_eq!(enabled.unique_words, default.unique_words); + assert_eq!(enabled.metadata_weight, default.metadata_weight); + assert_eq!(enabled.source_weight, default.source_weight); + assert_eq!(enabled.interaction, default.interaction); + assert_eq!(enabled.entity_density, default.entity_density); + assert_eq!(enabled.llm_importance, 2.0); + } +} diff --git a/src/openhuman/memory_tree/score/signals/unique_words.rs b/src/openhuman/memory/score/signals/unique_words.rs similarity index 100% rename from src/openhuman/memory_tree/score/signals/unique_words.rs rename to src/openhuman/memory/score/signals/unique_words.rs diff --git a/src/openhuman/memory_tree/score/store.rs b/src/openhuman/memory/score/store.rs similarity index 97% rename from src/openhuman/memory_tree/score/store.rs rename to src/openhuman/memory/score/store.rs index 701635f12..140a02788 100644 --- a/src/openhuman/memory_tree/score/store.rs +++ b/src/openhuman/memory/score/store.rs @@ -14,10 +14,10 @@ use serde::{Deserialize, Serialize}; use crate::openhuman::composio::providers::profile::{is_self_identity_any_toolkit, IdentityKind}; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::score::extract::EntityKind; -use crate::openhuman::memory_tree::score::resolver::CanonicalEntity; -use crate::openhuman::memory_tree::score::signals::ScoreSignals; -use crate::openhuman::memory_tree::store::with_connection; +use crate::openhuman::memory::score::extract::EntityKind; +use crate::openhuman::memory::score::resolver::CanonicalEntity; +use crate::openhuman::memory::score::signals::ScoreSignals; +use crate::openhuman::memory_store::chunks::store::with_connection; /// Map a memory-tree `EntityKind` to the Composio identity-registry /// [`IdentityKind`] used for self-matching, or `None` for kinds that @@ -304,7 +304,7 @@ pub(crate) fn index_summary_entity_ids_tx( Some((kind, _)) => kind, None => { log::warn!( - "[memory_tree::score::store] summary entity id missing ':' — \ + "[memory::score::store] summary entity id missing ':' — \ storing as-is: {canonical_id}" ); canonical_id.as_str() diff --git a/src/openhuman/memory_tree/score/store_tests.rs b/src/openhuman/memory/score/store_tests.rs similarity index 98% rename from src/openhuman/memory_tree/score/store_tests.rs rename to src/openhuman/memory/score/store_tests.rs index dc47eb79e..ca1013678 100644 --- a/src/openhuman/memory_tree/score/store_tests.rs +++ b/src/openhuman/memory/score/store_tests.rs @@ -159,7 +159,7 @@ fn lookup_limit_respected() { /// and summary hits. See PR #789 CodeRabbit review. #[test] fn summary_entity_index_kind_is_parseable() { - use crate::openhuman::memory_tree::store::with_connection; + use crate::openhuman::memory_store::chunks::store::with_connection; let (_tmp, cfg) = test_config(); diff --git a/src/openhuman/memory/stm_recall/constants.rs b/src/openhuman/memory/stm_recall/constants.rs index 88c90b7ab..7dc520998 100644 --- a/src/openhuman/memory/stm_recall/constants.rs +++ b/src/openhuman/memory/stm_recall/constants.rs @@ -33,3 +33,25 @@ pub const TOKEN_BUDGET: usize = 6_000; /// applied at the DB level via LIMIT; we over-fetch slightly and let dedup /// finish trimming. pub const FTS5_LIMIT: usize = 20; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stm_recall_constants_match_expected_defaults() { + assert_eq!(RECENCY_WINDOW_DAYS, 14.0); + assert_eq!(RECENCY_WINDOW_MAX_SEGMENTS, 100); + assert_eq!(COSINE_GATE, 0.65); + assert_eq!(MAX_SEGMENT_RECAPS, 5); + assert_eq!(MAX_EPISODIC_TURNS, 5); + assert_eq!(TOKEN_BUDGET, 6_000); + assert_eq!(FTS5_LIMIT, 20); + } + + #[test] + fn stm_token_budget_exceeds_single_section_limits() { + assert!(TOKEN_BUDGET > MAX_SEGMENT_RECAPS); + assert!(TOKEN_BUDGET > MAX_EPISODIC_TURNS); + } +} diff --git a/src/openhuman/memory/stm_recall/mod.rs b/src/openhuman/memory/stm_recall/mod.rs index 787d4ad07..883199563 100644 --- a/src/openhuman/memory/stm_recall/mod.rs +++ b/src/openhuman/memory/stm_recall/mod.rs @@ -3,7 +3,7 @@ //! Assembles a bounded, recency-weighted context block from two arms: //! //! - **Arm 1** — FTS5 over not-yet-compacted recent episodic entries from -//! OTHER sessions. Reuses [`crate::openhuman::memory::store::fts5::episodic_cross_session_search`]. +//! OTHER sessions. Reuses [`crate::openhuman::memory_store::fts5::episodic_cross_session_search`]. //! When no user query is available (preemptive/session-start case), falls back to //! a recency selection of recent non-current-session episodic turns. //! diff --git a/src/openhuman/memory/stm_recall/recall.rs b/src/openhuman/memory/stm_recall/recall.rs index 6ddacc302..e3641273a 100644 --- a/src/openhuman/memory/stm_recall/recall.rs +++ b/src/openhuman/memory/stm_recall/recall.rs @@ -8,8 +8,8 @@ use rusqlite::{params, Connection}; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; -use crate::openhuman::memory::store::fts5; -use crate::openhuman::memory::store::fts5::EpisodicEntry; +use crate::openhuman::memory_store::fts5; +use crate::openhuman::memory_store::fts5::EpisodicEntry; use super::{ COSINE_GATE, FTS5_LIMIT, MAX_EPISODIC_TURNS, MAX_SEGMENT_RECAPS, RECENCY_WINDOW_DAYS, diff --git a/src/openhuman/memory/stm_recall/recall_tests.rs b/src/openhuman/memory/stm_recall/recall_tests.rs index ca87bbc13..ec1bad988 100644 --- a/src/openhuman/memory/stm_recall/recall_tests.rs +++ b/src/openhuman/memory/stm_recall/recall_tests.rs @@ -3,10 +3,10 @@ use super::*; use crate::openhuman::agent::harness::archivist::ArchivistHook; use crate::openhuman::agent::hooks::{PostTurnHook, TurnContext}; -use crate::openhuman::memory::store::events::EVENTS_INIT_SQL; -use crate::openhuman::memory::store::fts5; -use crate::openhuman::memory::store::profile::PROFILE_INIT_SQL; -use crate::openhuman::memory::store::segments::SEGMENTS_INIT_SQL; +use crate::openhuman::memory_store::events::EVENTS_INIT_SQL; +use crate::openhuman::memory_store::fts5; +use crate::openhuman::memory_store::profile::PROFILE_INIT_SQL; +use crate::openhuman::memory_store::segments::SEGMENTS_INIT_SQL; use parking_lot::Mutex; use rusqlite::{params, Connection}; use std::sync::Arc; @@ -209,6 +209,214 @@ fn arm2_drops_below_gate_and_accepts_above() { ); } +#[test] +fn arm2_respects_model_signature_filter() { + let conn = setup_conn(); + let now = now_ts(); + + let mut emb = vec![0.0_f32; 8]; + emb[0] = 1.0; + + let id_a = insert_episodic(&conn, "session-a", now - 120.0, "user", "matching model"); + insert_segment_with_embedding( + &conn, + "seg-match", + "session-a", + id_a, + id_a, + "Segment with the selected embedding signature", + Some(emb.clone()), + now - 100.0, + "match:model:8", + ); + + let id_b = insert_episodic(&conn, "session-b", now - 90.0, "user", "other model"); + insert_segment_with_embedding( + &conn, + "seg-other", + "session-b", + id_b, + id_b, + "Segment with a different embedding signature", + Some(emb.clone()), + now - 80.0, + "other:model:8", + ); + + let opts = StmRecallOpts { + exclude_session: "current", + query: Some("matching model"), + model_signature: Some("match:model:8"), + }; + let block = stm_recall(&conn, &opts, Some(&emb)).unwrap(); + + let recap_ids: Vec<&str> = block + .items + .iter() + .filter_map(|it| match it { + StmItem::SegmentRecap { segment_id, .. } => Some(segment_id.as_str()), + _ => None, + }) + .collect(); + assert!(recap_ids.contains(&"seg-match")); + assert!(!recap_ids.contains(&"seg-other")); +} + +#[test] +fn arm2_skips_blank_summary_rows_even_when_embedding_matches() { + let conn = setup_conn(); + let now = now_ts(); + + let mut emb = vec![0.0_f32; 8]; + emb[0] = 1.0; + + let id = insert_episodic(&conn, "session-a", now - 100.0, "user", "blank summary row"); + insert_segment_with_embedding( + &conn, + "seg-blank", + "session-a", + id, + id, + " ", + Some(emb.clone()), + now - 90.0, + "test:model:8", + ); + + let opts = StmRecallOpts { + exclude_session: "current", + query: Some("blank summary row"), + model_signature: None, + }; + let block = stm_recall(&conn, &opts, Some(&emb)).unwrap(); + + assert!( + block.items.iter().all(|it| { + !matches!(it, StmItem::SegmentRecap { segment_id, .. } if segment_id == "seg-blank") + }), + "blank summaries should be skipped during recall assembly" + ); +} + +#[test] +fn empty_query_embedding_skips_arm2_but_preserves_arm1_fts5_hits() { + let conn = setup_conn(); + let now = now_ts(); + + insert_episodic( + &conn, + "other-session", + now - 100.0, + "user", + "Rust ownership and borrowing notes", + ); + + let opts = StmRecallOpts { + exclude_session: "current-session", + query: Some("Rust ownership"), + model_signature: None, + }; + let block = stm_recall(&conn, &opts, Some(&[])).unwrap(); + + assert_eq!( + block.cosine_candidates, 0, + "empty query embedding should skip Arm 2 entirely" + ); + assert!(block.fts5_candidates > 0, "Arm 1 should still run"); + assert!( + block + .items + .iter() + .any(|it| matches!(it, StmItem::EpisodicTurn { .. })), + "FTS5 hits should still surface episodic turns when Arm 2 is skipped" + ); +} + +#[test] +fn missing_query_embedding_skips_arm2_but_preserves_arm1_fts5_hits() { + let conn = setup_conn(); + let now = now_ts(); + + insert_episodic( + &conn, + "other-session", + now - 100.0, + "user", + "Rust ownership and borrowing notes", + ); + + let opts = StmRecallOpts { + exclude_session: "current-session", + query: Some("Rust ownership"), + model_signature: None, + }; + let block = stm_recall(&conn, &opts, None).unwrap(); + + assert_eq!( + block.cosine_candidates, 0, + "missing query embedding should skip Arm 2 entirely" + ); + assert!(block.fts5_candidates > 0, "Arm 1 should still run"); + assert!( + block + .items + .iter() + .any(|it| matches!(it, StmItem::EpisodicTurn { .. })), + "FTS5 hits should still surface episodic turns when Arm 2 is skipped" + ); +} + +#[test] +fn arm2_sql_filter_excludes_null_summary_segments() { + let conn = setup_conn(); + let now = now_ts(); + + let id = insert_episodic( + &conn, + "session-null", + now - 100.0, + "user", + "null summary row", + ); + { + let c = conn.lock(); + c.execute( + "INSERT INTO conversation_segments + (segment_id, session_id, namespace, start_episodic_id, end_episodic_id, + start_timestamp, end_timestamp, turn_count, summary, status, created_at, updated_at) + VALUES (?1,?2,'global',?3,?4,?5,?5,1,NULL,'summarised',?5,?5)", + params!["seg-null", "session-null", id, id, now - 90.0], + ) + .unwrap(); + let bytes: Vec = vec![0, 0, 128, 63, 0, 0, 0, 0]; // [1.0, 0.0] + c.execute( + "INSERT INTO segment_embeddings (segment_id, model_signature, vector, dim, created_at) + VALUES (?1,?2,?3,?4,?5)", + params!["seg-null", "test:model:2", bytes, 2_i64, now - 90.0], + ) + .unwrap(); + } + + let q_emb = vec![1.0_f32, 0.0]; + let opts = StmRecallOpts { + exclude_session: "current", + query: Some("null summary row"), + model_signature: None, + }; + let block = stm_recall(&conn, &opts, Some(&q_emb)).unwrap(); + + assert_eq!( + block.cosine_candidates, 0, + "NULL summary rows should be excluded before Arm 2 scoring" + ); + assert!( + block.items.iter().all(|it| { + !matches!(it, StmItem::SegmentRecap { segment_id, .. } if segment_id == "seg-null") + }), + "NULL summary rows must never surface as segment recaps" + ); +} + // ── exclude-own-session tests ───────────────────────────────────────────────── #[test] @@ -380,6 +588,79 @@ fn dedup_drops_episodic_row_inside_segment_span() { ); } +#[test] +fn open_ended_segment_span_does_not_dedup_episodic_hits() { + let conn = setup_conn(); + let now = now_ts(); + + let covered_id = insert_episodic( + &conn, + "other-session", + now - 120.0, + "user", + "memory safety follow-up", + ); + + let emb = vec![1.0_f32, 0.0, 0.0, 0.0]; + { + let c = conn.lock(); + c.execute( + "INSERT INTO conversation_segments + (segment_id, session_id, namespace, start_episodic_id, end_episodic_id, + start_timestamp, end_timestamp, turn_count, summary, status, created_at, updated_at) + VALUES (?1,?2,'global',?3,NULL,?4,?4,1,?5,'summarised',?4,?4)", + params![ + "seg-open-ended", + "other-session", + covered_id, + now - 60.0, + "Open-ended segment recap" + ], + ) + .unwrap(); + + let bytes: Vec = emb.iter().flat_map(|f| f.to_le_bytes()).collect(); + c.execute( + "INSERT INTO segment_embeddings (segment_id, model_signature, vector, dim, created_at) + VALUES (?1,?2,?3,?4,?5)", + params![ + "seg-open-ended", + "test:model:4", + bytes, + emb.len() as i64, + now - 60.0 + ], + ) + .unwrap(); + } + + let opts = StmRecallOpts { + exclude_session: "current", + query: Some("memory safety"), + model_signature: None, + }; + let block = stm_recall(&conn, &opts, Some(&emb)).unwrap(); + + assert!( + block.items.iter().any(|it| matches!( + it, + StmItem::SegmentRecap { segment_id, .. } if segment_id == "seg-open-ended" + )), + "matching segment recap should still be returned" + ); + assert!( + block.items.iter().any(|it| matches!( + it, + StmItem::EpisodicTurn { id, .. } if *id == Some(covered_id) + )), + "open-ended spans should not deduplicate episodic hits until an end id exists" + ); + assert_eq!( + block.dropped_dedup, 0, + "dedup counter should stay at zero for open-ended spans" + ); +} + // ── recency window bound test ───────────────────────────────────────────────── #[test] @@ -448,6 +729,91 @@ fn recency_window_excludes_old_segments() { ); } +#[test] +fn old_fts5_hit_is_filtered_out_of_stm_results() { + let conn = setup_conn(); + let now = now_ts(); + let old_ts = now - (super::super::RECENCY_WINDOW_DAYS + 2.0) * 86_400.0; + + insert_episodic( + &conn, + "old-session", + old_ts, + "user", + "legacy rust ownership note", + ); + + let opts = StmRecallOpts { + exclude_session: "current", + query: Some("legacy rust ownership"), + model_signature: None, + }; + let block = stm_recall(&conn, &opts, None).unwrap(); + + assert!( + block.fts5_candidates > 0, + "FTS5 should still find the historical match before STM recency filtering" + ); + assert!( + block.items.is_empty(), + "old FTS5 hits must be removed from the final STM block" + ); +} + +#[test] +fn merge_prefers_more_recent_item_across_segment_and_episodic_arms() { + let conn = setup_conn(); + let now = now_ts(); + + let old_id = insert_episodic( + &conn, + "episodic-session", + now - 300.0, + "user", + "shared keyword older turn", + ); + let recent_id = insert_episodic( + &conn, + "segment-session", + now - 200.0, + "assistant", + "shared keyword recap source", + ); + + let emb = vec![1.0_f32, 0.0, 0.0, 0.0]; + insert_segment_with_embedding( + &conn, + "seg-recent-first", + "segment-session", + recent_id, + recent_id, + "Recent recap for the shared keyword thread", + Some(emb.clone()), + now - 50.0, + "test:model:4", + ); + + let opts = StmRecallOpts { + exclude_session: "current", + query: Some("shared keyword"), + model_signature: None, + }; + let block = stm_recall(&conn, &opts, Some(&emb)).unwrap(); + + assert!( + block.items.len() >= 2, + "expected one recap and one uncovered episodic turn in the merged block" + ); + assert!(matches!( + &block.items[0], + StmItem::SegmentRecap { segment_id, .. } if segment_id == "seg-recent-first" + )); + assert!(matches!( + &block.items[1], + StmItem::EpisodicTurn { id, .. } if *id == Some(old_id) + )); +} + // ── token budget test ───────────────────────────────────────────────────────── #[test] @@ -579,6 +945,52 @@ fn render_empty_block_returns_empty_string() { assert!(block.is_empty()); } +#[test] +fn decode_vector_blob_rejects_misaligned_bytes() { + let decoded = super::decode_vector_blob(&[1_u8, 2, 3]); + assert!( + decoded.is_empty(), + "malformed blobs should be discarded instead of partially decoded" + ); +} + +#[test] +fn age_days_from_ts_is_zero_for_future_timestamps() { + let future = now_ts() + 86_400.0; + assert_eq!(super::age_days_from_ts(future), 0.0); +} + +#[test] +fn render_includes_recaps_and_episodic_turn_labels() { + let block = StmRecallBlock { + items: vec![ + StmItem::SegmentRecap { + segment_id: "seg-1".into(), + session_id: "thread-a".into(), + summary: "Summary text".into(), + start_episodic_id: 10, + end_episodic_id: Some(12), + updated_at: now_ts() - 60.0, + cosine: 0.9, + }, + StmItem::EpisodicTurn { + id: Some(42), + session_id: "thread-b".into(), + timestamp: now_ts() - 30.0, + role: "user".into(), + content: "Turn text".into(), + }, + ], + ..Default::default() + }; + + let rendered = block.render(); + assert!(rendered.contains("**Conversation recap**")); + assert!(rendered.contains("Summary text")); + assert!(rendered.contains("**user**")); + assert!(rendered.contains("Turn text")); +} + // ── end-to-end integration test ─────────────────────────────────────────────── // Drive the real chain: completed turns → episodic rows → segment close // (recap + embedding via the Phase 0+1 path using stub providers) → @@ -586,7 +998,7 @@ fn render_empty_block_returns_empty_string() { #[tokio::test] async fn e2e_stm_recall_chain() { - use crate::openhuman::memory_tree::chat::ChatPrompt; + use crate::openhuman::memory::chat::ChatPrompt; let conn = setup_conn(); @@ -597,7 +1009,7 @@ async fn e2e_stm_recall_chain() { // requiring a live LLM or Ollama daemon. struct StubChat; - use crate::openhuman::memory_tree::chat::ChatProvider; + use crate::openhuman::memory::chat::ChatProvider; #[async_trait::async_trait] impl ChatProvider for StubChat { fn name(&self) -> &str { @@ -611,10 +1023,9 @@ async fn e2e_stm_recall_chain() { } } - use crate::openhuman::memory_tree::score::embed::InertEmbedder; - let chat_provider: Arc = - Arc::new(StubChat); - let embedder: Arc = + use crate::openhuman::memory::score::embed::InertEmbedder; + let chat_provider: Arc = Arc::new(StubChat); + let embedder: Arc = Arc::new(InertEmbedder::new()); let archivist = ArchivistHook::new_with_stubs(conn.clone(), chat_provider, embedder); diff --git a/src/openhuman/memory/store/README.md b/src/openhuman/memory/store/README.md deleted file mode 100644 index 40c7b7a9c..000000000 --- a/src/openhuman/memory/store/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Memory store - -Storage backend for the memory subsystem. Houses the SQLite + FTS5 + vector + graph implementation (`UnifiedMemory`), the async client handle used by RPC controllers (`MemoryClient`), the `Memory` trait impl bridging both, and the factory functions used to bootstrap a memory instance. - -## Files - -- **`mod.rs`** — module root; re-exports `UnifiedMemory`, `MemoryClient`, factory functions, and the public types from `types.rs`. -- **`types.rs`** — public input/output structs (`NamespaceDocumentInput`, `NamespaceMemoryHit`, `NamespaceRetrievalContext`, `RetrievalScoreBreakdown`, `MemoryItemKind`, `StoredMemoryDocument`, `MemoryKvRecord`, `GraphRelationRecord`) plus the `GLOBAL_NAMESPACE` sentinel. -- **`client.rs`** — `MemoryClient` / `MemoryClientRef` / `MemoryState`. Async wrapper around `UnifiedMemory` that owns the singleton ingestion queue and exposes the surface called by RPC handlers (`put_doc`, `ingest_doc`, `query_namespace`, `recall_namespace_*`, `kv_*`, `graph_*`, skill-sync helpers). Always local — no remote sync. -- **`client_tests.rs`** — coverage for the client-facing storage and graph round-trips against a fresh temp workspace. -- **`factories.rs`** — `create_memory*` constructors that select the embedding provider from `MemoryConfig` and instantiate `UnifiedMemory`. `effective_memory_backend_name` always reports `"namespace"`. -- **`memory_trait.rs`** — `impl Memory for UnifiedMemory`, mapping the generic trait surface (`store`, `recall`, `get`, `list`, `forget`, `namespace_summaries`) onto the unified store. Includes namespace normalisation and episodic-session augmentation. -- **`../safety/`** — shared secret-detection + redaction helpers used by memory write paths (documents, KV, episodic) to prevent credentials/tokens from being persisted into long-lived memory. -- **`unified/`** — the SQLite implementation, broken into per-table submodules. See `unified/README.md`. - -## How it fits - -Callers (RPC controllers, the agent harness, learning pipelines) interact with `MemoryClient`. The client delegates persistence to `UnifiedMemory` and offloads heavier work (chunk + embed + graph extraction) to the singleton `IngestionQueue` defined in `../ingestion/`. Generic consumers that just need `Memory` trait behaviour go through `memory_trait.rs`. diff --git a/src/openhuman/memory/store/agentmemory/README.md b/src/openhuman/memory/store/agentmemory/README.md deleted file mode 100644 index f99e78bbc..000000000 --- a/src/openhuman/memory/store/agentmemory/README.md +++ /dev/null @@ -1,108 +0,0 @@ -# agentmemory backend - -Optional `Memory` backend that delegates every trait call to a locally-running -[agentmemory](https://github.com/rohitg00/agentmemory) REST server (default -`http://localhost:3111`). - -Selected via: - -```toml -[memory] -backend = "agentmemory" -``` - -The default backend stays `sqlite`; selecting `"agentmemory"` is opt-in and -non-breaking for existing configs. - -## Why another backend (and why not via MCP tools) - -OpenHuman already supports MCP servers as tools, but that path is -agent-facing — the LLM picks `recall` / `save` as tool calls per turn. The -`Memory` trait is the *host-facing* surface: harness, archivist, reflection, -prompt-section builders all consume it directly, without going through -tool-call latency. - -Plugging in as a `Memory` backend lets agentmemory back things like -`mem::context` injection in `loadMemoryRules`, reflection passes, and the -namespace summaries that drive agent-side discovery — none of which would -route through MCP today. - -It also lets operators who self-host agentmemory across multiple agents -(Claude Code, Cursor, Codex, OpenCode, plus OpenHuman) share a single -durable memory. - -## Config keys - -| Field | Default | Purpose | -|---|---|---| -| `agentmemory_url` | `http://localhost:3111` | Base URL for the agentmemory REST server | -| `agentmemory_secret` | `None` | Optional HMAC bearer token sent as `Authorization: Bearer ` | -| `agentmemory_timeout_ms` | `5000` | Per-request reqwest timeout | - -When `backend == "agentmemory"`, the existing `embedding_provider` / -`embedding_model` / `embedding_dimensions` fields are **ignored** — -agentmemory owns its own embedding stack via `~/.agentmemory/.env`. Setting -them on this path is a no-op. - -## Field mapping - -| OpenHuman `MemoryEntry` | agentmemory wire | -|---|---| -| `namespace` | `project` (defaults to `"default"` if empty) | -| `key` | `title` | -| `content` | `content` | -| `MemoryCategory::Core` | `type: "fact"` | -| `MemoryCategory::Daily` | `type: "conversation"` | -| `MemoryCategory::Conversation` | `type: "conversation"` | -| `MemoryCategory::Custom(s)` | `type: "fact"`, `concepts: [s]` | -| `session_id` | `sessionIds: [...]` | -| `timestamp` | `updatedAt` (RFC3339), falling back to `createdAt` | -| `score` (recall hits) | smart-search `score` | - -agentmemory has additional fields (`concepts`, `files`, `strength`, -`version`, `supersedes`) that this backend leaves at defaults — they're -internal to agentmemory's lifecycle layer. - -## Trait method → endpoint - -| `Memory` method | agentmemory REST | -|---|---| -| `store` | `POST /agentmemory/remember` | -| `recall` | `POST /agentmemory/smart-search` (hybrid BM25 + vector + graph) | -| `get` | `POST /agentmemory/smart-search` then exact-title filter | -| `list` | `GET /agentmemory/memories?latest=true&project=` | -| `forget` | `get(ns, key)` → `POST /agentmemory/forget` with the id | -| `namespace_summaries` | `GET /agentmemory/projects` | -| `count` | `GET /agentmemory/health` (`memories` field) | -| `health_check` | `GET /agentmemory/livez` | - -`RecallOpts.category` / `session_id` / `min_score` are applied as -client-side filters on the smart-search response (agentmemory's REST -surface doesn't expose them as server-side filters today). - -## Security: plaintext-bearer guard - -When `agentmemory_secret` is set, the client refuses to send the token to a -non-loopback host over `http://`. Loopback (`localhost`, `127.0.0.1`, `::1`) -+ plaintext is allowed for local dev; everything else needs `https://` or -the daemon must be reachable on loopback. - -Set `AGENTMEMORY_REQUIRE_HTTPS=1` as a process env var to harden this from -"warn on stderr" to "refuse to construct the client" — useful in -production where a misconfigured TLS terminator should fail loud rather -than leak the secret once. - -## Failure modes - -| Failure | Behaviour | -|---|---| -| agentmemory daemon down at construction time | `health_check()` returns false; trait methods bubble up the `reqwest` transport error | -| Network timeout | Returns `anyhow::Error` per trait contract; surfaces to caller | -| 4xx / 5xx response | Returns `anyhow::Error` with the response status + body snippet | -| Bearer over plaintext HTTP non-loopback | Warns on stderr (matches agentmemory's own client guard from v0.9.12 PR #315) | -| Bearer over plaintext HTTP + `AGENTMEMORY_REQUIRE_HTTPS=1` | Hard refusal at construction time | - -No automatic fallback to `sqlite` — if the daemon is down at boot, the -service fails loud. Operators flip back to `backend = "sqlite"` in config -to recover. Rationale (per issue #1664 alignment): "private, simple, -predictable" — a silent SQLite fallback hides a misconfigured daemon. diff --git a/src/openhuman/memory/store/agentmemory/backend.rs b/src/openhuman/memory/store/agentmemory/backend.rs deleted file mode 100644 index b8295812b..000000000 --- a/src/openhuman/memory/store/agentmemory/backend.rs +++ /dev/null @@ -1,342 +0,0 @@ -//! `impl Memory for AgentMemoryBackend` — the hot path for OpenHuman <→ -//! agentmemory traffic. -//! -//! The upstream agentmemory REST contract (endpoints, payloads, lifecycle -//! semantics) lives at . This -//! module pins the OpenHuman-visible projection of that contract; the -//! field-mapping table is in `mapping.rs` and the security guard is in -//! `client.rs`. - -use anyhow::Result; -use async_trait::async_trait; - -use crate::openhuman::config::MemoryConfig; -use crate::openhuman::memory::traits::{ - Memory, MemoryCategory, MemoryEntry, NamespaceSummary, RecallOpts, -}; - -use super::client::AgentMemoryClient; -use super::mapping::{ - ForgetRequest, ForgetResponse, HealthResponse, MemoriesResponse, ProjectsResponse, - RememberRequest, RememberResponse, SmartSearchRequest, SmartSearchResponse, WireMemory, - DEFAULT_PROJECT, -}; - -/// Memory backend that proxies every trait call through agentmemory's REST -/// surface. Construct via [`AgentMemoryBackend::from_config`]. -pub struct AgentMemoryBackend { - client: AgentMemoryClient, -} - -impl AgentMemoryBackend { - /// Build from a [`MemoryConfig`]. Reads the optional - /// `agentmemory_url` / `agentmemory_secret` / `agentmemory_timeout_ms` - /// fields and falls back to documented defaults - /// (`http://localhost:3111`, no secret, 5000ms timeout). - pub fn from_config(config: &MemoryConfig) -> Result { - let client = AgentMemoryClient::new( - config.agentmemory_url.as_deref(), - config.agentmemory_secret.as_deref(), - config.agentmemory_timeout_ms, - )?; - log::debug!( - "[memory::agentmemory] backend initialised against {}", - client.base() - ); - Ok(Self { client }) - } -} - -fn namespace_or_default(ns: &str) -> &str { - if ns.is_empty() { - DEFAULT_PROJECT - } else { - ns - } -} - -/// Lookup cap for `get()` / `forget()` exact-title resolution. -/// -/// agentmemory does not expose a `(project, title)` lookup endpoint, so -/// `get()` fans out via smart-search and filters client-side for an exact -/// title match. A small cap (e.g. 5) drops valid exact matches that rank -/// lower in BM25+vector score. 100 is high enough that an exact title -/// never falls off the page in practice while keeping the response -/// payload bounded. -const EXACT_LOOKUP_LIMIT: usize = 100; - -#[async_trait] -impl Memory for AgentMemoryBackend { - fn name(&self) -> &str { - "agentmemory" - } - - async fn store( - &self, - namespace: &str, - key: &str, - content: &str, - category: MemoryCategory, - session_id: Option<&str>, - ) -> Result<()> { - log::debug!( - "[memory::agentmemory] store namespace={namespace:?} key={key:?} session_id={session_id:?} category={category:?}" - ); - let body = RememberRequest::build( - namespace_or_default(namespace), - key, - content, - &category, - session_id, - ); - let _: RememberResponse = self.client.post_json("agentmemory/remember", &body).await?; - Ok(()) - } - - async fn recall( - &self, - query: &str, - limit: usize, - opts: RecallOpts<'_>, - ) -> Result> { - log::debug!( - "[memory::agentmemory] recall query={query:?} limit={limit} namespace={:?} category={:?} session={:?} min_score={:?} cross_session={}", - opts.namespace, opts.category, opts.session_id, opts.min_score, opts.cross_session, - ); - // Cross-session recall (#1505) is a no-op for the agentmemory - // backend today: the smart-search endpoint already searches - // across the workspace's project namespace. The same-user/ - // cross-chat continuity path for cloud-backed memory continues - // to flow through the conversational transcript ingestion - // pipeline (`learning::transcript_ingest`) which writes durable - // facts under the `conversation_memory` namespace and is picked - // up by the existing `[Prior conversations]` block. - let _ = opts.cross_session; - let project = opts.namespace.map(|s| { - if s.is_empty() { - DEFAULT_PROJECT.to_string() - } else { - s.to_string() - } - }); - let body = SmartSearchRequest { - query: query.to_string(), - limit, - project, - }; - let resp: SmartSearchResponse = self - .client - .post_json("agentmemory/smart-search", &body) - .await?; - - let mut entries: Vec = resp - .results - .into_iter() - .map(WireMemory::into_entry) - .collect(); - let before = entries.len(); - - if let Some(cat) = opts.category.as_ref() { - entries.retain(|e| &e.category == cat); - } - if let Some(session) = opts.session_id { - entries.retain(|e| e.session_id.as_deref() == Some(session)); - } - if let Some(min_score) = opts.min_score { - // Scoreless rows (e.g. direct fetches that never went through - // smart-search) cannot prove they meet the threshold — drop - // them rather than letting them through silently. - entries.retain(|e| e.score.is_some_and(|s| s >= min_score)); - } - if entries.len() != before { - log::trace!( - "[memory::agentmemory] recall client-filter retained {}/{} hits", - entries.len(), - before, - ); - } - Ok(entries) - } - - async fn get(&self, namespace: &str, key: &str) -> Result> { - log::debug!("[memory::agentmemory] get namespace={namespace:?} key={key:?}"); - let project = namespace_or_default(namespace); - let body = SmartSearchRequest { - query: key.to_string(), - limit: EXACT_LOOKUP_LIMIT, - project: Some(project.to_string()), - }; - let resp: SmartSearchResponse = self - .client - .post_json("agentmemory/smart-search", &body) - .await?; - let hit = resp - .results - .into_iter() - .find(|r| r.title.as_deref() == Some(key)) - .map(WireMemory::into_entry); - log::trace!( - "[memory::agentmemory] get namespace={namespace:?} key={key:?} matched={}", - hit.is_some() - ); - Ok(hit) - } - - async fn list( - &self, - namespace: Option<&str>, - category: Option<&MemoryCategory>, - session_id: Option<&str>, - ) -> Result> { - log::debug!( - "[memory::agentmemory] list namespace={namespace:?} category={category:?} session_id={session_id:?}" - ); - // When the caller passes Some(""), normalise to the "default" - // project so the wire query stays consistent. When they pass - // None, list across every project — matching the trait's - // optional-namespace contract. - let path = match namespace { - Some(ns) => format!( - "agentmemory/memories?latest=true&project={}", - url_encode(namespace_or_default(ns)) - ), - None => "agentmemory/memories?latest=true".to_string(), - }; - let resp: MemoriesResponse = self.client.get_json(&path).await?; - let mut entries: Vec = resp - .memories - .into_iter() - .map(WireMemory::into_entry) - .collect(); - let before = entries.len(); - if let Some(cat) = category { - entries.retain(|e| &e.category == cat); - } - if let Some(session) = session_id { - entries.retain(|e| e.session_id.as_deref() == Some(session)); - } - if entries.len() != before { - log::trace!( - "[memory::agentmemory] list client-filter retained {}/{} rows", - entries.len(), - before, - ); - } - Ok(entries) - } - - async fn forget(&self, namespace: &str, key: &str) -> Result { - log::debug!("[memory::agentmemory] forget namespace={namespace:?} key={key:?}"); - // agentmemory's /forget takes an id, not (project, title). Look - // the key up first via smart-search (mirrors `get` above), then - // POST /forget against that id. If no exact title match exists, - // return Ok(false) — same contract as the SQLite backend's - // delete-by-(namespace, key). - let Some(target) = self.get(namespace, key).await? else { - log::trace!( - "[memory::agentmemory] forget namespace={namespace:?} key={key:?} unresolved -> noop" - ); - return Ok(false); - }; - let body = ForgetRequest { - id: target.id.clone(), - }; - let resp: ForgetResponse = self.client.post_json("agentmemory/forget", &body).await?; - log::debug!( - "[memory::agentmemory] forget namespace={namespace:?} key={key:?} id={} forgotten={}", - target.id, - resp.forgotten, - ); - Ok(resp.forgotten) - } - - async fn namespace_summaries(&self) -> Result> { - log::debug!("[memory::agentmemory] namespace_summaries"); - let resp: ProjectsResponse = self.client.get_json("agentmemory/projects").await?; - Ok(resp - .projects - .into_iter() - .map(|p| NamespaceSummary { - namespace: p.name, - count: p.count, - last_updated: p.last_updated, - }) - .collect()) - } - - async fn count(&self) -> Result { - log::debug!("[memory::agentmemory] count"); - let resp: HealthResponse = self.client.get_json("agentmemory/health").await?; - Ok(resp.memories.unwrap_or(0)) - } - - async fn health_check(&self) -> bool { - let ok = self.client.livez().await; - log::debug!("[memory::agentmemory] health_check ok={ok}"); - ok - } -} - -/// Minimal `application/x-www-form-urlencoded` style encoder for query-string -/// values. We only need to escape `/`, `?`, `#`, `&`, `=`, `+`, space, and -/// non-ASCII bytes — anything else can pass through unencoded. This avoids -/// pulling in `percent-encoding` for one call site. -fn url_encode(s: &str) -> String { - let mut out = String::with_capacity(s.len()); - for ch in s.chars() { - match ch { - 'A'..='Z' | 'a'..='z' | '0'..='9' | '-' | '_' | '.' | '~' => out.push(ch), - _ => { - let mut buf = [0u8; 4]; - for byte in ch.encode_utf8(&mut buf).as_bytes() { - out.push_str(&format!("%{byte:02X}")); - } - } - } - } - out -} - -/// Probe whether an agentmemory daemon is reachable at the configured URL. -/// Used by the factory at startup so a `backend = "agentmemory"` config -/// against a daemon that isn't running can fail loud at boot rather than -/// silently swallow every store/recall call. -pub async fn probe_agentmemory_reachable(config: &MemoryConfig) -> Result<()> { - let client = AgentMemoryClient::new( - config.agentmemory_url.as_deref(), - config.agentmemory_secret.as_deref(), - config.agentmemory_timeout_ms, - )?; - if !client.livez().await { - anyhow::bail!( - "agentmemory daemon is not reachable at {} \ - (set MemoryConfig.backend = \"sqlite\" to fall back to the local store; \ - see https://github.com/rohitg00/agentmemory for daemon setup)", - client.base() - ); - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn url_encode_passes_through_safe_chars() { - assert_eq!(url_encode("plain_text-123.~"), "plain_text-123.~"); - } - - #[test] - fn url_encode_percent_escapes_specials() { - assert_eq!(url_encode("a b"), "a%20b"); - assert_eq!(url_encode("a/b"), "a%2Fb"); - assert_eq!(url_encode("a&b=c"), "a%26b%3Dc"); - } - - #[test] - fn url_encode_handles_unicode() { - // 中文 → utf-8 bytes E4 B8 AD E6 96 87 - assert_eq!(url_encode("中文"), "%E4%B8%AD%E6%96%87"); - } -} diff --git a/src/openhuman/memory/store/agentmemory/client.rs b/src/openhuman/memory/store/agentmemory/client.rs deleted file mode 100644 index 671a11ee5..000000000 --- a/src/openhuman/memory/store/agentmemory/client.rs +++ /dev/null @@ -1,331 +0,0 @@ -//! HTTP client wrapper for the agentmemory REST server. -//! -//! Wraps `reqwest::Client` with the agentmemory base URL, optional bearer -//! token, a configurable per-request timeout, and a plaintext-bearer guard -//! that refuses to send the secret over `http://` per the -//! v0.9.12 contract from upstream agentmemory PR #315 (see -//! ). - -use anyhow::{anyhow, Context, Result}; -use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION, CONTENT_TYPE}; -use reqwest::{Method, StatusCode}; -use serde::de::DeserializeOwned; -use serde::Serialize; -use std::time::Duration; -use url::Url; - -/// Default agentmemory REST endpoint when no override is configured. -pub const DEFAULT_AGENTMEMORY_URL: &str = "http://localhost:3111"; - -/// Default per-request timeout. Generous enough to absorb a cold-start on -/// the iii engine + a vector recall round-trip on a 100k-memory store. -pub const DEFAULT_TIMEOUT_MS: u64 = 5_000; - -/// Thin HTTP wrapper around the agentmemory REST surface. -pub struct AgentMemoryClient { - http: reqwest::Client, - base: Url, - secret: Option, -} - -impl AgentMemoryClient { - /// Builds a client configured for the given URL + optional secret + - /// timeout. The plaintext-bearer guard fires here, before any request - /// goes on the wire — a misconfigured deploy fails loud at - /// construction time rather than silently leaking the token. - pub fn new(url: Option<&str>, secret: Option<&str>, timeout_ms: Option) -> Result { - let raw = url.unwrap_or(DEFAULT_AGENTMEMORY_URL).trim(); - if raw.is_empty() { - return Err(anyhow!( - "agentmemory_url cannot be empty — leave it unset to use {DEFAULT_AGENTMEMORY_URL}" - )); - } - let parsed = Url::parse(raw) - .with_context(|| format!("agentmemory_url is not a valid URL: {raw}"))?; - - if let Some(secret) = secret.filter(|s| !s.trim().is_empty()) { - enforce_plaintext_bearer_guard(&parsed, secret)?; - } - - let timeout = Duration::from_millis(timeout_ms.unwrap_or(DEFAULT_TIMEOUT_MS)); - let http = reqwest::Client::builder() - .timeout(timeout) - .build() - .context("failed to build reqwest client for agentmemory backend")?; - - Ok(Self { - http, - base: parsed, - secret: secret - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()), - }) - } - - /// Base URL (mostly for log lines + error context). - pub fn base(&self) -> &Url { - &self.base - } - - /// `GET /`. Returns a 404 as `Ok(None)` for the get-by-key - /// shape; everything else surfaces as an error. - pub async fn get_optional(&self, path: &str) -> Result> { - let url = self.url_for(path)?; - log::trace!("[memory::agentmemory] GET {url}"); - let resp = self - .http - .request(Method::GET, url.clone()) - .headers(self.headers()?) - .send() - .await - .with_context(|| format!("GET {url}"))?; - - match resp.status() { - StatusCode::NOT_FOUND => Ok(None), - s if s.is_success() => { - Ok(Some(resp.json::().await.with_context(|| { - format!("failed to decode JSON response from GET {url}") - })?)) - } - s => Err(decode_error(&url, s, resp.text().await.ok())), - } - } - - /// `GET /` expecting a 200 + JSON body. - pub async fn get_json(&self, path: &str) -> Result { - self.get_optional(path) - .await? - .ok_or_else(|| anyhow!("agentmemory returned 404 for GET {path}")) - } - - /// `POST /` with a JSON body, expecting a 2xx response. - pub async fn post_json( - &self, - path: &str, - body: &B, - ) -> Result { - let url = self.url_for(path)?; - log::trace!("[memory::agentmemory] POST {url}"); - let resp = self - .http - .request(Method::POST, url.clone()) - .headers(self.headers()?) - .json(body) - .send() - .await - .with_context(|| format!("POST {url}"))?; - - let status = resp.status(); - if !status.is_success() { - return Err(decode_error(&url, status, resp.text().await.ok())); - } - resp.json::() - .await - .with_context(|| format!("failed to decode JSON response from POST {url}")) - } - - /// `GET /agentmemory/livez` — booleanises the health check. - pub async fn livez(&self) -> bool { - let Ok(url) = self.url_for("agentmemory/livez") else { - return false; - }; - let Ok(headers) = self.headers() else { - return false; - }; - match self - .http - .request(Method::GET, url) - .headers(headers) - .send() - .await - { - Ok(resp) => resp.status().is_success(), - Err(_) => false, - } - } - - fn url_for(&self, path: &str) -> Result { - let mut joined = self.base.clone(); - let trimmed = path.trim_start_matches('/'); - // Split off `?query` so it doesn't get appended as a literal path - // segment — `path_segments_mut().extend(split('/'))` would - // percent-encode the `?` and the server would 404. - let (path_part, query_part) = match trimmed.split_once('?') { - Some((p, q)) => (p, Some(q)), - None => (trimmed, None), - }; - joined - .path_segments_mut() - .map_err(|_| anyhow!("agentmemory base URL cannot be a base: {}", self.base))? - .pop_if_empty() - .extend(path_part.split('/')); - if let Some(q) = query_part { - joined.set_query(Some(q)); - } - Ok(joined) - } - - fn headers(&self) -> Result { - let mut h = HeaderMap::new(); - h.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); - if let Some(secret) = &self.secret { - let value = format!("Bearer {secret}"); - let header = HeaderValue::from_str(&value) - .context("agentmemory_secret is not a valid HTTP header value")?; - h.insert(AUTHORIZATION, header); - } - Ok(h) - } -} - -/// Mirrors the v0.9.12 plaintext-bearer guard from agentmemory's first-party -/// integration plugins: a bearer token must never cross plaintext HTTP to a -/// non-loopback host. Loopback (`localhost`, `127.0.0.1`, `::1`) is exempt; -/// `https://` is exempt. `AGENTMEMORY_REQUIRE_HTTPS=1` escalates the warning -/// path to a hard refusal even on loopback so a misconfigured production -/// deploy can fail loud rather than leak the secret once. -fn enforce_plaintext_bearer_guard(url: &Url, _secret: &str) -> Result<()> { - if url.scheme().eq_ignore_ascii_case("https") { - return Ok(()); - } - let host = url.host_str().unwrap_or(""); - let loopback = matches!(host, "localhost" | "127.0.0.1" | "::1" | "[::1]"); - let require_https = std::env::var("AGENTMEMORY_REQUIRE_HTTPS") - .map(|v| v == "1" || v.eq_ignore_ascii_case("true")) - .unwrap_or(false); - - if require_https && url.scheme() != "https" { - return Err(anyhow!( - "agentmemory_secret is set and AGENTMEMORY_REQUIRE_HTTPS=1 \ - refuses to send the bearer over scheme `{}` (host {host}). \ - Switch agentmemory_url to https:// or unset AGENTMEMORY_REQUIRE_HTTPS.", - url.scheme(), - )); - } - - if !loopback { - log::warn!( - "[memory::agentmemory] agentmemory_secret is set and agentmemory_url ({url}) \ - is plaintext HTTP to a non-loopback host ({host}). The bearer will be \ - observable on the wire. Set AGENTMEMORY_REQUIRE_HTTPS=1 to make this a \ - hard error, or switch to https://." - ); - } - Ok(()) -} - -fn decode_error(url: &Url, status: StatusCode, body: Option) -> anyhow::Error { - let body = body.unwrap_or_default(); - let snippet = if body.len() > 512 { - // Snap to the previous char boundary so we never slice through - // the middle of a multi-byte UTF-8 scalar — an emoji or accented - // character at byte 512 would otherwise panic the error-decode - // path with `byte index 512 is not a char boundary`, defeating - // the whole point of this helper. - let mut end = 512; - while end > 0 && !body.is_char_boundary(end) { - end -= 1; - } - format!("{}…", &body[..end]) - } else { - body - }; - anyhow!( - "agentmemory returned {status} for {url}: {snippet}", - snippet = snippet.trim() - ) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::sync::Mutex; - - /// Tests in this module mutate the process-global - /// `AGENTMEMORY_REQUIRE_HTTPS` env var, which is not thread-safe under - /// cargo's default parallel test runner. Serialise them through a - /// shared mutex so a stray `set_var` from one test can't race with a - /// `remove_var` in another (and so the cleanup path runs even when a - /// test panics mid-way through, via the mutex guard's `Drop`). - fn env_lock() -> std::sync::MutexGuard<'static, ()> { - static LOCK: Mutex<()> = Mutex::new(()); - LOCK.lock().unwrap_or_else(|e| e.into_inner()) - } - - struct EnvGuard { - prev: Option, - } - - impl EnvGuard { - fn set(value: &str) -> Self { - let prev = std::env::var_os("AGENTMEMORY_REQUIRE_HTTPS"); - // SAFETY: env mutation is wrapped because Rust 2024 marks it - // unsafe; the call is gated by the env_lock() critical section - // so no other test in this module is observing the env - // concurrently. - unsafe { std::env::set_var("AGENTMEMORY_REQUIRE_HTTPS", value) }; - Self { prev } - } - - fn remove() -> Self { - let prev = std::env::var_os("AGENTMEMORY_REQUIRE_HTTPS"); - unsafe { std::env::remove_var("AGENTMEMORY_REQUIRE_HTTPS") }; - Self { prev } - } - } - - impl Drop for EnvGuard { - fn drop(&mut self) { - // SAFETY: still under the same env_lock() critical section. - unsafe { - match self.prev.take() { - Some(v) => std::env::set_var("AGENTMEMORY_REQUIRE_HTTPS", v), - None => std::env::remove_var("AGENTMEMORY_REQUIRE_HTTPS"), - } - } - } - } - - #[test] - fn loopback_plaintext_is_allowed_without_require_https() { - let _lock = env_lock(); - let _guard = EnvGuard::remove(); - let url = Url::parse("http://localhost:3111").unwrap(); - assert!(enforce_plaintext_bearer_guard(&url, "secret").is_ok()); - - let url = Url::parse("http://127.0.0.1:3111").unwrap(); - assert!(enforce_plaintext_bearer_guard(&url, "secret").is_ok()); - } - - #[test] - fn https_is_always_allowed_even_with_require_https() { - let _lock = env_lock(); - let _guard = EnvGuard::set("1"); - let url = Url::parse("https://memory.example.com").unwrap(); - assert!(enforce_plaintext_bearer_guard(&url, "secret").is_ok()); - } - - #[test] - fn plaintext_non_loopback_with_require_https_is_refused() { - let _lock = env_lock(); - let _guard = EnvGuard::set("1"); - let url = Url::parse("http://memory.example.com:3111").unwrap(); - let err = enforce_plaintext_bearer_guard(&url, "secret").unwrap_err(); - assert!( - err.to_string().contains("refuses"), - "expected refusal, got: {err}" - ); - } - - #[test] - fn decode_error_does_not_panic_on_long_unicode_body() { - // Build a body whose byte length crosses the 512 boundary mid - // multi-byte scalar — pre-fix this would panic with - // `byte index 512 is not a char boundary`. - let unicode = "ü".repeat(400); // each "ü" is 2 bytes → 800 bytes total - let url = Url::parse("http://127.0.0.1/x").unwrap(); - let err = decode_error(&url, StatusCode::BAD_REQUEST, Some(unicode)); - let msg = err.to_string(); - assert!(msg.contains("400"), "expected status in message: {msg}"); - } -} diff --git a/src/openhuman/memory/store/agentmemory/mapping.rs b/src/openhuman/memory/store/agentmemory/mapping.rs deleted file mode 100644 index ce8a4bfa9..000000000 --- a/src/openhuman/memory/store/agentmemory/mapping.rs +++ /dev/null @@ -1,291 +0,0 @@ -//! Maps OpenHuman `MemoryEntry` / `MemoryCategory` to the agentmemory REST -//! wire shapes and back. -//! -//! Wire contract: — see the -//! upstream README for the full endpoint list and field semantics. -//! -//! agentmemory has a richer wire shape (concepts, files, strength, version, -//! supersedes) that the backend leaves at defaults — those fields are -//! internal to agentmemory's lifecycle layer and don't need to round-trip -//! through OpenHuman's trait. We map the OpenHuman-visible columns and let -//! agentmemory own the rest. - -use crate::openhuman::memory::traits::{MemoryCategory, MemoryEntry}; -use serde::{Deserialize, Serialize}; - -/// Globally well-known "default" project name used when an OpenHuman caller -/// doesn't pass a namespace. Matches the trait's `GLOBAL_NAMESPACE` semantics. -pub const DEFAULT_PROJECT: &str = "default"; - -/// agentmemory's per-memory `type` field. `MemoryCategory::Core` maps to -/// "fact", everything `MemoryCategory::Daily` / `Conversation` maps to -/// "conversation", and `Custom(s)` maps to "fact" with `s` rolled into the -/// `concepts` array so it remains queryable. -fn category_to_type(category: &MemoryCategory) -> &'static str { - match category { - MemoryCategory::Core | MemoryCategory::Custom(_) => "fact", - MemoryCategory::Daily | MemoryCategory::Conversation => "conversation", - } -} - -fn type_to_category(t: Option<&str>, concepts: &[String]) -> MemoryCategory { - match t { - Some("conversation") => MemoryCategory::Conversation, - Some("fact") | None => { - if let Some(first) = concepts.first() { - MemoryCategory::Custom(first.clone()) - } else { - MemoryCategory::Core - } - } - Some(other) => MemoryCategory::Custom(other.to_string()), - } -} - -/// Outgoing payload for `POST /agentmemory/remember`. -/// -/// Owned fields rather than borrowed slices so the value remains -/// `Send + 'static`-friendly when handed to an async runtime / event bus. -#[derive(Debug, Clone, Serialize)] -pub struct RememberRequest { - pub project: String, - pub title: String, - pub content: String, - #[serde(rename = "type")] - pub kind: String, - pub concepts: Vec, - #[serde(skip_serializing_if = "Option::is_none", rename = "sessionIds")] - pub session_ids: Option>, -} - -impl RememberRequest { - pub fn build( - namespace: &str, - key: &str, - content: &str, - category: &MemoryCategory, - session_id: Option<&str>, - ) -> Self { - let concepts = match category { - MemoryCategory::Custom(s) => vec![s.clone()], - _ => Vec::new(), - }; - let project = if namespace.is_empty() { - DEFAULT_PROJECT.to_string() - } else { - namespace.to_string() - }; - Self { - project, - title: key.to_string(), - content: content.to_string(), - kind: category_to_type(category).to_string(), - concepts, - session_ids: session_id.map(|s| vec![s.to_string()]), - } - } -} - -/// Outgoing payload for `POST /agentmemory/smart-search`. -#[derive(Debug, Clone, Serialize)] -pub struct SmartSearchRequest { - pub query: String, - pub limit: usize, - #[serde(skip_serializing_if = "Option::is_none")] - pub project: Option, -} - -/// Outgoing payload for `POST /agentmemory/forget`. -#[derive(Debug, Clone, Serialize)] -pub struct ForgetRequest { - pub id: String, -} - -/// Generic agentmemory memory row. agentmemory carries more fields than this -/// — we only deserialise what OpenHuman's `MemoryEntry` needs, leaving the -/// rest in a flatten-bag if a future caller wants them. -#[derive(Debug, Clone, Deserialize)] -pub struct WireMemory { - pub id: String, - #[serde(default)] - pub project: Option, - #[serde(default)] - pub title: Option, - #[serde(default)] - pub content: Option, - #[serde(default, rename = "type")] - pub kind: Option, - #[serde(default)] - pub concepts: Vec, - #[serde(default, rename = "sessionIds")] - pub session_ids: Vec, - #[serde(default, rename = "updatedAt")] - pub updated_at: Option, - #[serde(default, rename = "createdAt")] - pub created_at: Option, - /// Present on smart-search hits, absent on direct fetches. - #[serde(default)] - pub score: Option, -} - -impl WireMemory { - /// Project the wire row into an OpenHuman `MemoryEntry`. `key` falls - /// back to the agentmemory `id` when no title is present — for raw - /// observation rows that never went through `remember`. - pub fn into_entry(self) -> MemoryEntry { - let timestamp = self - .updated_at - .or(self.created_at) - .unwrap_or_else(|| "1970-01-01T00:00:00Z".to_string()); - let category = type_to_category(self.kind.as_deref(), &self.concepts); - let key = self.title.unwrap_or_else(|| self.id.clone()); - let session_id = self.session_ids.into_iter().next(); - MemoryEntry { - id: self.id, - key, - content: self.content.unwrap_or_default(), - namespace: self.project, - category, - timestamp, - session_id, - score: self.score, - } - } -} - -/// `POST /agentmemory/smart-search` response envelope. The `mode` field is -/// either `"full"` or `"compact"` depending on the requested `format`; both -/// modes share the same `results` array shape. -#[derive(Debug, Clone, Deserialize)] -pub struct SmartSearchResponse { - #[serde(default)] - pub results: Vec, -} - -/// `GET /agentmemory/memories` response envelope. -#[derive(Debug, Clone, Deserialize)] -pub struct MemoriesResponse { - #[serde(default)] - pub memories: Vec, -} - -/// `GET /agentmemory/health` response envelope. agentmemory returns a much -/// richer payload; we only need the `memories` count. -#[derive(Debug, Clone, Deserialize)] -pub struct HealthResponse { - #[serde(default)] - pub memories: Option, -} - -/// `GET /agentmemory/projects` response envelope. -#[derive(Debug, Clone, Deserialize)] -pub struct ProjectsResponse { - #[serde(default)] - pub projects: Vec, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct ProjectSummary { - pub name: String, - #[serde(default)] - pub count: usize, - #[serde(default, rename = "lastUpdated")] - pub last_updated: Option, -} - -/// `POST /agentmemory/remember` returns the saved memory's id. -#[derive(Debug, Clone, Deserialize)] -pub struct RememberResponse { - pub id: String, -} - -/// `POST /agentmemory/forget` returns whether anything was removed. -#[derive(Debug, Clone, Deserialize)] -pub struct ForgetResponse { - #[serde(default)] - pub forgotten: bool, -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn remember_request_maps_core_to_fact() { - let req = RememberRequest::build( - "demo", - "auth-stack", - "uses HMAC bearer tokens", - &MemoryCategory::Core, - None, - ); - assert_eq!(req.kind, "fact"); - assert!(req.concepts.is_empty()); - assert_eq!(req.project, "demo"); - assert_eq!(req.title, "auth-stack"); - assert!(req.session_ids.is_none()); - } - - #[test] - fn remember_request_maps_custom_into_concepts() { - let req = RememberRequest::build( - "demo", - "k", - "v", - &MemoryCategory::Custom("ops".into()), - Some("ses-1"), - ); - assert_eq!(req.kind, "fact"); - assert_eq!(req.concepts, vec!["ops".to_string()]); - assert_eq!(req.session_ids.as_deref(), Some(&["ses-1".to_string()][..])); - } - - #[test] - fn remember_request_falls_back_to_default_project_on_empty_namespace() { - let req = RememberRequest::build("", "k", "v", &MemoryCategory::Core, None); - assert_eq!(req.project, DEFAULT_PROJECT); - } - - #[test] - fn wire_memory_into_entry_preserves_score_on_search_hits() { - let wire = WireMemory { - id: "mem_1".into(), - project: Some("demo".into()), - title: Some("auth-stack".into()), - content: Some("uses HMAC".into()), - kind: Some("fact".into()), - concepts: vec!["auth".into()], - session_ids: vec!["ses-1".into()], - updated_at: Some("2026-05-14T00:00:00Z".into()), - created_at: None, - score: Some(0.87), - }; - let entry = wire.into_entry(); - assert_eq!(entry.id, "mem_1"); - assert_eq!(entry.key, "auth-stack"); - assert_eq!(entry.namespace.as_deref(), Some("demo")); - assert_eq!(entry.session_id.as_deref(), Some("ses-1")); - assert_eq!(entry.score, Some(0.87)); - assert_eq!(entry.category, MemoryCategory::Custom("auth".into())); - } - - #[test] - fn wire_memory_into_entry_falls_back_to_id_when_title_missing() { - let wire = WireMemory { - id: "mem_2".into(), - project: None, - title: None, - content: None, - kind: Some("conversation".into()), - concepts: vec![], - session_ids: vec![], - updated_at: None, - created_at: Some("2026-05-14T00:00:00Z".into()), - score: None, - }; - let entry = wire.into_entry(); - assert_eq!(entry.key, "mem_2"); - assert_eq!(entry.category, MemoryCategory::Conversation); - assert_eq!(entry.timestamp, "2026-05-14T00:00:00Z"); - } -} diff --git a/src/openhuman/memory/store/agentmemory/mod.rs b/src/openhuman/memory/store/agentmemory/mod.rs deleted file mode 100644 index 531da31e3..000000000 --- a/src/openhuman/memory/store/agentmemory/mod.rs +++ /dev/null @@ -1,29 +0,0 @@ -//! # AgentMemory Backend -//! -//! Thin REST adapter that implements OpenHuman's `Memory` trait against a -//! locally-running [agentmemory](https://github.com/rohitg00/agentmemory) -//! server. Selected via `MemoryConfig.backend = "agentmemory"`; the default -//! backend stays `sqlite` and the rest of the codebase is unaffected. -//! -//! Embedding model selection is owned by agentmemory's own config -//! (`~/.agentmemory/.env`) — OpenHuman's `embedding_provider` / -//! `embedding_model` / `embedding_dimensions` fields are ignored when this -//! backend is selected, because the agentmemory daemon does its own hybrid -//! BM25 + vector + graph retrieval and would otherwise re-embed every -//! incoming payload against a mismatched dim. -//! -//! See `agentmemory/README.md` for setup + the env-var contract. - -mod backend; -mod client; -mod mapping; - -pub use backend::AgentMemoryBackend; -pub use client::DEFAULT_AGENTMEMORY_URL; - -/// Returns the documented default base URL for the agentmemory daemon -/// (`http://localhost:3111`). Exposed for log lines / errors so callers -/// don't have to import the constant by name. -pub fn agentmemory_default_url() -> &'static str { - DEFAULT_AGENTMEMORY_URL -} diff --git a/src/openhuman/memory/tool_memory/test_helpers.rs b/src/openhuman/memory/tool_memory/test_helpers.rs deleted file mode 100644 index 0b73b2fd5..000000000 --- a/src/openhuman/memory/tool_memory/test_helpers.rs +++ /dev/null @@ -1,107 +0,0 @@ -//! Shared test infrastructure for the tool-scoped memory layer. -//! -//! Only compiled under `#[cfg(test)]`. - -use std::collections::HashMap; - -use async_trait::async_trait; -use parking_lot::Mutex; - -use crate::openhuman::memory::{Memory, MemoryCategory, MemoryEntry, NamespaceSummary, RecallOpts}; - -/// Minimal in-memory [`Memory`] backend for unit tests. -/// -/// Stores entries in a `HashMap` keyed by `(namespace, key)`. All methods -/// that are not needed by the store/capture tests are no-ops. -#[derive(Default)] -pub struct MockMemory { - pub entries: Mutex>, -} - -#[async_trait] -impl Memory for MockMemory { - fn name(&self) -> &str { - "mock" - } - async fn store( - &self, - namespace: &str, - key: &str, - content: &str, - category: MemoryCategory, - session_id: Option<&str>, - ) -> anyhow::Result<()> { - self.entries.lock().insert( - (namespace.to_string(), key.to_string()), - MemoryEntry { - id: format!("{namespace}/{key}"), - key: key.to_string(), - content: content.to_string(), - namespace: Some(namespace.to_string()), - category, - timestamp: "now".into(), - session_id: session_id.map(str::to_string), - score: None, - }, - ); - Ok(()) - } - async fn recall( - &self, - _query: &str, - _limit: usize, - _opts: RecallOpts<'_>, - ) -> anyhow::Result> { - Ok(Vec::new()) - } - async fn get(&self, namespace: &str, key: &str) -> anyhow::Result> { - Ok(self - .entries - .lock() - .get(&(namespace.to_string(), key.to_string())) - .cloned()) - } - async fn list( - &self, - namespace: Option<&str>, - _category: Option<&MemoryCategory>, - _session_id: Option<&str>, - ) -> anyhow::Result> { - let lock = self.entries.lock(); - Ok(match namespace { - Some(ns) => lock - .iter() - .filter(|((n, _), _)| n == ns) - .map(|(_, v)| v.clone()) - .collect(), - None => lock.iter().map(|(_, v)| v.clone()).collect(), - }) - } - async fn forget(&self, namespace: &str, key: &str) -> anyhow::Result { - Ok(self - .entries - .lock() - .remove(&(namespace.to_string(), key.to_string())) - .is_some()) - } - async fn namespace_summaries(&self) -> anyhow::Result> { - let mut counts: HashMap = HashMap::new(); - for ((ns, _), _) in self.entries.lock().iter() { - *counts.entry(ns.clone()).or_default() += 1; - } - Ok(counts - .into_iter() - .map(|(namespace, count)| NamespaceSummary { - namespace, - count, - last_updated: None, - }) - .collect()) - } - async fn count(&self) -> anyhow::Result { - Ok(self.entries.lock().len()) - } - async fn health_check(&self) -> bool { - true - } -} diff --git a/src/openhuman/memory_tree/rpc.rs b/src/openhuman/memory/tree_rpc.rs similarity index 71% rename from src/openhuman/memory_tree/rpc.rs rename to src/openhuman/memory/tree_rpc.rs index 75af2c881..72893b478 100644 --- a/src/openhuman/memory_tree/rpc.rs +++ b/src/openhuman/memory/tree_rpc.rs @@ -11,15 +11,15 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::canonicalize::{ - chat::ChatBatch, document::DocumentInput, email::EmailThread, -}; -use crate::openhuman::memory_tree::ingest::{ +use crate::openhuman::memory::ingest_pipeline::{ ingest_chat as do_ingest_chat, ingest_document as do_ingest_document, ingest_email as do_ingest_email, IngestResult, }; -use crate::openhuman::memory_tree::store::{self, ListChunksQuery}; -use crate::openhuman::memory_tree::types::{Chunk, SourceKind}; +use crate::openhuman::memory_store::chunks::store::{self as chunk_store, ListChunksQuery}; +use crate::openhuman::memory_store::chunks::types::{Chunk, SourceKind}; +use crate::openhuman::memory_sync::canonicalize::{ + chat::ChatBatch, document::DocumentInput, email::EmailThread, +}; use crate::rpc::RpcOutcome; /// Unified ingest request. The `payload` shape is adapter-specific and is @@ -58,7 +58,7 @@ pub async fn ingest_rpc( } = req; log::debug!( - "[memory_tree::rpc] ingest kind={} source_id={}", + "[memory::rpc] ingest kind={} source_id={}", source_kind.as_str(), source_id ); @@ -141,7 +141,7 @@ pub async fn list_chunks_rpc( }; let rows = tokio::task::spawn_blocking({ let config = config.clone(); - move || store::list_chunks(&config, &query) + move || chunk_store::list_chunks(&config, &query) }) .await .map_err(|e| format!("list_chunks join error: {e}"))? @@ -174,7 +174,7 @@ pub async fn get_chunk_rpc( let id = req.id.clone(); let chunk = tokio::task::spawn_blocking({ let config = config.clone(); - move || store::get_chunk(&config, &id) + move || chunk_store::get_chunk(&config, &id) }) .await .map_err(|e| format!("get_chunk join error: {e}"))? @@ -219,7 +219,7 @@ pub async fn trigger_digest_rpc( config: &Config, req: TriggerDigestRequest, ) -> Result, String> { - use crate::openhuman::memory_tree::jobs; + use crate::openhuman::memory::jobs; use chrono::{Duration as ChronoDuration, NaiveDate, Utc}; let date = match req @@ -276,13 +276,13 @@ pub struct BackfillStatusResponse { pub async fn backfill_status_rpc( config: &Config, ) -> Result, String> { - log::debug!("[memory_tree::rpc] backfill_status: entry"); + log::debug!("[memory::rpc] backfill_status: entry"); // SQLite I/O off the async runtime thread, matching the sibling // DB-backed handlers in this module (`get_chunk_rpc`, etc.). let pending_jobs: u64 = tokio::task::spawn_blocking({ let config = config.clone(); move || { - store::with_connection(&config, |conn| { + chunk_store::with_connection(&config, |conn| { let n: i64 = conn.query_row( "SELECT COUNT(*) FROM mem_tree_jobs WHERE kind = 'reembed_backfill' AND status IN ('ready', 'running')", @@ -297,11 +297,10 @@ pub async fn backfill_status_rpc( .map_err(|e| format!("memory_backfill_status join error: {e}"))? .map_err(|e| { let msg = format!("memory_backfill_status: {e}"); - log::debug!("[memory_tree::rpc] backfill_status: error: {msg}"); + log::debug!("[memory::rpc] backfill_status: error: {msg}"); msg })?; - let in_progress = - crate::openhuman::memory_tree::jobs::backfill_in_progress() || pending_jobs > 0; + let in_progress = crate::openhuman::memory::jobs::backfill_in_progress() || pending_jobs > 0; Ok(RpcOutcome::single_log( BackfillStatusResponse { in_progress, @@ -314,8 +313,11 @@ pub async fn backfill_status_rpc( #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory_tree::jobs::store::count_total; + use crate::openhuman::memory::jobs::store::count_total; + use crate::openhuman::memory_store::chunks::types::SourceKind; + use crate::openhuman::memory_sync::canonicalize::document::DocumentInput; use chrono::{Duration as ChronoDuration, Utc}; + use serde_json::json; use tempfile::TempDir; fn test_config() -> (TempDir, Config) { @@ -328,6 +330,157 @@ mod tests { (tmp, cfg) } + fn sample_document(title: &str, body: &str) -> DocumentInput { + DocumentInput { + provider: "notion".into(), + title: title.into(), + body: body.into(), + modified_at: Utc::now(), + source_ref: Some("notion://page/launch".into()), + } + } + + #[tokio::test] + async fn ingest_document_roundtrip_lists_and_gets_chunks() { + let (_tmp, cfg) = test_config(); + let outcome = ingest_rpc( + &cfg, + IngestRequest { + source_kind: SourceKind::Document, + source_id: "doc-launch".into(), + owner: "alice".into(), + tags: vec!["launch".into()], + payload: serde_json::to_value(sample_document( + "Launch Plan", + "Phoenix launch canary checklist with rollback steps.", + )) + .unwrap(), + }, + ) + .await + .unwrap(); + assert_eq!(outcome.value.source_id, "doc-launch"); + assert_eq!(outcome.value.chunks_dropped, 0); + assert!(!outcome.value.chunk_ids.is_empty()); + + let listed = list_chunks_rpc( + &cfg, + ListChunksRequest { + source_kind: Some("document".into()), + source_id: Some("doc-launch".into()), + owner: Some("alice".into()), + limit: Some(10), + ..Default::default() + }, + ) + .await + .unwrap() + .value + .chunks; + assert_eq!(listed.len(), outcome.value.chunks_written); + assert!(listed + .iter() + .all(|chunk| chunk.metadata.source_kind == SourceKind::Document)); + assert!(listed + .iter() + .any(|chunk| chunk.content.contains("Phoenix launch canary checklist"))); + + let fetched = get_chunk_rpc( + &cfg, + GetChunkRequest { + id: outcome.value.chunk_ids[0].clone(), + }, + ) + .await + .unwrap() + .value + .chunk + .expect("chunk should exist"); + assert_eq!(fetched.id, outcome.value.chunk_ids[0]); + assert_eq!(fetched.metadata.source_id, "doc-launch"); + assert_eq!(fetched.metadata.owner, "alice"); + } + + #[tokio::test] + async fn ingest_document_is_idempotent_for_duplicate_source_id() { + let (_tmp, cfg) = test_config(); + let req = IngestRequest { + source_kind: SourceKind::Document, + source_id: "doc-dup".into(), + owner: "alice".into(), + tags: vec![], + payload: serde_json::to_value(sample_document("Launch Plan", "First body")).unwrap(), + }; + + let first = ingest_rpc(&cfg, req.clone()).await.unwrap().value; + let second = ingest_rpc(&cfg, req).await.unwrap().value; + assert!(first.chunks_written > 0); + assert!(!first.already_ingested); + assert_eq!(second.chunks_written, 0); + assert!(second.already_ingested); + + let listed = list_chunks_rpc( + &cfg, + ListChunksRequest { + source_id: Some("doc-dup".into()), + limit: Some(10), + ..Default::default() + }, + ) + .await + .unwrap() + .value + .chunks; + assert_eq!(listed.len(), first.chunks_written); + } + + #[tokio::test] + async fn ingest_rpc_rejects_invalid_document_payload() { + let (_tmp, cfg) = test_config(); + let err = ingest_rpc( + &cfg, + IngestRequest { + source_kind: SourceKind::Document, + source_id: "doc-invalid".into(), + owner: String::new(), + tags: vec![], + payload: json!({"title": "Missing body"}), + }, + ) + .await + .unwrap_err(); + assert!(err.contains("invalid document payload")); + } + + #[tokio::test] + async fn list_chunks_rejects_unknown_source_kind() { + let (_tmp, cfg) = test_config(); + let err = list_chunks_rpc( + &cfg, + ListChunksRequest { + source_kind: Some("nonsense".into()), + ..Default::default() + }, + ) + .await + .unwrap_err(); + assert!(err.contains("unknown source kind: nonsense")); + } + + #[tokio::test] + async fn get_chunk_returns_none_for_missing_id() { + let (_tmp, cfg) = test_config(); + let outcome = get_chunk_rpc( + &cfg, + GetChunkRequest { + id: "missing-chunk".into(), + }, + ) + .await + .unwrap(); + assert!(outcome.value.chunk.is_none()); + } + #[tokio::test] async fn trigger_digest_with_explicit_date_enqueues() { let (_tmp, cfg) = test_config(); @@ -389,7 +542,7 @@ mod tests { /// underlying flag is a process-global shared across parallel tests. #[tokio::test] async fn backfill_status_reports_pending_jobs() { - use crate::openhuman::memory_tree::jobs; + use crate::openhuman::memory::jobs; let (_tmp, cfg) = test_config(); let s0 = backfill_status_rpc(&cfg).await.unwrap().value; diff --git a/src/openhuman/memory_tree/util/README.md b/src/openhuman/memory/util/README.md similarity index 100% rename from src/openhuman/memory_tree/util/README.md rename to src/openhuman/memory/util/README.md diff --git a/src/openhuman/memory_tree/util/mod.rs b/src/openhuman/memory/util/mod.rs similarity index 100% rename from src/openhuman/memory_tree/util/mod.rs rename to src/openhuman/memory/util/mod.rs diff --git a/src/openhuman/memory_tree/util/redact.rs b/src/openhuman/memory/util/redact.rs similarity index 100% rename from src/openhuman/memory_tree/util/redact.rs rename to src/openhuman/memory/util/redact.rs diff --git a/src/openhuman/memory_archivist/README.md b/src/openhuman/memory_archivist/README.md new file mode 100644 index 000000000..701ad22cf --- /dev/null +++ b/src/openhuman/memory_archivist/README.md @@ -0,0 +1,60 @@ +# memory_archivist + +Bridge from chat conversation to memory tree. One responsibility: take a +sequence of turns, drop tool-call noise, and append the cleaned blob to +a tree as a single leaf. The tree handles everything downstream +(summarisation, retrieval, vector embedding). + +## Flow + +```text +Vec (raw conversation, tool calls inline) + │ + ▼ +clip::clean_conversation() (drop tool_calls_json, drop "tool" turns) + │ + ▼ +compose::compose_conversation_md() (one md blob: ## role\n\n... per turn) + │ + ▼ +tree_writer::archive_to_tree() (append_leaf to memory_tree) + │ + ▼ +memory_store::trees (cascade seal, summary nodes) +``` + +## API + +| Function | Purpose | +| --- | --- | +| `clean_conversation(&[Turn]) -> Vec` | Pure transform — strips `tool_calls_json` and drops `role == "tool"` turns. | +| `compose_conversation_md(&[Turn]) -> String` | Pure transform — yields the markdown blob that becomes one tree leaf. | +| `archive_to_tree(config, &Tree, session_id, &[Turn]) -> TreeWriteOutcome` | End-to-end: clean → compose → `append_leaf`. Returns sealed-summary ids from any cascade. | + +## Layout + +| Path | Role | +| --- | --- | +| [`mod.rs`](mod.rs) | Module root + re-exports. | +| [`types.rs`](types.rs) | `Turn { role, content, tool_calls_json, timestamp }`. | +| [`clip.rs`](clip.rs) | `clean_conversation` + tests. | +| [`compose.rs`](compose.rs) | `compose_conversation_md` + tests. | +| [`tree_writer.rs`](tree_writer.rs) | `archive_to_tree` — the end-to-end orchestration. Chunk id = `sha256(session_id ‖ md)[..32]`. | + +## Why "clip"? + +Tool-call JSON is verbose, model-specific, and rarely meaningful out of +context. Tool-result turns are noisy (stdout dumps, JSON responses) and +distort vector embeddings of the surrounding human conversation. +Stripping both before the conversation lands in the tree keeps +summaries focused on natural-language content. + +## Layer rules + +- Depends only on `memory_store::trees` (the `Tree` type) and + `memory_tree::tree::bucket_seal::append_leaf` (the write contract). +- No SQLite. No on-disk md storage of its own — the tree owns + persistence. +- Replaces the legacy `unified::fts5` per-turn capture path. Callers + migrating from the archivist hook should batch turns and call + `archive_to_tree` at conversation boundaries. diff --git a/src/openhuman/memory_archivist/clip.rs b/src/openhuman/memory_archivist/clip.rs new file mode 100644 index 000000000..938f5d650 --- /dev/null +++ b/src/openhuman/memory_archivist/clip.rs @@ -0,0 +1,70 @@ +//! Drop tool-call payloads from a conversation. +//! +//! Pure transform — no IO, no allocation beyond the result vec. + +use crate::openhuman::memory_archivist::types::Turn; + +/// Return a new conversation with every turn's `tool_calls_json` stripped +/// to `None`. Also drops tool-role turns entirely (their content is the +/// tool result, which is noisy and rarely useful out of context). +pub fn clean_conversation(turns: &[Turn]) -> Vec { + turns + .iter() + .filter(|t| t.role != "tool") + .map(|t| Turn { + role: t.role.clone(), + content: t.content.clone(), + tool_calls_json: None, + timestamp: t.timestamp, + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + + fn t(role: &str, content: &str, tool_calls: Option<&str>) -> Turn { + Turn { + role: role.into(), + content: content.into(), + tool_calls_json: tool_calls.map(|s| s.into()), + timestamp: Utc::now(), + } + } + + #[test] + fn drops_tool_calls_json_on_assistant_turns() { + let convo = vec![ + t("user", "what's the time?", None), + t("assistant", "let me check", Some(r#"[{"name":"clock"}]"#)), + ]; + let cleaned = clean_conversation(&convo); + assert_eq!(cleaned.len(), 2); + assert!(cleaned[1].tool_calls_json.is_none()); + assert_eq!(cleaned[1].content, "let me check"); + } + + #[test] + fn drops_tool_role_turns_entirely() { + let convo = vec![ + t("user", "list files", None), + t("assistant", "running ls", Some(r#"[{"name":"bash"}]"#)), + t("tool", "a.txt\nb.txt", None), + t("assistant", "two files", None), + ]; + let cleaned = clean_conversation(&convo); + assert_eq!(cleaned.len(), 3); + assert!(cleaned.iter().all(|t| t.role != "tool")); + } + + #[test] + fn preserves_user_and_system_turns_unchanged() { + let convo = vec![t("system", "be brief", None), t("user", "hi", None)]; + let cleaned = clean_conversation(&convo); + assert_eq!(cleaned.len(), 2); + assert_eq!(cleaned[0].role, "system"); + assert_eq!(cleaned[1].content, "hi"); + } +} diff --git a/src/openhuman/memory_archivist/compose.rs b/src/openhuman/memory_archivist/compose.rs new file mode 100644 index 000000000..083684aff --- /dev/null +++ b/src/openhuman/memory_archivist/compose.rs @@ -0,0 +1,58 @@ +//! Compose a cleaned conversation into a single markdown blob. +//! +//! The output is the body of one tree leaf — newline-separated `## role` +//! sections with the turn content underneath. Plain markdown; no YAML +//! front-matter (the tree leaf already carries timestamps + provenance). + +use crate::openhuman::memory_archivist::types::Turn; + +pub fn compose_conversation_md(turns: &[Turn]) -> String { + let mut out = String::new(); + for (idx, turn) in turns.iter().enumerate() { + if idx > 0 { + out.push('\n'); + } + out.push_str("## "); + out.push_str(&turn.role); + out.push('\n'); + out.push_str(&turn.content); + if !turn.content.ends_with('\n') { + out.push('\n'); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + + fn t(role: &str, content: &str) -> Turn { + Turn { + role: role.into(), + content: content.into(), + tool_calls_json: None, + timestamp: Utc::now(), + } + } + + #[test] + fn empty_input_gives_empty_string() { + assert_eq!(compose_conversation_md(&[]), ""); + } + + #[test] + fn role_headings_separate_turns() { + let md = compose_conversation_md(&[t("user", "hi"), t("assistant", "hello")]); + assert!(md.contains("## user\nhi\n")); + assert!(md.contains("## assistant\nhello\n")); + } + + #[test] + fn turns_separated_by_blank_line() { + let md = compose_conversation_md(&[t("user", "a"), t("user", "b")]); + // turn boundaries get one blank line between them + assert!(md.contains("a\n\n## user\nb")); + } +} diff --git a/src/openhuman/memory_archivist/mod.rs b/src/openhuman/memory_archivist/mod.rs new file mode 100644 index 000000000..12edd45e4 --- /dev/null +++ b/src/openhuman/memory_archivist/mod.rs @@ -0,0 +1,53 @@ +//! Memory archivist — chat conversation → tree. +//! +//! The archivist's one job is to take a chat conversation, strip the +//! noisy tool-call payloads from it, and push the resulting text into a +//! memory tree as a single leaf. The tree owns persistence + retrieval +//! from there on. +//! +//! ## Flow +//! +//! ```text +//! Vec (raw conversation, tool calls included) +//! │ +//! ▼ +//! clip::clean() (strip tool_calls_json; keep role + content) +//! │ +//! ▼ +//! compose::md() (one md blob: ## role\n\n\n... per turn) +//! │ +//! ▼ +//! memory_tree::TreeWriteRequest +//! │ +//! ▼ +//! memory_store::trees (append_leaf + cascade seal) +//! ``` +//! +//! ## API +//! +//! - [`Turn`] — input shape, one per role/content/tool_calls record. +//! - [`clean_conversation`] — pure transform; returns a `Vec` with +//! tool-call payloads dropped. +//! - [`compose_conversation_md`] — pure transform; returns the markdown +//! blob that will become a single tree leaf. +//! - [`archive_to_tree`] — end-to-end: clean → compose → append leaf to +//! the named tree via `memory_tree`. +//! +//! ## Why "clip"? +//! +//! Tool-call JSON is verbose, model-specific, and rarely meaningful out +//! of context. Stripping it before the conversation lands in the tree +//! keeps summaries focused on natural-language content and keeps the +//! vector embedding signal clean. + +pub mod clip; +pub mod compose; +pub mod store; +pub mod tree_writer; +pub mod types; + +pub use clip::clean_conversation; +pub use compose::compose_conversation_md; +pub use store::{record_turn, session_entries}; +pub use tree_writer::archive_to_tree; +pub use types::{ArchivedTurn, Turn}; diff --git a/src/openhuman/memory_archivist/store.rs b/src/openhuman/memory_archivist/store.rs new file mode 100644 index 000000000..b19f913b4 --- /dev/null +++ b/src/openhuman/memory_archivist/store.rs @@ -0,0 +1,279 @@ +//! Disk-backed archivist store. +//! +//! Layout: +//! ```text +//! /episodic//.md +//! ``` +//! +//! Writes use the same atomic tempfile+rename contract as +//! `memory_store::content::atomic::write_if_new`, with one important +//! difference: we want to *append* turns to a session, so the seq is +//! computed from the existing directory contents on each call. + +use std::fs; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; + +use crate::openhuman::config::Config; +use crate::openhuman::memory_archivist::types::ArchivedTurn; +use crate::openhuman::memory_store::content::atomic::write_if_new; + +const EPISODIC_DIR: &str = "episodic"; + +fn session_dir(config: &Config, session_id: &str) -> PathBuf { + config + .memory_tree_content_root() + .join(EPISODIC_DIR) + .join(sanitize_session(session_id)) +} + +fn sanitize_session(s: &str) -> String { + s.chars() + .map(|c| { + if c.is_alphanumeric() || c == '-' || c == '_' { + c + } else { + '_' + } + }) + .collect() +} + +fn next_seq(dir: &Path) -> u32 { + let mut max = -1i64; + if let Ok(rd) = fs::read_dir(dir) { + for entry in rd.flatten() { + let name = entry.file_name(); + let s = name.to_string_lossy(); + if let Some(stem) = s.strip_suffix(".md") { + if let Ok(n) = stem.parse::() { + if n > max { + max = n; + } + } + } + } + } + (max + 1) as u32 +} + +fn compose_turn(turn: &ArchivedTurn) -> String { + let mut yaml = String::from("---\n"); + yaml.push_str(&format!("session_id: {}\n", turn.session_id)); + yaml.push_str(&format!("seq: {}\n", turn.seq)); + yaml.push_str(&format!("timestamp_ms: {}\n", turn.timestamp_ms)); + yaml.push_str(&format!("role: {}\n", turn.role)); + yaml.push_str(&format!("cost_microdollars: {}\n", turn.cost_microdollars)); + if let Some(lesson) = turn.lesson.as_ref() { + yaml.push_str(&format!("lesson: {}\n", yaml_escape(lesson))); + } + if let Some(tc) = turn.tool_calls_json.as_ref() { + yaml.push_str(&format!("tool_calls_json: {}\n", yaml_escape(tc))); + } + yaml.push_str("---\n\n"); + yaml.push_str(&turn.content); + if !turn.content.ends_with('\n') { + yaml.push('\n'); + } + yaml +} + +fn yaml_escape(s: &str) -> String { + // Quote any string that contains characters with YAML semantic meaning. + // Simple double-quote escaping is good enough for these single-line + // front-matter fields. + let needs_quote = s + .chars() + .any(|c| matches!(c, ':' | '#' | '\n' | '"' | '\'' | '[' | ']' | '{' | '}')); + if needs_quote { + format!("\"{}\"", s.replace('\\', "\\\\").replace('"', "\\\"")) + } else { + s.to_string() + } +} + +/// Append a turn to its session's archive. Returns the assigned `seq`. +/// +/// `turn.seq` is ignored on input — the on-disk directory is the source of +/// truth and the returned `ArchivedTurn` carries the actually-assigned seq. +pub fn record_turn(config: &Config, mut turn: ArchivedTurn) -> Result { + let dir = session_dir(config, &turn.session_id); + fs::create_dir_all(&dir).with_context(|| format!("failed to mkdir -p {}", dir.display()))?; + turn.seq = next_seq(&dir); + let path = dir.join(format!("{:06}.md", turn.seq)); + let bytes = compose_turn(&turn).into_bytes(); + write_if_new(&path, &bytes) + .with_context(|| format!("failed to write episodic turn {}", path.display()))?; + log::debug!( + "[memory_archivist] recorded session={} seq={} role={} bytes={}", + turn.session_id, + turn.seq, + turn.role, + bytes.len() + ); + Ok(turn) +} + +/// Read every turn for `session_id`, sorted by seq ascending. +pub fn session_entries(config: &Config, session_id: &str) -> Result> { + let dir = session_dir(config, session_id); + if !dir.exists() { + return Ok(Vec::new()); + } + let mut files: Vec<(u32, PathBuf)> = fs::read_dir(&dir) + .with_context(|| format!("failed to read_dir {}", dir.display()))? + .filter_map(|e| e.ok()) + .filter_map(|e| { + let name = e.file_name(); + let s = name.to_string_lossy(); + let stem = s.strip_suffix(".md")?; + let seq = stem.parse::().ok()?; + Some((seq, e.path())) + }) + .collect(); + files.sort_by_key(|(seq, _)| *seq); + let mut out = Vec::with_capacity(files.len()); + for (_, path) in files { + let bytes = + fs::read(&path).with_context(|| format!("failed to read {}", path.display()))?; + let text = String::from_utf8_lossy(&bytes); + if let Some(turn) = parse_turn(&text) { + out.push(turn); + } + } + Ok(out) +} + +fn parse_turn(text: &str) -> Option { + let body_start = text.strip_prefix("---\n")?; + let end = body_start.find("\n---\n")?; + let (yaml, rest) = body_start.split_at(end); + let body = rest.strip_prefix("\n---\n").unwrap_or(rest).to_string(); + let mut turn = ArchivedTurn::default(); + for line in yaml.lines() { + let Some((k, v)) = line.split_once(':') else { + continue; + }; + let k = k.trim(); + let v = v.trim(); + let v_unquoted = v + .strip_prefix('"') + .and_then(|s| s.strip_suffix('"')) + .map(|s| s.replace("\\\"", "\"").replace("\\\\", "\\")) + .unwrap_or_else(|| v.to_string()); + match k { + "session_id" => turn.session_id = v_unquoted, + "seq" => turn.seq = v_unquoted.parse().unwrap_or(0), + "timestamp_ms" => turn.timestamp_ms = v_unquoted.parse().unwrap_or(0), + "role" => turn.role = v_unquoted, + "cost_microdollars" => turn.cost_microdollars = v_unquoted.parse().unwrap_or(0), + "lesson" => turn.lesson = Some(v_unquoted), + "tool_calls_json" => turn.tool_calls_json = Some(v_unquoted), + _ => {} + } + } + // Strip the single blank line compose() writes between the closing + // `---\n` and the body, then trim trailing newline. Internal blank + // lines in the body are preserved. + turn.content = body + .strip_prefix('\n') + .unwrap_or(body.as_str()) + .trim_end() + .to_string(); + Some(turn) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn test_config() -> (TempDir, Config) { + let tmp = TempDir::new().unwrap(); + let mut cfg = Config::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + (tmp, cfg) + } + + fn turn(session: &str, role: &str, content: &str) -> ArchivedTurn { + ArchivedTurn { + session_id: session.into(), + seq: 0, + timestamp_ms: 1_700_000_000_000, + role: role.into(), + content: content.into(), + lesson: None, + tool_calls_json: None, + cost_microdollars: 0, + } + } + + #[test] + fn round_trip_single_turn() { + let (_tmp, cfg) = test_config(); + let stored = record_turn(&cfg, turn("s1", "user", "hello world")).unwrap(); + assert_eq!(stored.seq, 0); + let read = session_entries(&cfg, "s1").unwrap(); + assert_eq!(read.len(), 1); + assert_eq!(read[0].content, "hello world"); + assert_eq!(read[0].role, "user"); + assert_eq!(read[0].session_id, "s1"); + assert_eq!(read[0].seq, 0); + } + + #[test] + fn append_increments_seq() { + let (_tmp, cfg) = test_config(); + let a = record_turn(&cfg, turn("s1", "user", "one")).unwrap(); + let b = record_turn(&cfg, turn("s1", "assistant", "two")).unwrap(); + let c = record_turn(&cfg, turn("s1", "user", "three")).unwrap(); + assert_eq!((a.seq, b.seq, c.seq), (0, 1, 2)); + let read = session_entries(&cfg, "s1").unwrap(); + assert_eq!( + read.iter().map(|t| t.seq).collect::>(), + vec![0, 1, 2] + ); + assert_eq!(read[1].role, "assistant"); + assert_eq!(read[2].content, "three"); + } + + #[test] + fn missing_session_returns_empty() { + let (_tmp, cfg) = test_config(); + assert!(session_entries(&cfg, "never").unwrap().is_empty()); + } + + #[test] + fn preserves_lesson_and_tool_calls() { + let (_tmp, cfg) = test_config(); + let mut t = turn("s1", "assistant", "did the thing"); + t.lesson = Some("be careful with X: it bites".into()); + t.tool_calls_json = Some(r#"[{"name":"bash","args":{"cmd":"ls"}}]"#.into()); + t.cost_microdollars = 1234; + record_turn(&cfg, t.clone()).unwrap(); + let read = session_entries(&cfg, "s1").unwrap(); + assert_eq!( + read[0].lesson.as_deref(), + Some("be careful with X: it bites") + ); + assert_eq!( + read[0].tool_calls_json.as_deref(), + Some(r#"[{"name":"bash","args":{"cmd":"ls"}}]"#) + ); + assert_eq!(read[0].cost_microdollars, 1234); + } + + #[test] + fn distinct_sessions_dont_mix() { + let (_tmp, cfg) = test_config(); + record_turn(&cfg, turn("a", "user", "hi a")).unwrap(); + record_turn(&cfg, turn("b", "user", "hi b")).unwrap(); + record_turn(&cfg, turn("a", "user", "more a")).unwrap(); + let a = session_entries(&cfg, "a").unwrap(); + let b = session_entries(&cfg, "b").unwrap(); + assert_eq!(a.len(), 2); + assert_eq!(b.len(), 1); + assert_eq!(b[0].content, "hi b"); + } +} diff --git a/src/openhuman/memory_archivist/tree_writer.rs b/src/openhuman/memory_archivist/tree_writer.rs new file mode 100644 index 000000000..8360fd6cb --- /dev/null +++ b/src/openhuman/memory_archivist/tree_writer.rs @@ -0,0 +1,265 @@ +//! End-to-end: clean → compose → push the conversation into a tree as one +//! leaf. Uses the [`crate::openhuman::memory_tree`] write contract so the +//! archivist stays unaware of tree internals. +//! +//! The archivist intentionally writes one leaf per archived conversation +//! rather than persisting another bespoke store. `chunk_id_for_session` +//! hashes `(session_id, composed_markdown)` so retries are deterministic for +//! the same conversation snapshot while distinct sessions or edits produce a +//! fresh leaf id. +//! +//! These archivist leaves are synthetic conversation snapshots, not +//! `mem_tree_chunks` rows. That means they currently participate in the L0 +//! buffer contract only: downstream source-tree sealing still expects +//! chunk-store-backed leaves when rehydrating inputs. Multi-conversation +//! summarisation for archivist-only source trees will need a dedicated +//! hydration path before these synthetic leaves can seal upward. + +use anyhow::Result; +use chrono::Utc; +use sha2::{Digest, Sha256}; + +use crate::openhuman::config::Config; +use crate::openhuman::memory_archivist::clip::clean_conversation; +use crate::openhuman::memory_archivist::compose::compose_conversation_md; +use crate::openhuman::memory_archivist::types::Turn; +use crate::openhuman::memory_store::trees::{Tree, TreeKind}; +use crate::openhuman::memory_tree::io::{ + TreeLabelStrategy, TreeLeafPayload, TreeWriteOutcome, TreeWriteRequest, +}; +use crate::openhuman::memory_tree::tree::bucket_seal::{append_leaf, LabelStrategy}; + +const TOKEN_DIVISOR: usize = 4; + +/// Clean the conversation, compose it as md, and append a single leaf to +/// the supplied tree. Returns the resulting [`TreeWriteOutcome`] including +/// any summary ids that sealed during the cascade. +pub async fn archive_to_tree( + config: &Config, + tree: &Tree, + session_id: &str, + turns: &[Turn], +) -> Result { + let cleaned = clean_conversation(turns); + let md = compose_conversation_md(&cleaned); + let chunk_id = chunk_id_for_session(session_id, &md); + let token_count = (md.len() / TOKEN_DIVISOR).max(1) as u32; + let timestamp = cleaned.last().map(|t| t.timestamp).unwrap_or_else(Utc::now); + + let request = TreeWriteRequest { + tree_id: tree.id.clone(), + tree_kind: tree.kind, + leaf: TreeLeafPayload { + chunk_id: chunk_id.clone(), + token_count, + timestamp, + content: md, + entities: Vec::new(), + topics: Vec::new(), + score: 0.0, + }, + label_strategy: TreeLabelStrategy::Inherit, + deferred: false, + }; + + let leaf_ref = (&request.leaf).into(); + // Cleaned conversations have no extractor-derived entities/topics + // riding along, so the only meaningful strategy is `Empty`. Callers + // that want extraction can extend memory_tree::io::TreeLabelStrategy + // and the dispatch below. + let _ = request.label_strategy; + let strategy = LabelStrategy::Empty; + let new_summary_ids = append_leaf(config, tree, &leaf_ref, &strategy).await?; + log::debug!( + "[memory_archivist] archive_to_tree tree_id={} session={} chunk_id={} new_summaries={}", + tree.id, + session_id, + chunk_id, + new_summary_ids.len() + ); + Ok(TreeWriteOutcome { + new_summary_ids, + seal_pending: false, + }) +} + +fn chunk_id_for_session(session_id: &str, md: &str) -> String { + let mut h = Sha256::new(); + h.update(session_id.as_bytes()); + h.update(b"\0"); + h.update(md.as_bytes()); + let digest = h.finalize(); + let hex = hex::encode(digest); + format!("archivist:{}", &hex[..32]) +} + +// Kind helper so callers don't have to import TreeKind themselves when +// they pass a `Tree` they already have. (Re-export for ergonomic match.) +#[allow(dead_code)] +fn _kind_compile_check(t: &Tree) -> TreeKind { + t.kind +} + +#[cfg(test)] +mod tests { + use chrono::TimeZone; + use tempfile::TempDir; + + use super::{archive_to_tree, chunk_id_for_session}; + use crate::openhuman::config::Config; + use crate::openhuman::memory_archivist::types::Turn; + use crate::openhuman::memory_store::trees::store as tree_store; + use crate::openhuman::memory_store::trees::{Tree, TreeKind, TreeStatus}; + use crate::openhuman::memory_tree::sources::registry::get_or_create_source_tree; + + #[test] + fn chunk_id_is_stable_for_same_session_and_markdown() { + let a = chunk_id_for_session("session-1", "## user\nhello\n"); + let b = chunk_id_for_session("session-1", "## user\nhello\n"); + assert_eq!(a, b); + assert!(a.starts_with("archivist:")); + } + + #[test] + fn chunk_id_changes_when_session_changes() { + let a = chunk_id_for_session("session-1", "## user\nhello\n"); + let b = chunk_id_for_session("session-2", "## user\nhello\n"); + assert_ne!(a, b); + } + + #[test] + fn chunk_id_changes_when_markdown_changes() { + let a = chunk_id_for_session("session-1", "## user\nhello\n"); + let b = chunk_id_for_session("session-1", "## user\nhello again\n"); + assert_ne!(a, b); + } + + fn test_config(tmp: &TempDir) -> Config { + Config { + workspace_dir: tmp.path().join("workspace"), + config_path: tmp.path().join("config.toml"), + ..Config::default() + } + } + + fn source_tree(scope: &str) -> Tree { + Tree { + id: format!("tree:{scope}"), + kind: TreeKind::Source, + scope: scope.to_string(), + root_id: None, + max_level: 0, + status: TreeStatus::Active, + created_at: chrono::Utc::now(), + last_sealed_at: None, + } + } + + #[tokio::test] + async fn archive_to_tree_writes_a_leaf_for_conversation_turns() { + let tmp = TempDir::new().unwrap(); + let cfg = test_config(&tmp); + std::fs::create_dir_all(&cfg.workspace_dir).unwrap(); + let tree = get_or_create_source_tree(&cfg, "chat:slack:#eng").unwrap(); + + let turns = vec![ + Turn { + role: "user".into(), + content: "How does ownership work in Rust?".into(), + tool_calls_json: None, + timestamp: chrono::Utc.with_ymd_and_hms(2026, 5, 24, 10, 0, 0).unwrap(), + }, + Turn { + role: "assistant".into(), + content: "Ownership gives each value a single owner.".into(), + tool_calls_json: Some("{\"tool\":\"ignored\"}".into()), + timestamp: chrono::Utc.with_ymd_and_hms(2026, 5, 24, 10, 1, 0).unwrap(), + }, + ]; + + let outcome = archive_to_tree(&cfg, &tree, "session-1", &turns) + .await + .expect("archive_to_tree"); + assert!( + outcome.new_summary_ids.is_empty(), + "single archivist leaf should not seal summaries immediately" + ); + assert!(!outcome.seal_pending); + + let buffer = tree_store::get_buffer(&cfg, &tree.id, 0).unwrap(); + assert_eq!(buffer.item_ids.len(), 1); + let expected_md = "## user\nHow does ownership work in Rust?\n\n## assistant\nOwnership gives each value a single owner.\n"; + assert_eq!( + buffer.item_ids[0], + chunk_id_for_session("session-1", expected_md) + ); + assert_eq!( + buffer.token_sum, + ((expected_md.len() / 4).max(1)) as i64, + "token count should follow archivist TOKEN_DIVISOR heuristic" + ); + } + + #[tokio::test] + async fn archive_to_tree_handles_empty_turns_via_fallback_markdown() { + let tmp = TempDir::new().unwrap(); + let cfg = test_config(&tmp); + std::fs::create_dir_all(&cfg.workspace_dir).unwrap(); + let tree = get_or_create_source_tree(&cfg, "chat:empty").unwrap(); + + let outcome = archive_to_tree(&cfg, &tree, "session-empty", &[]) + .await + .expect("archive_to_tree empty"); + assert!(outcome.new_summary_ids.is_empty()); + + let buffer = tree_store::get_buffer(&cfg, &tree.id, 0).unwrap(); + assert_eq!(buffer.item_ids.len(), 1); + assert_eq!( + buffer.item_ids[0], + chunk_id_for_session("session-empty", ""), + "empty conversation still generates a deterministic archivist chunk id" + ); + assert_eq!(buffer.token_sum, 1); + } + + #[tokio::test] + async fn archive_to_tree_accumulates_multiple_sessions_in_buffer_order() { + let tmp = TempDir::new().unwrap(); + let cfg = test_config(&tmp); + std::fs::create_dir_all(&cfg.workspace_dir).unwrap(); + let tree = get_or_create_source_tree(&cfg, "chat:slack:#buffer-order").unwrap(); + + let mut expected_ids = Vec::new(); + for idx in 0..3 { + let turns = vec![Turn { + role: "user".into(), + content: format!("Conversation {idx} about the phoenix rollout."), + tool_calls_json: None, + timestamp: chrono::Utc + .with_ymd_and_hms(2026, 5, 24, 10, idx, 0) + .unwrap(), + }]; + let outcome = archive_to_tree(&cfg, &tree, &format!("session-{idx}"), &turns) + .await + .expect("archive_to_tree multi-session batch"); + assert!( + outcome.new_summary_ids.is_empty(), + "archivist writes should remain buffered until a later seal-compatible path exists" + ); + + let expected_md = format!("## user\nConversation {idx} about the phoenix rollout.\n"); + expected_ids.push(chunk_id_for_session( + &format!("session-{idx}"), + &expected_md, + )); + } + + let l0 = tree_store::get_buffer(&cfg, &tree.id, 0).unwrap(); + assert_eq!(l0.item_ids, expected_ids); + assert_eq!(l0.item_ids.len(), 3); + assert!( + l0.token_sum >= 3, + "each archivist conversation contributes at least one token" + ); + } +} diff --git a/src/openhuman/memory_archivist/types.rs b/src/openhuman/memory_archivist/types.rs new file mode 100644 index 000000000..83dd6bece --- /dev/null +++ b/src/openhuman/memory_archivist/types.rs @@ -0,0 +1,117 @@ +//! Input shapes for archivist. +//! +//! Two distinct types because they cover two distinct flows: +//! +//! - [`Turn`] — input to the batch `archive_to_tree` flow +//! (clip-and-push-to-tree). +//! - [`ArchivedTurn`] — per-turn capture record persisted as a single md +//! file under `/episodic//.md`. +//! Mirrors the legacy `unified::fts5::EpisodicEntry` so +//! the harness archivist can dual-write while we +//! migrate off FTS5. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +/// One per-turn capture record persisted by [`crate::openhuman::memory_archivist::store::record_turn`]. +/// Field names match the legacy `EpisodicEntry` so the harness archivist +/// can call into both surfaces with the same payload during the +/// migration window. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct ArchivedTurn { + pub session_id: String, + /// Per-session sequence number, assigned by `record_turn` on write. + pub seq: u32, + /// Wall-clock timestamp of the turn (epoch milliseconds). + pub timestamp_ms: i64, + /// `"user"` / `"assistant"` / `"system"` / `"tool"`. + pub role: String, + pub content: String, + /// Optional post-turn lesson (kept verbatim from the harness). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub lesson: Option, + /// Serialized tool-call payload, when the turn issued any. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_calls_json: Option, + /// Cost in microdollars; 0 when not yet billed. + #[serde(default)] + pub cost_microdollars: u64, +} + +/// One conversation turn. `tool_calls_json` carries the raw model-side +/// tool-call payload when present; [`crate::openhuman::memory_archivist::clean_conversation`] +/// strips it before the turn lands in the tree. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Turn { + /// `"user"` / `"assistant"` / `"system"` / `"tool"` — free-form so we + /// don't fight any specific harness's role taxonomy. + pub role: String, + /// Natural-language body. + pub content: String, + /// Raw JSON of any tool invocations the turn issued. Dropped during + /// clipping. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_calls_json: Option, + /// Wall-clock timestamp the turn occurred. Used as the tree leaf + /// timestamp. + pub timestamp: DateTime, +} + +impl Turn { + pub fn new(role: impl Into, content: impl Into) -> Self { + Self { + role: role.into(), + content: content.into(), + tool_calls_json: None, + timestamp: Utc::now(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn archived_turn_defaults_are_empty_and_zero() { + let turn = ArchivedTurn::default(); + assert!(turn.session_id.is_empty()); + assert_eq!(turn.seq, 0); + assert_eq!(turn.timestamp_ms, 0); + assert!(turn.role.is_empty()); + assert!(turn.content.is_empty()); + assert!(turn.lesson.is_none()); + assert!(turn.tool_calls_json.is_none()); + assert_eq!(turn.cost_microdollars, 0); + } + + #[test] + fn turn_new_sets_role_content_and_no_tool_calls() { + let turn = Turn::new("user", "hello"); + assert_eq!(turn.role, "user"); + assert_eq!(turn.content, "hello"); + assert!(turn.tool_calls_json.is_none()); + } + + #[test] + fn archived_turn_serde_skips_absent_optional_fields() { + let turn = ArchivedTurn { + session_id: "s1".into(), + seq: 1, + timestamp_ms: 123, + role: "assistant".into(), + content: "done".into(), + lesson: None, + tool_calls_json: None, + cost_microdollars: 55, + }; + let value = serde_json::to_value(&turn).unwrap(); + assert_eq!(value["session_id"], json!("s1")); + assert!(value.get("lesson").is_none()); + assert!(value.get("tool_calls_json").is_none()); + + let decoded: ArchivedTurn = serde_json::from_value(value).unwrap(); + assert_eq!(decoded, turn); + } +} diff --git a/src/openhuman/memory/conversations/README.md b/src/openhuman/memory_conversations/README.md similarity index 100% rename from src/openhuman/memory/conversations/README.md rename to src/openhuman/memory_conversations/README.md diff --git a/src/openhuman/memory/conversations/bus.rs b/src/openhuman/memory_conversations/bus.rs similarity index 92% rename from src/openhuman/memory/conversations/bus.rs rename to src/openhuman/memory_conversations/bus.rs index d0bc2e0af..3877a3868 100644 --- a/src/openhuman/memory/conversations/bus.rs +++ b/src/openhuman/memory_conversations/bus.rs @@ -368,4 +368,30 @@ mod tests { assert_eq!(messages.len(), 1); assert_eq!(messages[0].id, "user:m1"); } + + #[test] + fn persisted_channel_thread_id_ignores_blank_thread_ts() { + let without = persisted_channel_thread_id("slack", "alice", "general", None); + let with_blank = persisted_channel_thread_id("slack", "alice", "general", Some(" ")); + assert_eq!(without, with_blank); + } + + #[test] + fn channel_thread_title_uses_thread_suffix_only_for_non_telegram_threads() { + assert_eq!( + channel_thread_title("slack", "alice", "general", Some(" 123 ")), + "slack · alice · general · thread 123" + ); + assert_eq!( + channel_thread_title("telegram", "alice", "chat-1", Some("123")), + "telegram · alice · chat-1" + ); + } + + #[test] + fn non_empty_trimmed_rejects_blank_strings() { + assert_eq!(non_empty_trimmed(" hello "), Some("hello")); + assert_eq!(non_empty_trimmed(" "), None); + assert_eq!(non_empty_trimmed(""), None); + } } diff --git a/src/openhuman/memory/conversations/mod.rs b/src/openhuman/memory_conversations/mod.rs similarity index 71% rename from src/openhuman/memory/conversations/mod.rs rename to src/openhuman/memory_conversations/mod.rs index af888d6ca..c5f3c1e8c 100644 --- a/src/openhuman/memory/conversations/mod.rs +++ b/src/openhuman/memory_conversations/mod.rs @@ -3,6 +3,11 @@ //! Conversations are stored as JSONL files under `/memory/conversations/`. //! Thread metadata is append-only in `threads.jsonl`; each thread's messages live //! in a dedicated JSONL file for straightforward inspection and recovery. +//! +//! This module was split out of `openhuman::memory` into the top-level +//! `openhuman::memory_conversations` namespace so the high-level memory policy +//! layer does not also own UI thread persistence. `openhuman::memory` re-exports +//! this module as `memory::conversations` during the migration. mod bus; mod store; diff --git a/src/openhuman/memory/conversations/store.rs b/src/openhuman/memory_conversations/store.rs similarity index 100% rename from src/openhuman/memory/conversations/store.rs rename to src/openhuman/memory_conversations/store.rs diff --git a/src/openhuman/memory/conversations/store_tests.rs b/src/openhuman/memory_conversations/store_tests.rs similarity index 100% rename from src/openhuman/memory/conversations/store_tests.rs rename to src/openhuman/memory_conversations/store_tests.rs diff --git a/src/openhuman/memory_conversations/types.rs b/src/openhuman/memory_conversations/types.rs new file mode 100644 index 000000000..4593c4bf1 --- /dev/null +++ b/src/openhuman/memory_conversations/types.rs @@ -0,0 +1,138 @@ +//! Wire/storage types for the workspace-backed conversation store: threads, +//! messages, create requests, and partial-update patches. + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +/// A persisted conversation thread, mirroring one entry in `threads.jsonl`. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct ConversationThread { + pub id: String, + pub title: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub chat_id: Option, + pub is_active: bool, + pub message_count: usize, + pub last_message_at: String, + pub created_at: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parent_thread_id: Option, + #[serde(default)] + pub labels: Vec, +} + +/// A single message appended to a thread's JSONL log. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct ConversationMessage { + pub id: String, + pub content: String, + #[serde(rename = "type")] + pub message_type: String, + #[serde(default)] + pub extra_metadata: Value, + pub sender: String, + pub created_at: String, +} + +/// Input payload to create-or-update a thread via [`super::ensure_thread`]. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CreateConversationThread { + pub id: String, + pub title: String, + pub created_at: String, + #[serde(default)] + pub parent_thread_id: Option, + #[serde(default)] + pub labels: Option>, +} + +/// Partial update to apply to a stored message (e.g. rewriting `extraMetadata`). +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct ConversationMessagePatch { + #[serde(default)] + pub extra_metadata: Option, +} + +/// A single match returned by +/// [`super::store::ConversationStore::search_cross_thread_messages`]. Carries +/// the source `thread_id` so the caller can render provenance into the +/// `[Cross-chat context]` block (issue #1505). +#[derive(Debug, Clone, PartialEq)] +pub struct CrossThreadHit { + pub thread_id: String, + pub message_id: String, + pub role: String, + pub content: String, + pub created_at: String, + pub score: f64, +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn conversation_thread_serde_uses_camel_case_and_defaults_labels() { + let raw = json!({ + "id": "thread-1", + "title": "Memory", + "chatId": 42, + "isActive": true, + "messageCount": 3, + "lastMessageAt": "2026-05-24T08:00:00Z", + "createdAt": "2026-05-24T07:00:00Z", + "parentThreadId": "parent-1" + }); + + let thread: ConversationThread = serde_json::from_value(raw).unwrap(); + assert_eq!(thread.chat_id, Some(42)); + assert_eq!(thread.parent_thread_id.as_deref(), Some("parent-1")); + assert!(thread.labels.is_empty(), "labels should default to []"); + + let encoded = serde_json::to_value(&thread).unwrap(); + assert_eq!(encoded["chatId"], json!(42)); + assert_eq!(encoded["parentThreadId"], json!("parent-1")); + assert!(encoded.get("chat_id").is_none()); + assert!(encoded.get("parent_thread_id").is_none()); + } + + #[test] + fn conversation_message_patch_defaults_to_no_changes() { + let patch: ConversationMessagePatch = serde_json::from_value(json!({})).unwrap(); + assert!(patch.extra_metadata.is_none()); + + let patch_with_metadata: ConversationMessagePatch = + serde_json::from_value(json!({"extraMetadata": {"source": "mock"}})).unwrap(); + assert_eq!( + patch_with_metadata.extra_metadata, + Some(json!({"source": "mock"})) + ); + } + + #[test] + fn create_thread_optional_fields_roundtrip() { + let create = CreateConversationThread { + id: "thread-2".into(), + title: "Thread".into(), + created_at: "2026-05-24T08:00:00Z".into(), + parent_thread_id: None, + labels: Some(vec!["important".into(), "memory".into()]), + }; + + let encoded = serde_json::to_value(&create).unwrap(); + assert_eq!(encoded["labels"], json!(["important", "memory"])); + assert_eq!(encoded["parentThreadId"], Value::Null); + + let decoded: CreateConversationThread = serde_json::from_value(encoded).unwrap(); + assert_eq!( + decoded.labels, + Some(vec!["important".to_string(), "memory".to_string()]) + ); + assert!(decoded.parent_thread_id.is_none()); + } +} diff --git a/src/openhuman/memory_entities/README.md b/src/openhuman/memory_entities/README.md new file mode 100644 index 000000000..f8c2488dd --- /dev/null +++ b/src/openhuman/memory_entities/README.md @@ -0,0 +1,75 @@ +# memory_entities + +Md-backed registry of people and other named things. Replacement for the +SQLite-backed `people/` module — the user's vault is the source of truth +so Obsidian, grep, and vector search all see the data without going +through a separate store. + +## On disk + +```text +/entities//.md +``` + +Each file: + +```markdown +--- +id: person:alice +kind: person +display_name: Alice Cooper +aliases: + - Ali +emails: + - alice@example.com +handles: + - kind: slack + value: U12345 +created_at: 2026-05-23T22:00:00Z +updated_at: 2026-05-23T22:00:00Z +--- + +Free-form notes the user can edit in Obsidian. Preserved across upserts. +``` + +`kind` matches `memory::score::extract::EntityKind` so canonical ids the +scorer emits round-trip through here unchanged. + +## API + +| Function | Purpose | +| --- | --- | +| `put_entity(config, Entity) -> Entity` | Upsert. Preserves user-edited notes body. | +| `get_entity(config, kind, canonical_id) -> Option` | Read by id. | +| `list_entities(config, kind) -> Vec` | Walk a kind directory. | +| `lookup_alias(config, kind, needle) -> Option` | Find by alias / email / handle value / display name (case-insensitive). | + +## Layout + +| Path | Role | +| --- | --- | +| [`mod.rs`](mod.rs) | Module root + re-exports. | +| [`types.rs`](types.rs) | `Entity`, `EntityKind`, `EntityHandle`. | +| [`store.rs`](store.rs) | Disk-backed read/write, YAML compose/parse, atomic upsert that preserves notes body. Tests. | + +## Migration from `people/` + +| `people/` | `memory_entities/` | +| --- | --- | +| `Person { id, display_name, primary_email, primary_phone, handles, created_at, updated_at }` | `Entity { id, kind: Person, display_name, emails, handles, aliases, created_at, updated_at }` | +| `Handle::IMessage(s)` | `EntityHandle { kind: "imessage", value: s }` | +| `Handle::Email(s)` | added to `emails` | +| `Handle::DisplayName(s)` | added to `aliases` | +| `PeopleStore::insert_person / lookup / get / list` | `put_entity / lookup_alias / get_entity / list_entities` | + +The SQLite-backed `people/` keeps running in parallel — this is a scaffold, +not a cut-over. + +## Layer rules + +- Borrows nothing from memory_store internals beyond the + `` path (resolved via `Config::memory_tree_content_root`). +- No SQLite. No async. No upward deps. +- Filenames are content-addressed slugs of the canonical id; the + authoritative id lives in the file's YAML `id:` field, so the on-disk + layout can change without breaking parsers. diff --git a/src/openhuman/memory_entities/mod.rs b/src/openhuman/memory_entities/mod.rs new file mode 100644 index 000000000..6e9d693a0 --- /dev/null +++ b/src/openhuman/memory_entities/mod.rs @@ -0,0 +1,80 @@ +//! Memory entities — Obsidian-md-backed registry of people and other named +//! things in the user's world. +//! +//! Replacement for the SQLite-backed `people/` module. The data lives as +//! markdown files in the content store so the user's vault is the source +//! of truth and arbitrary tools (Obsidian itself, grep, vector search) +//! can introspect or edit it. +//! +//! ## On disk +//! +//! ```text +//! /entities//.md +//! ``` +//! +//! Each file: +//! +//! ```markdown +//! --- +//! id: +//! kind: person | organization | topic | email | url | hashtag | ... +//! display_name: +//! aliases: +//! - "" +//! - "" +//! emails: +//! - "" +//! handles: +//! - kind: imessage +//! value: "+15555550100" +//! created_at: +//! updated_at: +//! --- +//! +//! +//! ``` +//! +//! `kind` matches [`memory::score::extract::EntityKind`] verbatim so the +//! same canonical-id format the scorer emits round-trips through here. +//! +//! ## API +//! +//! - [`store::put_entity`] — upsert by canonical id (atomic write). +//! - [`store::get_entity`] — read by canonical id. +//! - [`store::list_entities`] — walk a kind directory. +//! - [`store::lookup_alias`] — find a canonical id by alias / email / +//! handle (linear scan; fine for the order-of-magnitudes a single user +//! accumulates). +//! +//! ## Migration from `people/` +//! +//! `people::Person` maps onto [`Entity { kind: Person, ... }`]. The handle +//! types (`IMessage`, `Email`, `DisplayName`) become entries in the +//! `handles` / `emails` / `aliases` fields. The SQLite resolver and +//! address-book code in `people/` continues to work in parallel until +//! every caller switches to this module; this is a scaffold, not a +//! cut-over. + +pub mod store; +pub mod types; + +pub use store::{get_entity, list_entities, lookup_alias, put_entity}; +pub use types::{Entity, EntityHandle, EntityKind}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn entity_reexports_are_constructible() { + let handle = EntityHandle { + kind: "slack".into(), + value: "@alice".into(), + }; + let mut entity = Entity::new("person:alice", EntityKind::Person); + entity.handles.push(handle.clone()); + + assert_eq!(entity.kind, EntityKind::Person); + assert_eq!(entity.handles, vec![handle]); + } +} diff --git a/src/openhuman/memory_entities/store.rs b/src/openhuman/memory_entities/store.rs new file mode 100644 index 000000000..74fbf565e --- /dev/null +++ b/src/openhuman/memory_entities/store.rs @@ -0,0 +1,435 @@ +//! Disk-backed entity store. +//! +//! Atomic md write contract via `memory_store::content::atomic::write_if_new`, +//! with an explicit overwrite for upsert. Notes body is preserved across +//! upserts so the user can hand-edit it in Obsidian without losing edits. + +use std::fs; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use chrono::{DateTime, Utc}; + +use crate::openhuman::config::Config; +use crate::openhuman::memory_entities::types::{Entity, EntityHandle, EntityKind}; + +const ENTITIES_DIR: &str = "entities"; + +fn slugify_id(id: &str) -> String { + id.chars() + .map(|c| match c { + '/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|' | '\0' => '_', + c if c.is_control() => '_', + c => c, + }) + .collect() +} + +fn kind_dir(config: &Config, kind: EntityKind) -> PathBuf { + config + .memory_tree_content_root() + .join(ENTITIES_DIR) + .join(kind.as_str()) +} + +fn entity_path(config: &Config, kind: EntityKind, canonical_id: &str) -> PathBuf { + kind_dir(config, kind).join(format!("{}.md", slugify_id(canonical_id))) +} + +/// Upsert. Preserves any user-edited notes body that already exists on +/// disk; only the YAML front-matter is rewritten. Returns the stored +/// entity with `updated_at` refreshed. +pub fn put_entity(config: &Config, mut entity: Entity) -> Result { + let dir = kind_dir(config, entity.kind); + fs::create_dir_all(&dir).with_context(|| format!("failed to mkdir -p {}", dir.display()))?; + let path = entity_path(config, entity.kind, &entity.id); + + // Preserve any free-form notes the user typed in Obsidian. + let existing_notes = match fs::read_to_string(&path) { + Ok(text) => extract_notes(&text), + Err(_) => String::new(), + }; + + entity.updated_at = Utc::now(); + let bytes = compose(&entity, &existing_notes).into_bytes(); + fs::write(&path, &bytes) + .with_context(|| format!("failed to write entity {}", path.display()))?; + log::debug!( + "[memory_entities] put kind={} id={} bytes={}", + entity.kind.as_str(), + entity.id, + bytes.len() + ); + Ok(entity) +} + +/// Read by canonical id. Returns `Ok(None)` when the file doesn't exist. +pub fn get_entity(config: &Config, kind: EntityKind, canonical_id: &str) -> Result> { + let path = entity_path(config, kind, canonical_id); + if !path.exists() { + return Ok(None); + } + let text = + fs::read_to_string(&path).with_context(|| format!("failed to read {}", path.display()))?; + Ok(parse(&text)) +} + +/// List every stored entity of a given kind. Order is filesystem-dependent +/// — callers that need a sort impose their own. +pub fn list_entities(config: &Config, kind: EntityKind) -> Result> { + let dir = kind_dir(config, kind); + if !dir.exists() { + return Ok(Vec::new()); + } + let mut out = Vec::new(); + for entry in + fs::read_dir(&dir).with_context(|| format!("failed to read_dir {}", dir.display()))? + { + let entry = entry?; + let name = entry.file_name(); + let s = name.to_string_lossy(); + if !s.ends_with(".md") { + continue; + } + let text = fs::read_to_string(entry.path()) + .with_context(|| format!("failed to read {}", entry.path().display()))?; + if let Some(e) = parse(&text) { + out.push(e); + } + } + Ok(out) +} + +/// Find an entity whose `aliases`, `emails`, or `handles[*].value` matches +/// `needle` (case-insensitive). Returns the first match in walk order; +/// `kind` narrows the search. `None` when no match. +/// +/// Linear scan — for a single-user workspace with thousands (not millions) +/// of entities this is fine and avoids any additional index. +pub fn lookup_alias(config: &Config, kind: EntityKind, needle: &str) -> Result> { + let lower = needle.to_lowercase(); + for e in list_entities(config, kind)? { + if e.aliases.iter().any(|a| a.to_lowercase() == lower) { + return Ok(Some(e)); + } + if e.emails.iter().any(|m| m.to_lowercase() == lower) { + return Ok(Some(e)); + } + if e.handles.iter().any(|h| h.value.to_lowercase() == lower) { + return Ok(Some(e)); + } + if e.display_name + .as_deref() + .map(|n| n.to_lowercase() == lower) + .unwrap_or(false) + { + return Ok(Some(e)); + } + } + Ok(None) +} + +// ───────────────────────── compose / parse ───────────────────────── + +fn compose(entity: &Entity, notes: &str) -> String { + let mut out = String::from("---\n"); + out.push_str(&format!("id: {}\n", entity.id)); + out.push_str(&format!("kind: {}\n", entity.kind.as_str())); + if let Some(name) = entity.display_name.as_deref() { + out.push_str(&format!("display_name: {}\n", yaml_string(name))); + } + if !entity.aliases.is_empty() { + out.push_str("aliases:\n"); + for a in &entity.aliases { + out.push_str(&format!(" - {}\n", yaml_string(a))); + } + } + if !entity.emails.is_empty() { + out.push_str("emails:\n"); + for e in &entity.emails { + out.push_str(&format!(" - {}\n", yaml_string(e))); + } + } + if !entity.handles.is_empty() { + out.push_str("handles:\n"); + for h in &entity.handles { + out.push_str(&format!( + " - kind: {}\n value: {}\n", + yaml_string(&h.kind), + yaml_string(&h.value) + )); + } + } + out.push_str(&format!("created_at: {}\n", entity.created_at.to_rfc3339())); + out.push_str(&format!("updated_at: {}\n", entity.updated_at.to_rfc3339())); + out.push_str("---\n\n"); + out.push_str(notes); + if !notes.ends_with('\n') { + out.push('\n'); + } + out +} + +fn yaml_string(s: &str) -> String { + let needs_quote = s + .chars() + .any(|c| matches!(c, ':' | '#' | '\n' | '"' | '\'' | '[' | ']' | '{' | '}')); + if needs_quote { + format!("\"{}\"", s.replace('\\', "\\\\").replace('"', "\\\"")) + } else { + s.to_string() + } +} + +fn unquote(s: &str) -> String { + s.strip_prefix('"') + .and_then(|x| x.strip_suffix('"')) + .map(|x| x.replace("\\\"", "\"").replace("\\\\", "\\")) + .unwrap_or_else(|| s.to_string()) +} + +fn split_front_matter(text: &str) -> Option<(&str, &str)> { + let rest = text.strip_prefix("---\n")?; + let end = rest.find("\n---\n")?; + let (yaml, after) = rest.split_at(end); + let body = after.strip_prefix("\n---\n").unwrap_or(after); + Some((yaml, body)) +} + +fn extract_notes(text: &str) -> String { + split_front_matter(text) + .map(|(_, body)| body.to_string()) + .unwrap_or_default() +} + +fn parse(text: &str) -> Option { + let (yaml, body) = split_front_matter(text)?; + let mut id = String::new(); + let mut kind: Option = None; + let mut display_name: Option = None; + let mut aliases = Vec::new(); + let mut emails = Vec::new(); + let mut handles = Vec::new(); + let mut created_at: Option> = None; + let mut updated_at: Option> = None; + + let mut current_list: Option<&'static str> = None; + let mut handle_buf: Option = None; + + for raw in yaml.lines() { + if raw.starts_with(" - kind:") { + // Flush previous handle, start a new one. + if let Some(h) = handle_buf.take() { + handles.push(h); + } + let v = raw.trim_start_matches(" - kind:").trim(); + handle_buf = Some(EntityHandle { + kind: unquote(v), + value: String::new(), + }); + current_list = Some("handles"); + continue; + } + if raw.starts_with(" value:") { + let v = raw.trim_start_matches(" value:").trim(); + if let Some(h) = handle_buf.as_mut() { + h.value = unquote(v); + } + continue; + } + if let Some(v) = raw.strip_prefix(" - ") { + let v = unquote(v.trim()); + match current_list { + Some("aliases") => aliases.push(v), + Some("emails") => emails.push(v), + _ => {} + } + continue; + } + // Flush any in-progress handle when we leave the handle list. + if !raw.starts_with(' ') && !raw.starts_with(" - kind") { + if let Some(h) = handle_buf.take() { + handles.push(h); + } + current_list = None; + } + let Some((k, v)) = raw.split_once(':') else { + continue; + }; + let v = v.trim(); + match k.trim() { + "id" => id = unquote(v), + "kind" => kind = EntityKind::parse(&unquote(v)).ok(), + "display_name" => display_name = Some(unquote(v)), + "aliases" => current_list = Some("aliases"), + "emails" => current_list = Some("emails"), + "handles" => current_list = Some("handles"), + "created_at" => { + created_at = DateTime::parse_from_rfc3339(&unquote(v)) + .ok() + .map(|d| d.with_timezone(&Utc)) + } + "updated_at" => { + updated_at = DateTime::parse_from_rfc3339(&unquote(v)) + .ok() + .map(|d| d.with_timezone(&Utc)) + } + _ => {} + } + } + if let Some(h) = handle_buf { + handles.push(h); + } + + let now = Utc::now(); + let _ = body; // notes are preserved on write but not surfaced in Entity + Some(Entity { + id, + kind: kind?, + display_name, + aliases, + emails, + handles, + created_at: created_at.unwrap_or(now), + updated_at: updated_at.unwrap_or(now), + }) +} + +// ───────────────────────── tests ───────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn cfg() -> (TempDir, Config) { + let tmp = TempDir::new().unwrap(); + let mut c = Config::default(); + c.workspace_dir = tmp.path().to_path_buf(); + (tmp, c) + } + + fn alice() -> Entity { + let mut e = Entity::new("person:alice", EntityKind::Person); + e.display_name = Some("Alice Cooper".into()); + e.aliases = vec!["Ali".into(), "A. Cooper".into()]; + e.emails = vec!["alice@example.com".into()]; + e.handles = vec![EntityHandle { + kind: "slack".into(), + value: "U12345".into(), + }]; + e + } + + #[test] + fn round_trip_person() { + let (_t, c) = cfg(); + let stored = put_entity(&c, alice()).unwrap(); + let got = get_entity(&c, EntityKind::Person, "person:alice") + .unwrap() + .expect("entity present"); + assert_eq!(got.id, stored.id); + assert_eq!(got.display_name.as_deref(), Some("Alice Cooper")); + assert_eq!(got.aliases, vec!["Ali".to_string(), "A. Cooper".into()]); + assert_eq!(got.emails, vec!["alice@example.com".to_string()]); + assert_eq!(got.handles.len(), 1); + assert_eq!(got.handles[0].kind, "slack"); + assert_eq!(got.handles[0].value, "U12345"); + } + + #[test] + fn missing_entity_returns_none() { + let (_t, c) = cfg(); + assert!(get_entity(&c, EntityKind::Person, "person:nope") + .unwrap() + .is_none()); + } + + #[test] + fn list_entities_by_kind() { + let (_t, c) = cfg(); + put_entity(&c, alice()).unwrap(); + let mut bob = Entity::new("person:bob", EntityKind::Person); + bob.display_name = Some("Bob".into()); + put_entity(&c, bob).unwrap(); + let mut org = Entity::new("organization:acme", EntityKind::Organization); + org.display_name = Some("Acme".into()); + put_entity(&c, org).unwrap(); + + let people = list_entities(&c, EntityKind::Person).unwrap(); + assert_eq!(people.len(), 2); + let orgs = list_entities(&c, EntityKind::Organization).unwrap(); + assert_eq!(orgs.len(), 1); + assert_eq!(orgs[0].display_name.as_deref(), Some("Acme")); + } + + #[test] + fn lookup_alias_finds_by_alias_email_handle_or_name() { + let (_t, c) = cfg(); + put_entity(&c, alice()).unwrap(); + assert_eq!( + lookup_alias(&c, EntityKind::Person, "Ali") + .unwrap() + .unwrap() + .id, + "person:alice" + ); + assert_eq!( + lookup_alias(&c, EntityKind::Person, "alice@example.com") + .unwrap() + .unwrap() + .id, + "person:alice" + ); + assert_eq!( + lookup_alias(&c, EntityKind::Person, "U12345") + .unwrap() + .unwrap() + .id, + "person:alice" + ); + assert_eq!( + lookup_alias(&c, EntityKind::Person, "alice cooper") + .unwrap() + .unwrap() + .id, + "person:alice" + ); + assert!(lookup_alias(&c, EntityKind::Person, "noone") + .unwrap() + .is_none()); + } + + #[test] + fn upsert_preserves_user_notes_body() { + let (_t, c) = cfg(); + put_entity(&c, alice()).unwrap(); + // User hand-edits the file in Obsidian to add notes. + let path = entity_path(&c, EntityKind::Person, "person:alice"); + let original = fs::read_to_string(&path).unwrap(); + let with_notes = format!("{original}\nMet at the conference in March.\n"); + fs::write(&path, &with_notes).unwrap(); + + // Re-upsert with new alias — notes should survive. + let mut updated = alice(); + updated.aliases.push("Coop".into()); + put_entity(&c, updated).unwrap(); + + let body = fs::read_to_string(&path).unwrap(); + assert!(body.contains("Met at the conference in March.")); + assert!(body.contains("Coop")); + } + + #[test] + fn slugify_strips_filesystem_unsafe_chars() { + // `:` is stripped for Windows compatibility even though it's legal + // on Unix; the round-trip uses the in-file `id` field as the + // canonical id, so the on-disk filename is just a content-addressed + // handle. + assert_eq!(slugify_id("person:alice"), "person_alice"); + assert_eq!( + slugify_id("url:https://x.com/path"), + "url_https___x.com_path" + ); + } +} diff --git a/src/openhuman/memory_entities/types.rs b/src/openhuman/memory_entities/types.rs new file mode 100644 index 000000000..7d7990f8a --- /dev/null +++ b/src/openhuman/memory_entities/types.rs @@ -0,0 +1,202 @@ +//! Entity shape. +//! +//! One serde struct covers every kind. `kind` field discriminates; the +//! optional fields (`emails`, `handles`, `aliases`, `notes`) populate as +//! relevant. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +/// Kinds an entity can take. Mirrors +/// [`crate::openhuman::memory::score::extract::EntityKind`] so canonical +/// ids the scorer emits round-trip through this module unchanged. Kept as +/// a local enum (not a re-export) so memory_entities stays usable +/// independently of the score module's exact internals. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EntityKind { + Person, + Organization, + Topic, + Email, + Url, + Handle, + Hashtag, + Location, + Event, + Product, + Datetime, + Technology, + Artifact, + Quantity, + Misc, +} + +impl EntityKind { + pub fn as_str(self) -> &'static str { + match self { + EntityKind::Person => "person", + EntityKind::Organization => "organization", + EntityKind::Topic => "topic", + EntityKind::Email => "email", + EntityKind::Url => "url", + EntityKind::Handle => "handle", + EntityKind::Hashtag => "hashtag", + EntityKind::Location => "location", + EntityKind::Event => "event", + EntityKind::Product => "product", + EntityKind::Datetime => "datetime", + EntityKind::Technology => "technology", + EntityKind::Artifact => "artifact", + EntityKind::Quantity => "quantity", + EntityKind::Misc => "misc", + } + } + + pub fn parse(s: &str) -> Result { + match s { + "person" => Ok(Self::Person), + "organization" => Ok(Self::Organization), + "topic" => Ok(Self::Topic), + "email" => Ok(Self::Email), + "url" => Ok(Self::Url), + "handle" => Ok(Self::Handle), + "hashtag" => Ok(Self::Hashtag), + "location" => Ok(Self::Location), + "event" => Ok(Self::Event), + "product" => Ok(Self::Product), + "datetime" => Ok(Self::Datetime), + "technology" => Ok(Self::Technology), + "artifact" => Ok(Self::Artifact), + "quantity" => Ok(Self::Quantity), + "misc" => Ok(Self::Misc), + other => Err(format!("unknown entity kind: {other}")), + } + } +} + +/// A handle is an opaque label by which this entity is known to a source. +/// Generalisation of `people::Handle` — works for emails, phone numbers, +/// social handles, anything that identifies the entity in one channel +/// without being its canonical id. +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct EntityHandle { + /// e.g. `"imessage"`, `"slack"`, `"discord"`, `"gmail"`. + pub kind: String, + pub value: String, +} + +/// One entity. Persisted as `/entities//.md`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Entity { + /// Canonical id — `:` (e.g. `person:alice`, + /// `email:alice@example.com`). Stable across renames and aliases. + pub id: String, + pub kind: EntityKind, + /// Free-form display name. `None` when the user hasn't named the + /// entity yet. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub display_name: Option, + /// Alternate strings the entity is known by (nicknames, old names). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub aliases: Vec, + /// Email addresses associated with the entity. Pulled out of the + /// generic `handles` for Person convenience. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub emails: Vec, + /// Source-specific handles (slack, discord, imessage, …). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub handles: Vec, + /// First write timestamp. + pub created_at: DateTime, + /// Last upsert timestamp. + pub updated_at: DateTime, +} + +impl Entity { + /// Construct a fresh entity. `id` should already be canonicalized + /// (`:`); callers are responsible for that. + pub fn new(id: impl Into, kind: EntityKind) -> Self { + let now = Utc::now(); + Self { + id: id.into(), + kind, + display_name: None, + aliases: Vec::new(), + emails: Vec::new(), + handles: Vec::new(), + created_at: now, + updated_at: now, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn entity_kind_roundtrips() { + for kind in [ + EntityKind::Person, + EntityKind::Organization, + EntityKind::Topic, + EntityKind::Email, + EntityKind::Url, + EntityKind::Handle, + EntityKind::Hashtag, + EntityKind::Location, + EntityKind::Event, + EntityKind::Product, + EntityKind::Datetime, + EntityKind::Technology, + EntityKind::Artifact, + EntityKind::Quantity, + EntityKind::Misc, + ] { + assert_eq!(EntityKind::parse(kind.as_str()).unwrap(), kind); + } + } + + #[test] + fn entity_new_sets_empty_collections_and_timestamps() { + let entity = Entity::new("person:alice", EntityKind::Person); + assert_eq!(entity.id, "person:alice"); + assert_eq!(entity.kind, EntityKind::Person); + assert!(entity.display_name.is_none()); + assert!(entity.aliases.is_empty()); + assert!(entity.emails.is_empty()); + assert!(entity.handles.is_empty()); + assert_eq!(entity.created_at, entity.updated_at); + } + + #[test] + fn entity_handle_and_entity_serde_roundtrip() { + let entity = Entity { + id: "person:alice".into(), + kind: EntityKind::Person, + display_name: Some("Alice".into()), + aliases: vec!["A".into()], + emails: vec!["alice@example.com".into()], + handles: vec![EntityHandle { + kind: "slack".into(), + value: "@alice".into(), + }], + created_at: Utc::now(), + updated_at: Utc::now(), + }; + let value = serde_json::to_value(&entity).unwrap(); + assert_eq!(value["id"], json!("person:alice")); + assert_eq!(value["kind"], json!("person")); + assert_eq!(value["display_name"], json!("Alice")); + + let decoded: Entity = serde_json::from_value(value).unwrap(); + assert_eq!(decoded.id, entity.id); + assert_eq!(decoded.kind, entity.kind); + assert_eq!(decoded.display_name, entity.display_name); + assert_eq!(decoded.aliases, entity.aliases); + assert_eq!(decoded.emails, entity.emails); + assert_eq!(decoded.handles, entity.handles); + } +} diff --git a/src/openhuman/memory_graph/README.md b/src/openhuman/memory_graph/README.md new file mode 100644 index 000000000..738ee3961 --- /dev/null +++ b/src/openhuman/memory_graph/README.md @@ -0,0 +1,35 @@ +# memory_graph + +Placeholder over `mem_tree_entity_index`. Derives entity relationships +on demand instead of writing a parallel triple-store table. + +**Premise**: the graph IS the tree mapped out. Two entities that +co-occur on the same tree node form an edge; weight is the count of +distinct shared nodes. + +## API + +| Function | Returns | +| --- | --- | +| `co_occurring_entities(config, subject, limit)` | `Vec` sorted by weight DESC, then object ASC. | +| `neighbors(config, subject, limit)` | `Vec` — neighbor entity ids only. | +| `query::group_by_weight(edges)` | `HashMap>` for UIs that want strong vs weak buckets. | + +## Layout + +| Path | Role | +| --- | --- | +| [`mod.rs`](mod.rs) | Module root + re-exports. | +| [`types.rs`](types.rs) | `GraphEdge { subject, object, weight }`. | +| [`query.rs`](query.rs) | Co-occurrence SELF-JOIN over `mem_tree_entity_index`. Tests. | + +## Layer rules + +- Read-only. No new tables, no new schema. Everything derives from the + entity index that the tree summariser already maintains. +- Reads through `memory_store::chunks::store::with_connection` — same + SQLite connection used by the rest of memory_store. +- Intentionally does **not** cover the LLM-extracted + `(subject, predicate, object)` triples that ingestion writes via + `unified::graph::graph_upsert_namespace`. That surface needs a + separate decision (drop it, or persist triples as md files). diff --git a/src/openhuman/memory_graph/mod.rs b/src/openhuman/memory_graph/mod.rs new file mode 100644 index 000000000..74b506400 --- /dev/null +++ b/src/openhuman/memory_graph/mod.rs @@ -0,0 +1,48 @@ +//! Memory graph — placeholder over the existing tree entity index. +//! +//! The premise: a separate triple store (`unified::graph`) is redundant +//! when every chunk already lands an entity row in `mem_tree_entity_index`. +//! The graph IS the tree mapped out — two entities co-occurring on the +//! same leaf form an edge. +//! +//! This module derives those edges on demand instead of writing a parallel +//! storage table. It's a placeholder while the existing `unified::graph` +//! callers (ingestion's LLM-extracted triples + the public client RPC) +//! get migrated or retired; the LLM-extracted (subject, predicate, object) +//! triple surface is intentionally not covered here. +//! +//! ## API +//! +//! - [`co_occurring_entities`] — for a subject entity, return every other +//! entity that has appeared on the same node, with a co-occurrence +//! count. +//! - [`neighbors`] — convenience: just the entity ids, no counts. +//! +//! ## Layer rules +//! +//! - Reads from `mem_tree_entity_index` via +//! `memory_store::chunks::store::with_connection`. No writes. +//! - No new tables, no new schema. Anything you can't derive from the +//! entity index is intentionally out of scope here. + +pub mod query; +pub mod types; + +pub use query::{co_occurring_entities, neighbors}; +pub use types::GraphEdge; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn graph_edge_reexport_is_constructible() { + let edge = GraphEdge { + subject: "person:alice".into(), + object: "topic:phoenix".into(), + weight: 2, + }; + assert_eq!(edge.weight, 2); + assert_eq!(edge.subject, "person:alice"); + } +} diff --git a/src/openhuman/memory_graph/query.rs b/src/openhuman/memory_graph/query.rs new file mode 100644 index 000000000..d3d769ced --- /dev/null +++ b/src/openhuman/memory_graph/query.rs @@ -0,0 +1,177 @@ +//! Read-only graph queries derived from `mem_tree_entity_index`. + +use std::collections::HashMap; + +use anyhow::{Context, Result}; +use rusqlite::params; + +use crate::openhuman::config::Config; +use crate::openhuman::memory_graph::types::GraphEdge; +use crate::openhuman::memory_store::chunks::store::with_connection; + +/// Return every entity that shares at least one node with `subject_entity`, +/// with a `weight` equal to the number of distinct shared nodes. Sorted by +/// weight DESC, then object id ASC for deterministic output. `limit` caps +/// the result set; `None` defaults to 100. +pub fn co_occurring_entities( + config: &Config, + subject_entity: &str, + limit: Option, +) -> Result> { + let cap = limit.unwrap_or(100).min(i64::MAX as usize) as i64; + with_connection(config, |conn| { + // SELF JOIN on node_id — every (subject, other) pair counted once + // per distinct shared node. Excludes self-edges. + let mut stmt = conn + .prepare( + "SELECT b.entity_id AS object, COUNT(DISTINCT a.node_id) AS weight + FROM mem_tree_entity_index a + JOIN mem_tree_entity_index b ON a.node_id = b.node_id + WHERE a.entity_id = ?1 + AND b.entity_id <> ?1 + GROUP BY b.entity_id + ORDER BY weight DESC, object ASC + LIMIT ?2", + ) + .context("prepare co_occurring_entities")?; + let rows: Vec<(String, i64)> = stmt + .query_map(params![subject_entity, cap], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) + }) + .context("query co_occurring_entities")? + .collect::>>() + .context("collect co_occurring_entities rows")?; + Ok(rows + .into_iter() + .map(|(object, weight)| GraphEdge { + subject: subject_entity.to_string(), + object, + weight: weight.max(0) as u32, + }) + .collect()) + }) +} + +/// Convenience wrapper around [`co_occurring_entities`] that returns just +/// the neighbor entity ids in weight-descending order. +pub fn neighbors( + config: &Config, + subject_entity: &str, + limit: Option, +) -> Result> { + Ok(co_occurring_entities(config, subject_entity, limit)? + .into_iter() + .map(|e| e.object) + .collect()) +} + +/// Group the result of [`co_occurring_entities`] by weight. Useful for UIs +/// that want to render strong vs weak relationships separately. Kept here +/// rather than in `types.rs` so it stays a pure derivation helper. +pub fn group_by_weight(edges: Vec) -> HashMap> { + let mut out: HashMap> = HashMap::new(); + for e in edges { + out.entry(e.weight).or_default().push(e.object); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::memory::score::extract::EntityKind; + use crate::openhuman::memory::score::resolver::CanonicalEntity; + use crate::openhuman::memory::score::store::index_entity; + use tempfile::TempDir; + + fn test_config() -> (TempDir, Config) { + let tmp = TempDir::new().unwrap(); + let mut cfg = Config::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + (tmp, cfg) + } + + fn entity(id: &str, kind: EntityKind, surface: &str) -> CanonicalEntity { + CanonicalEntity { + canonical_id: id.into(), + kind, + surface: surface.into(), + span_start: 0, + span_end: surface.len() as u32, + score: 1.0, + } + } + + #[test] + fn empty_when_no_co_occurrence() { + let (_tmp, cfg) = test_config(); + let alice = entity( + "email:alice@example.com", + EntityKind::Email, + "alice@example.com", + ); + index_entity(&cfg, &alice, "leaf-1", "leaf", 100, None).unwrap(); + let neighbors = co_occurring_entities(&cfg, "email:alice@example.com", None).unwrap(); + assert!(neighbors.is_empty()); + } + + #[test] + fn single_co_occurrence_weight_one() { + let (_tmp, cfg) = test_config(); + let alice = entity("email:alice@example.com", EntityKind::Email, "a"); + let bob = entity("email:bob@example.com", EntityKind::Email, "b"); + index_entity(&cfg, &alice, "leaf-1", "leaf", 100, None).unwrap(); + index_entity(&cfg, &bob, "leaf-1", "leaf", 100, None).unwrap(); + let edges = co_occurring_entities(&cfg, "email:alice@example.com", None).unwrap(); + assert_eq!(edges.len(), 1); + assert_eq!(edges[0].object, "email:bob@example.com"); + assert_eq!(edges[0].weight, 1); + } + + #[test] + fn weight_counts_distinct_nodes_not_rows() { + let (_tmp, cfg) = test_config(); + let alice = entity("email:alice@example.com", EntityKind::Email, "a"); + let bob = entity("email:bob@example.com", EntityKind::Email, "b"); + // Both on leaf-1, leaf-2, leaf-3 -> weight 3. + for leaf in &["leaf-1", "leaf-2", "leaf-3"] { + index_entity(&cfg, &alice, leaf, "leaf", 100, None).unwrap(); + index_entity(&cfg, &bob, leaf, "leaf", 100, None).unwrap(); + } + let edges = co_occurring_entities(&cfg, "email:alice@example.com", None).unwrap(); + assert_eq!(edges[0].weight, 3); + } + + #[test] + fn excludes_self_edges() { + let (_tmp, cfg) = test_config(); + let alice = entity("email:alice@example.com", EntityKind::Email, "a"); + index_entity(&cfg, &alice, "leaf-1", "leaf", 100, None).unwrap(); + index_entity(&cfg, &alice, "leaf-2", "leaf", 100, None).unwrap(); + let edges = co_occurring_entities(&cfg, "email:alice@example.com", None).unwrap(); + assert!(edges.is_empty()); + } + + #[test] + fn neighbors_returns_ids_in_weight_order() { + let (_tmp, cfg) = test_config(); + let alice = entity("email:alice@example.com", EntityKind::Email, "a"); + let bob = entity("email:bob@example.com", EntityKind::Email, "b"); + let carol = entity("email:carol@example.com", EntityKind::Email, "c"); + // alice + bob: 2 shared nodes. alice + carol: 1 shared node. + index_entity(&cfg, &alice, "leaf-1", "leaf", 100, None).unwrap(); + index_entity(&cfg, &bob, "leaf-1", "leaf", 100, None).unwrap(); + index_entity(&cfg, &alice, "leaf-2", "leaf", 100, None).unwrap(); + index_entity(&cfg, &bob, "leaf-2", "leaf", 100, None).unwrap(); + index_entity(&cfg, &alice, "leaf-3", "leaf", 100, None).unwrap(); + index_entity(&cfg, &carol, "leaf-3", "leaf", 100, None).unwrap(); + let ids = neighbors(&cfg, "email:alice@example.com", None).unwrap(); + assert_eq!( + ids, + vec![ + "email:bob@example.com".to_string(), + "email:carol@example.com".to_string(), + ] + ); + } +} diff --git a/src/openhuman/memory_graph/types.rs b/src/openhuman/memory_graph/types.rs new file mode 100644 index 000000000..09da25021 --- /dev/null +++ b/src/openhuman/memory_graph/types.rs @@ -0,0 +1,42 @@ +//! Derived graph edge shape. + +use serde::{Deserialize, Serialize}; + +/// A derived co-occurrence edge between two entities. +/// +/// Not a triple in the classical sense — there's no explicit predicate. The +/// `weight` field is the count of distinct nodes the pair has both appeared +/// on, which serves as a cheap proxy for relationship strength. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct GraphEdge { + pub subject: String, + pub object: String, + pub weight: u32, +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn graph_edge_roundtrips_via_serde() { + let edge = GraphEdge { + subject: "person:alice".into(), + object: "project:openhuman".into(), + weight: 3, + }; + let value = serde_json::to_value(&edge).unwrap(); + assert_eq!( + value, + json!({ + "subject": "person:alice", + "object": "project:openhuman", + "weight": 3 + }) + ); + + let decoded: GraphEdge = serde_json::from_value(value).unwrap(); + assert_eq!(decoded, edge); + } +} diff --git a/src/openhuman/memory_tree/jobs/README.md b/src/openhuman/memory_queue/README.md similarity index 100% rename from src/openhuman/memory_tree/jobs/README.md rename to src/openhuman/memory_queue/README.md diff --git a/src/openhuman/memory_tree/jobs/handlers/README.md b/src/openhuman/memory_queue/handlers/README.md similarity index 100% rename from src/openhuman/memory_tree/jobs/handlers/README.md rename to src/openhuman/memory_queue/handlers/README.md diff --git a/src/openhuman/memory_tree/jobs/handlers/mod.rs b/src/openhuman/memory_queue/handlers/mod.rs similarity index 88% rename from src/openhuman/memory_tree/jobs/handlers/mod.rs rename to src/openhuman/memory_queue/handlers/mod.rs index 46fb8f303..e98661dc2 100644 --- a/src/openhuman/memory_tree/jobs/handlers/mod.rs +++ b/src/openhuman/memory_queue/handlers/mod.rs @@ -12,26 +12,25 @@ use anyhow::{Context, Result}; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::content_store::{ - self as content_store, read as content_read, tags as content_tags, -}; -use crate::openhuman::memory_tree::jobs::store; -use crate::openhuman::memory_tree::jobs::types::{ +use crate::openhuman::memory::jobs::store; +use crate::openhuman::memory::jobs::types::{ AppendBufferPayload, AppendTarget, DigestDailyPayload, ExtractChunkPayload, FlushStalePayload, Job, JobKind, JobOutcome, NewJob, NodeRef, ReembedBackfillPayload, SealPayload, TopicRoutePayload, }; -use crate::openhuman::memory_tree::score; -use crate::openhuman::memory_tree::score::embed::{build_embedder_from_config, pack_checked}; -use crate::openhuman::memory_tree::score::extract::build_summary_extractor; -use crate::openhuman::memory_tree::score::store as score_store; -use crate::openhuman::memory_tree::store as chunk_store; -use crate::openhuman::memory_tree::tree_global::digest::{self, DigestOutcome}; -use crate::openhuman::memory_tree::tree_source::store as summary_store; -use crate::openhuman::memory_tree::tree_source::{ - build_summariser, get_or_create_source_tree, LabelStrategy, LeafRef, +use crate::openhuman::memory::score; +use crate::openhuman::memory::score::embed::{build_embedder_from_config, pack_checked}; +use crate::openhuman::memory::score::extract::build_summary_extractor; +use crate::openhuman::memory::score::store as score_store; +use crate::openhuman::memory_store::chunks::store as chunk_store; +use crate::openhuman::memory_store::content::{ + self as content_store, read as content_read, tags as content_tags, }; -use crate::openhuman::memory_tree::tree_topic::curator; +use crate::openhuman::memory_tree::global::digest::{self, DigestOutcome}; +use crate::openhuman::memory_tree::sources::get_or_create_source_tree; +use crate::openhuman::memory_tree::topic::curator; +use crate::openhuman::memory_tree::tree::store as summary_store; +use crate::openhuman::memory_tree::tree::{LabelStrategy, LeafRef}; /// Default age for L0 flush_stale when the caller doesn't override. /// 1 hour means low-volume sources get summaries within a working session. @@ -60,7 +59,7 @@ async fn handle_extract(config: &Config, job: &Job) -> Result { serde_json::from_str(&job.payload_json).context("parse ExtractChunk payload")?; let Some(chunk) = chunk_store::get_chunk(config, &payload.chunk_id)? else { log::warn!( - "[memory_tree::jobs] extract chunk missing chunk_id={}", + "[memory::jobs] extract chunk missing chunk_id={}", payload.chunk_id ); return Ok(JobOutcome::Done); @@ -212,14 +211,14 @@ async fn handle_extract(config: &Config, job: &Job) -> Result { if let Err(e) = content_tags::update_chunk_tags(&abs_path, &obsidian_tags) { log::warn!( - "[memory_tree::jobs] failed to update tags in chunk file chunk_id={} path_hash={}: {e}", + "[memory::jobs] failed to update tags in chunk file chunk_id={} path_hash={}: {e}", chunk.id, - crate::openhuman::memory_tree::util::redact::redact(&content_path), + crate::openhuman::memory::util::redact::redact(&content_path), ); // Non-fatal: tag rewrite failure does not block the pipeline. } else { log::debug!( - "[memory_tree::jobs] updated {} obsidian tags in chunk file chunk_id={}", + "[memory::jobs] updated {} obsidian tags in chunk file chunk_id={}", obsidian_tags.len(), chunk.id, ); @@ -239,8 +238,8 @@ async fn handle_extract(config: &Config, job: &Job) -> Result { } async fn handle_append_buffer(config: &Config, job: &Job) -> Result { - use crate::openhuman::memory_tree::tree_source::bucket_seal::should_seal; - use crate::openhuman::memory_tree::tree_source::store as src_store; + use crate::openhuman::memory_tree::tree::bucket_seal::should_seal; + use crate::openhuman::memory_tree::tree::store as src_store; let payload: AppendBufferPayload = serde_json::from_str(&job.payload_json).context("parse AppendBuffer payload")?; @@ -251,7 +250,7 @@ async fn handle_append_buffer(config: &Config, job: &Job) -> Result let (leaf, chunk_id_for_lifecycle): (LeafRef, Option) = match &payload.node { NodeRef::Leaf { chunk_id } => { let Some(chunk) = chunk_store::get_chunk(config, chunk_id)? else { - log::warn!("[memory_tree::jobs] append_buffer chunk missing chunk_id={chunk_id}"); + log::warn!("[memory::jobs] append_buffer chunk missing chunk_id={chunk_id}"); return Ok(JobOutcome::Done); }; let score_row = score_store::get_score(config, &chunk.id)? @@ -275,9 +274,7 @@ async fn handle_append_buffer(config: &Config, job: &Job) -> Result } NodeRef::Summary { summary_id } => { let Some(summary) = src_store::get_summary(config, summary_id)? else { - log::warn!( - "[memory_tree::jobs] append_buffer summary missing summary_id={summary_id}" - ); + log::warn!("[memory::jobs] append_buffer summary missing summary_id={summary_id}"); return Ok(JobOutcome::Done); }; // Read the full body from disk — `summary.content` is a ≤500-char @@ -377,15 +374,15 @@ async fn handle_append_buffer(config: &Config, job: &Job) -> Result } async fn handle_seal(config: &Config, job: &Job) -> Result { - use crate::openhuman::memory_tree::tree_source::bucket_seal::{seal_one_level, should_seal}; - use crate::openhuman::memory_tree::tree_source::store as src_store; - use crate::openhuman::memory_tree::tree_source::types::TreeKind; + use crate::openhuman::memory_store::trees::types::TreeKind; + use crate::openhuman::memory_tree::tree::bucket_seal::{seal_one_level, should_seal}; + use crate::openhuman::memory_tree::tree::store as src_store; let payload: SealPayload = serde_json::from_str(&job.payload_json).context("parse Seal payload")?; let Some(tree) = src_store::get_tree(config, &payload.tree_id)? else { log::warn!( - "[memory_tree::jobs] seal tree missing tree_id={}", + "[memory::jobs] seal tree missing tree_id={}", payload.tree_id ); return Ok(JobOutcome::Done); @@ -398,7 +395,7 @@ async fn handle_seal(config: &Config, job: &Job) -> Result { let forced = payload.force_now_ms.is_some(); if buf.is_empty() { log::debug!( - "[memory_tree::jobs] seal skipped — empty buffer tree_id={} level={}", + "[memory::jobs] seal skipped — empty buffer tree_id={} level={}", tree.id, payload.level ); @@ -408,7 +405,7 @@ async fn handle_seal(config: &Config, job: &Job) -> Result { // Another job sealed this level out from under us (or the buffer // hasn't crossed the gate yet); idempotent no-op. log::debug!( - "[memory_tree::jobs] seal gate not met tree_id={} level={} token_sum={}", + "[memory::jobs] seal gate not met tree_id={} level={} token_sum={}", tree.id, payload.level, buf.token_sum @@ -427,24 +424,20 @@ async fn handle_seal(config: &Config, job: &Job) -> Result { TreeKind::Global => LabelStrategy::Empty, }; - let summariser = build_summariser(config); // `seal_one_level` with `enqueue_follow_ups: true` atomically inserts // the parent-cascade seal (if the parent buffer now meets its gate) // and the summary-side `topic_route` (for source trees) inside the // same SQLite transaction that commits the seal. This eliminates the // crash window where the seal succeeds but the follow-up enqueues // are silently lost. - let summary_id = - seal_one_level(config, &tree, &buf, summariser.as_ref(), &strategy, true).await?; + let summary_id = seal_one_level(config, &tree, &buf, &strategy, true).await?; // Phase MD-content: rewrite the `tags:` block in the sealed summary's // on-disk .md file. Entity index rows were committed inside // `seal_one_level` (via `index_summary_entity_ids_tx`), so they are // visible here. Best-effort: failure does not abort the seal. if let Err(e) = content_store::update_summary_tags(config, &summary_id) { - log::warn!( - "[memory_tree::jobs] update_summary_tags failed for summary_id={summary_id}: {e:#}" - ); + log::warn!("[memory::jobs] update_summary_tags failed for summary_id={summary_id}: {e:#}"); } super::worker::wake_workers(); @@ -461,18 +454,16 @@ async fn handle_topic_route(config: &Config, job: &Job) -> Result { let node_id: String = match &payload.node { NodeRef::Leaf { chunk_id } => { if chunk_store::get_chunk(config, chunk_id)?.is_none() { - log::warn!("[memory_tree::jobs] topic_route chunk missing chunk_id={chunk_id}"); + log::warn!("[memory::jobs] topic_route chunk missing chunk_id={chunk_id}"); return Ok(JobOutcome::Done); } chunk_id.clone() } NodeRef::Summary { summary_id } => { - if crate::openhuman::memory_tree::tree_source::store::get_summary(config, summary_id)? + if crate::openhuman::memory_store::trees::store::get_summary(config, summary_id)? .is_none() { - log::warn!( - "[memory_tree::jobs] topic_route summary missing summary_id={summary_id}" - ); + log::warn!("[memory::jobs] topic_route summary missing summary_id={summary_id}"); return Ok(JobOutcome::Done); } summary_id.clone() @@ -481,16 +472,15 @@ async fn handle_topic_route(config: &Config, job: &Job) -> Result { let entity_ids = score_store::list_entity_ids_for_node(config, &node_id)?; if entity_ids.is_empty() { - log::debug!("[memory_tree::jobs] topic_route no entities for node_id={node_id} — skipping"); + log::debug!("[memory::jobs] topic_route no entities for node_id={node_id} — skipping"); return Ok(JobOutcome::Done); } - let summariser = build_summariser(config); for entity_id in entity_ids { - let _ = curator::maybe_spawn_topic_tree(config, &entity_id, summariser.as_ref()).await?; - if let Some(tree) = crate::openhuman::memory_tree::tree_source::store::get_tree_by_scope( + let _ = curator::maybe_spawn_topic_tree(config, &entity_id).await?; + if let Some(tree) = crate::openhuman::memory_store::trees::store::get_tree_by_scope( config, - crate::openhuman::memory_tree::tree_source::types::TreeKind::Topic, + crate::openhuman::memory_store::trees::types::TreeKind::Topic, &entity_id, )? { let job = NewJob::append_buffer(&AppendBufferPayload { @@ -512,14 +502,13 @@ async fn handle_digest_daily(config: &Config, job: &Job) -> Result { serde_json::from_str(&job.payload_json).context("parse DigestDaily payload")?; let day = chrono::NaiveDate::parse_from_str(&payload.date_iso, "%Y-%m-%d") .with_context(|| format!("invalid digest date {}", payload.date_iso))?; - let summariser = build_summariser(config); - match digest::end_of_day_digest(config, day, summariser.as_ref()).await? { + match digest::end_of_day_digest(config, day).await? { DigestOutcome::Emitted { daily_id, .. } => { - log::info!("[memory_tree::jobs] emitted digest daily_id={daily_id}"); + log::info!("[memory::jobs] emitted digest daily_id={daily_id}"); } DigestOutcome::EmptyDay => {} DigestOutcome::Skipped { existing_id } => { - log::debug!("[memory_tree::jobs] digest skipped existing_id={existing_id}"); + log::debug!("[memory::jobs] digest skipped existing_id={existing_id}"); } } Ok(JobOutcome::Done) @@ -535,8 +524,7 @@ async fn handle_flush_stale(config: &Config, job: &Job) -> Result { // that set max_age_secs explicitly. let age_secs = payload.max_age_secs.unwrap_or(L0_DEFAULT_FLUSH_AGE_SECS); let cutoff = chrono::Utc::now() - chrono::Duration::seconds(age_secs); - let buffers = - crate::openhuman::memory_tree::tree_source::store::list_stale_buffers(config, cutoff)?; + let buffers = crate::openhuman::memory_store::trees::store::list_stale_buffers(config, cutoff)?; for buf in buffers { let seal = SealPayload { tree_id: buf.tree_id.clone(), @@ -577,7 +565,7 @@ fn try_mark_chunk_reembed_skipped( chunk_store::mark_chunk_reembed_skipped(config, chunk_id, model_signature, reason) { log::warn!( - "[memory_tree::jobs] reembed_backfill: failed to persist chunk tombstone chunk_id={chunk_id} sig={model_signature}: {e}" + "[memory::jobs] reembed_backfill: failed to persist chunk tombstone chunk_id={chunk_id} sig={model_signature}: {e}" ); } } @@ -592,7 +580,7 @@ fn try_mark_summary_reembed_skipped( summary_store::mark_summary_reembed_skipped(config, summary_id, model_signature, reason) { log::warn!( - "[memory_tree::jobs] reembed_backfill: failed to persist summary tombstone summary_id={summary_id} sig={model_signature}: {e}" + "[memory::jobs] reembed_backfill: failed to persist summary tombstone summary_id={summary_id} sig={model_signature}: {e}" ); } } @@ -605,7 +593,7 @@ async fn handle_reembed_backfill(config: &Config, job: &Job) -> Result Result Result chunk_vecs.push((id.clone(), v)), Ok(_) => { log::warn!( - "[memory_tree::jobs] reembed_backfill: chunk {id} embed wrong dim, skipping (sig={active_sig})" + "[memory::jobs] reembed_backfill: chunk {id} embed wrong dim, skipping (sig={active_sig})" ); try_mark_chunk_reembed_skipped(config, id, &active_sig, "embed wrong dim"); } Err(e) => { log::warn!( - "[memory_tree::jobs] reembed_backfill: chunk {id} embed failed: {e}; skipping (sig={active_sig})" + "[memory::jobs] reembed_backfill: chunk {id} embed failed: {e}; skipping (sig={active_sig})" ); try_mark_chunk_reembed_skipped( config, @@ -718,7 +706,7 @@ async fn handle_reembed_backfill(config: &Config, job: &Job) -> Result { log::warn!( - "[memory_tree::jobs] reembed_backfill: chunk {id} body read failed: {e}; skipping (sig={active_sig})" + "[memory::jobs] reembed_backfill: chunk {id} body read failed: {e}; skipping (sig={active_sig})" ); try_mark_chunk_reembed_skipped( config, @@ -736,13 +724,13 @@ async fn handle_reembed_backfill(config: &Config, job: &Job) -> Result summary_vecs.push((id.clone(), v)), Ok(_) => { log::warn!( - "[memory_tree::jobs] reembed_backfill: summary {id} embed wrong dim, skipping (sig={active_sig})" + "[memory::jobs] reembed_backfill: summary {id} embed wrong dim, skipping (sig={active_sig})" ); try_mark_summary_reembed_skipped(config, id, &active_sig, "embed wrong dim"); } Err(e) => { log::warn!( - "[memory_tree::jobs] reembed_backfill: summary {id} embed failed: {e}; skipping (sig={active_sig})" + "[memory::jobs] reembed_backfill: summary {id} embed failed: {e}; skipping (sig={active_sig})" ); try_mark_summary_reembed_skipped( config, @@ -754,7 +742,7 @@ async fn handle_reembed_backfill(config: &Config, job: &Job) -> Result { log::warn!( - "[memory_tree::jobs] reembed_backfill: summary {id} body read failed: {e}; skipping (sig={active_sig})" + "[memory::jobs] reembed_backfill: summary {id} body read failed: {e}; skipping (sig={active_sig})" ); try_mark_summary_reembed_skipped( config, @@ -773,8 +761,11 @@ async fn handle_reembed_backfill(config: &Config, job: &Job) -> Result Result Result crate::openhuman::memory_tree::tree_source::types::Tree { - use crate::openhuman::memory_tree::store::upsert_chunks; - use crate::openhuman::memory_tree::types::{ + ) -> crate::openhuman::memory_store::trees::types::Tree { + use crate::openhuman::memory_store::chunks::store::upsert_chunks; + use crate::openhuman::memory_store::chunks::types::{ chunk_id, Chunk, Metadata, SourceKind, SourceRef, }; let tree = get_or_create_source_tree(cfg, "slack:#eng").unwrap(); @@ -892,7 +883,7 @@ mod tests { let staged = content_store::stage_chunks(&content_root, &[chunk.clone()]).unwrap(); with_connection(cfg, |conn| { let tx = conn.unchecked_transaction()?; - crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?; + crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(&tx, &staged)?; tx.commit()?; Ok(()) }) @@ -953,7 +944,7 @@ mod tests { match p.node { NodeRef::Summary { summary_id } => { // Format: `summary:<13-digit-ms>:L-<8hex>` — - // see `tree_source::registry::new_summary_id`. + // see `tree::registry::new_summary_id`. assert!( summary_id.starts_with("summary:") && summary_id.contains(":L1-"), "expected summary id with L1 segment, got {summary_id}" @@ -968,15 +959,14 @@ mod tests { let (_tmp, cfg) = test_config(); // Spawn a topic tree directly via the registry (skipping curator's // hotness gate — we just need a TreeKind::Topic with leaves). - let topic_tree = - crate::openhuman::memory_tree::tree_topic::registry::get_or_create_topic_tree( - &cfg, - "topic:phoenix-migration", - ) - .unwrap(); + let topic_tree = crate::openhuman::memory_store::trees::registry::get_or_create_topic_tree( + &cfg, + "topic:phoenix-migration", + ) + .unwrap(); // Push a single 10k-token leaf so L0 is gate-ready. - use crate::openhuman::memory_tree::store::upsert_chunks; - use crate::openhuman::memory_tree::types::{ + use crate::openhuman::memory_store::chunks::store::upsert_chunks; + use crate::openhuman::memory_store::chunks::types::{ chunk_id, Chunk, Metadata, SourceKind, SourceRef, }; let ts = chrono::Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(); @@ -1005,7 +995,7 @@ mod tests { let staged = content_store::stage_chunks(&content_root, &[chunk.clone()]).unwrap(); with_connection(&cfg, |conn| { let tx = conn.unchecked_transaction()?; - crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?; + crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(&tx, &staged)?; tx.commit()?; Ok(()) }) @@ -1045,12 +1035,11 @@ mod tests { let (_tmp, cfg) = test_config(); // 1. Create a target topic tree with a clean L0 buffer. - let topic_tree = - crate::openhuman::memory_tree::tree_topic::registry::get_or_create_topic_tree( - &cfg, - "email:alice@example.com", - ) - .unwrap(); + let topic_tree = crate::openhuman::memory_store::trees::registry::get_or_create_topic_tree( + &cfg, + "email:alice@example.com", + ) + .unwrap(); let l0_before = src_store::get_buffer(&cfg, &topic_tree.id, 0).unwrap(); assert!(l0_before.is_empty()); @@ -1058,11 +1047,11 @@ mod tests { // is to create a separate source tree, push two 6k leaves into // it, and let the seal produce a summary we can address. let source_tree = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); - use crate::openhuman::memory_tree::store::upsert_chunks; - use crate::openhuman::memory_tree::tree_source::bucket_seal::seal_one_level; - use crate::openhuman::memory_tree::types::{ + use crate::openhuman::memory_store::chunks::store::upsert_chunks; + use crate::openhuman::memory_store::chunks::types::{ chunk_id, Chunk, Metadata, SourceKind, SourceRef, }; + use crate::openhuman::memory_tree::tree::bucket_seal::seal_one_level; let ts = chrono::Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(); let content_root = cfg.memory_tree_content_root(); std::fs::create_dir_all(&content_root).unwrap(); @@ -1090,7 +1079,9 @@ mod tests { let staged = content_store::stage_chunks(&content_root, &[chunk.clone()]).unwrap(); with_connection(&cfg, |conn| { let tx = conn.unchecked_transaction()?; - crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?; + crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx( + &tx, &staged, + )?; tx.commit()?; Ok(()) }) @@ -1107,20 +1098,24 @@ mod tests { let _ = append_leaf_deferred(&cfg, &source_tree, &leaf).unwrap(); } // Force-seal the source tree's L0 to mint the summary. + use crate::openhuman::memory::chat::{test_override, ChatProvider, StaticChatProvider}; let buf = src_store::get_buffer(&cfg, &source_tree.id, 0).unwrap(); - let summariser = build_summariser(&cfg); - let summary_id = seal_one_level( - &cfg, - &source_tree, - &buf, - summariser.as_ref(), - &crate::openhuman::memory_tree::tree_source::bucket_seal::LabelStrategy::Empty, - // No follow-up enqueues — the test scopes assertions to the - // append_buffer handler, not seal-side fan-out. - false, - ) - .await - .unwrap(); + let provider: std::sync::Arc = + std::sync::Arc::new(StaticChatProvider::new("test summary content")); + let summary_id = test_override::with_provider(provider, async { + seal_one_level( + &cfg, + &source_tree, + &buf, + &crate::openhuman::memory_tree::tree::bucket_seal::LabelStrategy::Empty, + // No follow-up enqueues — the test scopes assertions to the + // append_buffer handler, not seal-side fan-out. + false, + ) + .await + .unwrap() + }) + .await; // 3. Build an append_buffer payload routing the summary into the // topic tree. @@ -1164,11 +1159,11 @@ mod tests { /// deterministic effects are what this test pins.) #[tokio::test] async fn reembed_backfill_repopulates_then_completes() { - use crate::openhuman::memory_tree::store::{ + use crate::openhuman::memory_store::chunks::store::{ get_chunk_embedding_for_signature, tree_active_signature, upsert_chunks, upsert_staged_chunks_tx, }; - use crate::openhuman::memory_tree::types::{ + use crate::openhuman::memory_store::chunks::types::{ chunk_id, Chunk, Metadata, SourceKind, SourceRef, }; @@ -1268,11 +1263,11 @@ mod tests { /// is covered (or the chain would re-arm on every config save). #[tokio::test] async fn reembed_backfill_tombstones_orphan_and_terminates() { - use crate::openhuman::memory_tree::store::{ + use crate::openhuman::memory_store::chunks::store::{ get_chunk_content_path, get_chunk_embedding_for_signature, tree_active_signature, upsert_chunks, upsert_staged_chunks_tx, }; - use crate::openhuman::memory_tree::types::{ + use crate::openhuman::memory_store::chunks::types::{ chunk_id, Chunk, Metadata, SourceKind, SourceRef, }; @@ -1383,11 +1378,11 @@ mod tests { /// #2358: clearing a tombstone re-opens the row for the backfill worklist. #[tokio::test] async fn clear_chunk_reembed_skipped_reopens_worklist() { - use crate::openhuman::memory_tree::store::{ + use crate::openhuman::memory_store::chunks::store::{ clear_chunk_reembed_skipped, get_chunk_content_path, mark_chunk_reembed_skipped, tree_active_signature, upsert_chunks, upsert_staged_chunks_tx, }; - use crate::openhuman::memory_tree::types::{ + use crate::openhuman::memory_store::chunks::types::{ chunk_id, Chunk, Metadata, SourceKind, SourceRef, }; @@ -1456,9 +1451,11 @@ mod tests { /// empty/covered space. #[tokio::test] async fn ensure_reembed_backfill_enqueues_only_when_uncovered() { - use crate::openhuman::memory_tree::jobs::ensure_reembed_backfill; - use crate::openhuman::memory_tree::store::{upsert_chunks, upsert_staged_chunks_tx}; - use crate::openhuman::memory_tree::types::{ + use crate::openhuman::memory::jobs::ensure_reembed_backfill; + use crate::openhuman::memory_store::chunks::store::{ + upsert_chunks, upsert_staged_chunks_tx, + }; + use crate::openhuman::memory_store::chunks::types::{ chunk_id, Chunk, Metadata, SourceKind, SourceRef, }; diff --git a/src/openhuman/memory_tree/jobs/mod.rs b/src/openhuman/memory_queue/mod.rs similarity index 76% rename from src/openhuman/memory_tree/jobs/mod.rs rename to src/openhuman/memory_queue/mod.rs index 30e00352d..78429a40a 100644 --- a/src/openhuman/memory_tree/jobs/mod.rs +++ b/src/openhuman/memory_queue/mod.rs @@ -24,6 +24,11 @@ //! All persistence lives in the same `chunks.db` as `mem_tree_chunks` so a //! producer can insert its side-effect and its follow-up job in one tx. //! See [`store::enqueue_tx`] for the in-tx producer entry point. +//! +//! This queue used to live under `openhuman::memory::jobs`; it now has a +//! dedicated top-level home (`openhuman::memory_queue`) because it is an +//! execution/runtime concern rather than a leaf of the memory policy API. +//! `openhuman::memory` re-exports it as `memory::jobs` during the migration. mod handlers; mod redact; @@ -71,9 +76,9 @@ pub fn backfill_in_progress() -> bool { /// covered space enqueues nothing. Errors are logged, never propagated — /// a failed enqueue must not fail the user's settings save. pub fn ensure_reembed_backfill(config: &crate::openhuman::config::Config) { - let sig = crate::openhuman::memory_tree::store::tree_active_signature(config); - let result = crate::openhuman::memory_tree::store::with_connection(config, |conn| { - Ok(crate::openhuman::memory_tree::store::has_uncovered_reembed_work(conn, &sig)?) + let sig = crate::openhuman::memory_store::chunks::store::tree_active_signature(config); + let result = crate::openhuman::memory_store::chunks::store::with_connection(config, |conn| { + Ok(crate::openhuman::memory_store::chunks::store::has_uncovered_reembed_work(conn, &sig)?) }); match result { Ok(true) => { @@ -82,9 +87,7 @@ pub fn ensure_reembed_backfill(config: &crate::openhuman::config::Config) { }) { Ok(j) => j, Err(e) => { - log::warn!( - "[memory_tree::jobs] ensure_reembed_backfill: build job failed: {e}" - ); + log::warn!("[memory::jobs] ensure_reembed_backfill: build job failed: {e}"); return; } }; @@ -92,21 +95,21 @@ pub fn ensure_reembed_backfill(config: &crate::openhuman::config::Config) { Ok(_) => { set_backfill_in_progress(true); log::info!( - "[memory_tree::jobs] ensure_reembed_backfill: enqueued chain for sig={sig}" + "[memory::jobs] ensure_reembed_backfill: enqueued chain for sig={sig}" ); } Err(e) => log::warn!( - "[memory_tree::jobs] ensure_reembed_backfill: enqueue failed for sig={sig}: {e}" + "[memory::jobs] ensure_reembed_backfill: enqueue failed for sig={sig}: {e}" ), } } Ok(false) => { log::debug!( - "[memory_tree::jobs] ensure_reembed_backfill: sig={sig} fully covered; nothing to do" + "[memory::jobs] ensure_reembed_backfill: sig={sig} fully covered; nothing to do" ); } Err(e) => log::warn!( - "[memory_tree::jobs] ensure_reembed_backfill: coverage probe failed for sig={sig}: {e}" + "[memory::jobs] ensure_reembed_backfill: coverage probe failed for sig={sig}: {e}" ), } } @@ -122,3 +125,20 @@ pub use types::{ Job, JobKind, JobOutcome, JobStatus, NewJob, NodeRef, SealPayload, TopicRoutePayload, }; pub use worker::{start, wake_workers}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn backfill_flag_roundtrip() { + set_backfill_in_progress(false); + assert!(!backfill_in_progress()); + + set_backfill_in_progress(true); + assert!(backfill_in_progress()); + + set_backfill_in_progress(false); + assert!(!backfill_in_progress()); + } +} diff --git a/src/openhuman/memory_tree/jobs/redact.rs b/src/openhuman/memory_queue/redact.rs similarity index 100% rename from src/openhuman/memory_tree/jobs/redact.rs rename to src/openhuman/memory_queue/redact.rs diff --git a/src/openhuman/memory_tree/jobs/scheduler.rs b/src/openhuman/memory_queue/scheduler.rs similarity index 68% rename from src/openhuman/memory_tree/jobs/scheduler.rs rename to src/openhuman/memory_queue/scheduler.rs index f0cf61815..9b67cf5e1 100644 --- a/src/openhuman/memory_tree/jobs/scheduler.rs +++ b/src/openhuman/memory_queue/scheduler.rs @@ -9,8 +9,8 @@ use anyhow::Result; use chrono::{Datelike, Duration as ChronoDuration, NaiveDate, TimeZone, Timelike, Utc}; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::jobs::store; -use crate::openhuman::memory_tree::jobs::types::{DigestDailyPayload, FlushStalePayload, NewJob}; +use crate::openhuman::memory::jobs::store; +use crate::openhuman::memory::jobs::types::{DigestDailyPayload, FlushStalePayload, NewJob}; static STARTED: std::sync::Once = std::sync::Once::new(); @@ -24,7 +24,7 @@ pub fn start(config: Config) { tokio::spawn(async move { loop { if let Err(err) = enqueue_daily_jobs(&cfg1) { - log::warn!("[memory_tree::jobs] scheduler enqueue failed: {err:#}"); + log::warn!("[memory::jobs] scheduler enqueue failed: {err:#}"); } tokio::time::sleep(next_sleep_duration()).await; } @@ -60,12 +60,12 @@ fn enqueue_flush_stale(config: &Config) { } Ok(None) => {} // dedupe-suppressed — OK Err(err) => { - log::warn!("[memory_tree::jobs] periodic flush_stale enqueue failed: {err:#}"); + log::warn!("[memory::jobs] periodic flush_stale enqueue failed: {err:#}"); } } } Err(err) => { - log::warn!("[memory_tree::jobs] flush_stale job build failed: {err:#}"); + log::warn!("[memory::jobs] flush_stale job build failed: {err:#}"); } } } @@ -115,14 +115,14 @@ pub fn trigger_digest(config: &Config, date: NaiveDate) -> Result let job_id = store::enqueue(config, &NewJob::digest_daily(&payload)?)?; if job_id.is_some() { log::info!( - "[memory_tree::jobs] manual digest trigger enqueued date={} id={:?}", + "[memory::jobs] manual digest trigger enqueued date={} id={:?}", payload.date_iso, job_id.as_deref() ); super::worker::wake_workers(); } else { log::debug!( - "[memory_tree::jobs] manual digest trigger dedupe-suppressed date={} \ + "[memory::jobs] manual digest trigger dedupe-suppressed date={} \ (an active job for this date already exists)", payload.date_iso ); @@ -150,7 +150,7 @@ pub fn backfill_missing_digests(config: &Config, days_back: i64) -> Result Duration { #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory_tree::jobs::store::{ + use crate::openhuman::memory::jobs::store::{ claim_next, count_by_status, count_total, mark_done, DEFAULT_LOCK_DURATION_MS, }; - use crate::openhuman::memory_tree::jobs::types::JobStatus; + use crate::openhuman::memory::jobs::types::{ + DigestDailyPayload, FlushStalePayload, JobKind, JobStatus, + }; use tempfile::TempDir; fn test_config() -> (TempDir, Config) { @@ -250,6 +252,14 @@ mod tests { assert_eq!(count_total(&cfg).unwrap(), 0); } + #[test] + fn backfill_missing_digests_negative_window_is_noop() { + let (_tmp, cfg) = test_config(); + let n = backfill_missing_digests(&cfg, -3).unwrap(); + assert_eq!(n, 0); + assert_eq!(count_total(&cfg).unwrap(), 0); + } + #[test] fn backfill_missing_digests_is_idempotent_while_active() { let (_tmp, cfg) = test_config(); @@ -259,4 +269,91 @@ mod tests { assert_eq!(n2, 0, "second call must be fully dedupe-suppressed"); assert_eq!(count_total(&cfg).unwrap(), 3); } + + #[test] + fn enqueue_flush_stale_enqueues_at_most_one_job_per_current_block() { + let (_tmp, cfg) = test_config(); + enqueue_flush_stale(&cfg); + enqueue_flush_stale(&cfg); + + assert_eq!( + count_by_status(&cfg, JobStatus::Ready).unwrap(), + 1, + "second enqueue in same 3h block should be dedupe-suppressed" + ); + + let claimed = claim_next(&cfg, DEFAULT_LOCK_DURATION_MS).unwrap().unwrap(); + assert_eq!(claimed.kind, JobKind::FlushStale); + let payload: FlushStalePayload = serde_json::from_str(&claimed.payload_json).unwrap(); + assert_eq!(payload.max_age_secs, None); + } + + #[test] + fn enqueue_daily_jobs_adds_digest_and_flush_jobs() { + let (_tmp, cfg) = test_config(); + enqueue_daily_jobs(&cfg).unwrap(); + + assert_eq!(count_total(&cfg).unwrap(), 2); + + let first = claim_next(&cfg, DEFAULT_LOCK_DURATION_MS).unwrap().unwrap(); + assert_eq!( + first.kind, + JobKind::DigestDaily, + "digest_daily should be claimed ahead of flush_stale" + ); + let digest: DigestDailyPayload = serde_json::from_str(&first.payload_json).unwrap(); + assert!(!digest.date_iso.is_empty()); + mark_done(&cfg, &first).unwrap(); + + let second = claim_next(&cfg, DEFAULT_LOCK_DURATION_MS).unwrap().unwrap(); + assert_eq!(second.kind, JobKind::FlushStale); + let flush: FlushStalePayload = serde_json::from_str(&second.payload_json).unwrap(); + assert_eq!(flush.max_age_secs, None); + } + + #[test] + fn enqueue_daily_jobs_is_fully_deduped_while_jobs_remain_active() { + let (_tmp, cfg) = test_config(); + enqueue_daily_jobs(&cfg).unwrap(); + enqueue_daily_jobs(&cfg).unwrap(); + + assert_eq!( + count_total(&cfg).unwrap(), + 2, + "same-day scheduler rerun should not create duplicate active jobs" + ); + } + + #[test] + fn enqueue_daily_jobs_reenqueues_after_prior_rows_complete() { + let (_tmp, cfg) = test_config(); + enqueue_daily_jobs(&cfg).unwrap(); + + let first = claim_next(&cfg, DEFAULT_LOCK_DURATION_MS).unwrap().unwrap(); + mark_done(&cfg, &first).unwrap(); + let second = claim_next(&cfg, DEFAULT_LOCK_DURATION_MS).unwrap().unwrap(); + mark_done(&cfg, &second).unwrap(); + + enqueue_daily_jobs(&cfg).unwrap(); + + assert_eq!( + count_total(&cfg).unwrap(), + 4, + "completed daily jobs should allow a fresh digest+flush pair" + ); + assert_eq!(count_by_status(&cfg, JobStatus::Ready).unwrap(), 2); + } + + #[test] + fn next_sleep_duration_targets_near_next_midnight_utc_plus_five_minutes() { + let sleep = next_sleep_duration(); + assert!( + sleep.as_secs() > 0, + "scheduler sleep should always be positive" + ); + assert!( + sleep.as_secs() <= 24 * 60 * 60 + 5 * 60, + "scheduler should never sleep for more than ~24h+5m" + ); + } } diff --git a/src/openhuman/memory_tree/jobs/store.rs b/src/openhuman/memory_queue/store.rs similarity index 97% rename from src/openhuman/memory_tree/jobs/store.rs rename to src/openhuman/memory_queue/store.rs index f4b1b9709..5f3cddaaf 100644 --- a/src/openhuman/memory_tree/jobs/store.rs +++ b/src/openhuman/memory_queue/store.rs @@ -22,9 +22,9 @@ use rusqlite::{params, Connection, OptionalExtension, Transaction}; use uuid::Uuid; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::jobs::redact::scrub_for_log; -use crate::openhuman::memory_tree::jobs::types::{Job, JobKind, JobStatus, NewJob}; -use crate::openhuman::memory_tree::store::with_connection; +use crate::openhuman::memory::jobs::redact::scrub_for_log; +use crate::openhuman::memory::jobs::types::{Job, JobKind, JobStatus, NewJob}; +use crate::openhuman::memory_store::chunks::store::with_connection; /// Default visibility lock — a worker that crashes mid-job will have its /// row recovered after this window. 5 min is comfortably larger than any @@ -77,14 +77,14 @@ pub(crate) fn enqueue_conn(conn: &Connection, job: &NewJob) -> Result Result> .context("Failed to claim next mem_tree_jobs row")?; if let Some(j) = &row { log::debug!( - "[memory_tree::jobs] claimed id={} kind={} attempt={}/{}", + "[memory::jobs] claimed id={} kind={} attempt={}/{}", j.id, j.kind.as_str(), j.attempts, @@ -179,7 +179,7 @@ pub fn mark_done(config: &Config, job: &Job) -> Result<()> { // expired and a second worker re-claimed the row. Log and move on — // this is a known race outcome, not a bug in the current worker. log::warn!( - "[memory_tree::jobs] mark_done id={job_id} was a no-op \ + "[memory::jobs] mark_done id={job_id} was a no-op \ (stale lease: attempts={claim_attempts} started_at_ms={claim_started_at:?})" ); } @@ -209,7 +209,7 @@ pub fn mark_failed(config: &Config, job: &Job, error: &str) -> Result<()> { let error_for_log = scrub_for_log(error); if attempts >= max_attempts { log::warn!( - "[memory_tree::jobs] terminal failure id={job_id} \ + "[memory::jobs] terminal failure id={job_id} \ attempts={attempts}/{max_attempts} err={error_for_log}" ); let n = conn.execute( @@ -225,7 +225,7 @@ pub fn mark_failed(config: &Config, job: &Job, error: &str) -> Result<()> { )?; if n == 0 { log::warn!( - "[memory_tree::jobs] mark_failed(terminal) id={job_id} was a no-op \ + "[memory::jobs] mark_failed(terminal) id={job_id} was a no-op \ (stale lease: attempts={attempts} started_at_ms={claim_started_at:?})" ); } @@ -233,7 +233,7 @@ pub fn mark_failed(config: &Config, job: &Job, error: &str) -> Result<()> { let backoff = backoff_ms(attempts as u32); let next_at = now_ms.saturating_add(backoff); log::info!( - "[memory_tree::jobs] retry id={job_id} attempt={attempts}/{max_attempts} \ + "[memory::jobs] retry id={job_id} attempt={attempts}/{max_attempts} \ next_at_ms={next_at} err={error_for_log}" ); let n = conn.execute( @@ -249,7 +249,7 @@ pub fn mark_failed(config: &Config, job: &Job, error: &str) -> Result<()> { )?; if n == 0 { log::warn!( - "[memory_tree::jobs] mark_failed(retry) id={job_id} was a no-op \ + "[memory::jobs] mark_failed(retry) id={job_id} was a no-op \ (stale lease: attempts={attempts} started_at_ms={claim_started_at:?})" ); } @@ -301,7 +301,7 @@ pub fn mark_deferred(config: &Config, job: &Job, until_ms: i64, reason: &str) -> )?; if n == 0 { log::warn!( - "[memory_tree::jobs] mark_deferred id={job_id} was a no-op \ + "[memory::jobs] mark_deferred id={job_id} was a no-op \ (stale lease: attempts={claim_attempts} started_at_ms={claim_started_at:?})" ); } @@ -325,7 +325,7 @@ pub fn recover_stale_locks(config: &Config) -> Result { params![now_ms], )?; if n > 0 { - log::warn!("[memory_tree::jobs] recovered {n} stale-locked job(s) at startup"); + log::warn!("[memory::jobs] recovered {n} stale-locked job(s) at startup"); } Ok(n) }) @@ -420,7 +420,7 @@ fn backoff_ms(attempts_so_far: u32) -> i64 { #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory_tree::jobs::types::{ + use crate::openhuman::memory::jobs::types::{ AppendBufferPayload, AppendTarget, ExtractChunkPayload, NodeRef, }; use tempfile::TempDir; diff --git a/src/openhuman/memory_tree/jobs/testing.rs b/src/openhuman/memory_queue/testing.rs similarity index 50% rename from src/openhuman/memory_tree/jobs/testing.rs rename to src/openhuman/memory_queue/testing.rs index c739443fa..f42cae83e 100644 --- a/src/openhuman/memory_tree/jobs/testing.rs +++ b/src/openhuman/memory_queue/testing.rs @@ -15,3 +15,23 @@ pub async fn drain_until_idle(config: &Config) -> Result<()> { } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::config::Config; + use tempfile::TempDir; + + fn test_config() -> (TempDir, Config) { + let tmp = TempDir::new().unwrap(); + let mut cfg = Config::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + (tmp, cfg) + } + + #[tokio::test] + async fn drain_until_idle_is_noop_when_queue_is_empty() { + let (_tmp, cfg) = test_config(); + drain_until_idle(&cfg).await.unwrap(); + } +} diff --git a/src/openhuman/memory_tree/jobs/types.rs b/src/openhuman/memory_queue/types.rs similarity index 89% rename from src/openhuman/memory_tree/jobs/types.rs rename to src/openhuman/memory_queue/types.rs index eb6a7c950..730148801 100644 --- a/src/openhuman/memory_tree/jobs/types.rs +++ b/src/openhuman/memory_queue/types.rs @@ -550,4 +550,62 @@ mod tests { _ => panic!("wrong variant"), } } + + #[test] + fn new_job_extract_chunk_builder_sets_kind_payload_and_dedupe_key() { + let payload = ExtractChunkPayload { + chunk_id: "chunk-123".into(), + }; + let job = NewJob::extract_chunk(&payload).unwrap(); + assert_eq!(job.kind, JobKind::ExtractChunk); + assert_eq!(job.dedupe_key.as_deref(), Some("extract:chunk-123")); + assert_eq!(job.available_at_ms, None); + assert_eq!(job.max_attempts, None); + let roundtrip: ExtractChunkPayload = serde_json::from_str(&job.payload_json).unwrap(); + assert_eq!(roundtrip.chunk_id, "chunk-123"); + } + + #[test] + fn new_job_append_buffer_builder_uses_payload_dedupe_key() { + let payload = AppendBufferPayload { + node: NodeRef::Summary { + summary_id: "summary-9".into(), + }, + target: AppendTarget::Topic { + tree_id: "topic:ops".into(), + }, + }; + let job = NewJob::append_buffer(&payload).unwrap(); + assert_eq!(job.kind, JobKind::AppendBuffer); + assert_eq!( + job.dedupe_key.as_deref(), + Some("append:topic:topic:ops:summary:summary-9") + ); + let roundtrip: AppendBufferPayload = serde_json::from_str(&job.payload_json).unwrap(); + assert_eq!(roundtrip.dedupe_key(), payload.dedupe_key()); + } + + #[test] + fn new_job_flush_stale_builder_uses_supplied_time_bucket() { + let payload = FlushStalePayload { + max_age_secs: Some(600), + }; + let job = NewJob::flush_stale(&payload, "2026-05-24", 4).unwrap(); + assert_eq!(job.kind, JobKind::FlushStale); + assert_eq!(job.dedupe_key.as_deref(), Some("flush_stale:2026-05-24-h4")); + let roundtrip: FlushStalePayload = serde_json::from_str(&job.payload_json).unwrap(); + assert_eq!(roundtrip.max_age_secs, Some(600)); + } + + #[test] + fn new_job_reembed_backfill_builder_is_one_chain_per_signature() { + let payload = ReembedBackfillPayload { + signature: "embed-v2".into(), + }; + let job = NewJob::reembed_backfill(&payload).unwrap(); + assert_eq!(job.kind, JobKind::ReembedBackfill); + assert_eq!(job.dedupe_key.as_deref(), Some("reembed_backfill:embed-v2")); + let roundtrip: ReembedBackfillPayload = serde_json::from_str(&job.payload_json).unwrap(); + assert_eq!(roundtrip.signature, "embed-v2"); + } } diff --git a/src/openhuman/memory_tree/jobs/worker.rs b/src/openhuman/memory_queue/worker.rs similarity index 78% rename from src/openhuman/memory_tree/jobs/worker.rs rename to src/openhuman/memory_queue/worker.rs index b8f520fb2..07012466e 100644 --- a/src/openhuman/memory_tree/jobs/worker.rs +++ b/src/openhuman/memory_queue/worker.rs @@ -15,13 +15,13 @@ use anyhow::Result; use tokio::sync::Notify; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::jobs::handlers; -use crate::openhuman::memory_tree::jobs::redact::scrub_for_log; -use crate::openhuman::memory_tree::jobs::store::{ +use crate::openhuman::memory::jobs::handlers; +use crate::openhuman::memory::jobs::redact::scrub_for_log; +use crate::openhuman::memory::jobs::store::{ claim_next, mark_deferred, mark_done, mark_failed, recover_stale_locks, DEFAULT_LOCK_DURATION_MS, }; -use crate::openhuman::memory_tree::jobs::types::JobOutcome; +use crate::openhuman::memory::jobs::types::JobOutcome; /// Number of concurrent job-worker tasks. Each worker claims one job /// at a time via `claim_next` (atomic UPDATE under SQLite WAL with @@ -67,7 +67,7 @@ pub fn start(config: Config) { .get_or_init(|| Arc::new(Notify::new())) .clone(); if let Err(err) = recover_stale_locks(&config) { - log::warn!("[memory_tree::jobs] recover_stale_locks failed at startup: {err:#}"); + log::warn!("[memory::jobs] recover_stale_locks failed at startup: {err:#}"); } for idx in 0..WORKER_COUNT { @@ -95,7 +95,7 @@ pub fn start(config: Config) { // succeed. See OPENHUMAN-TAURI-BP. if is_sqlite_busy(&err) { log::warn!( - "[memory_tree::jobs] worker {idx} hit SQLite busy/locked, \ + "[memory::jobs] worker {idx} hit SQLite busy/locked, \ backing off 1s: {err:#}" ); tokio::time::sleep(Duration::from_secs(1)).await; @@ -108,7 +108,7 @@ pub fn start(config: Config) { // are NOT reported to Sentry (they are transient and were // flooding ~19K events/4 days, see #2206). log::warn!( - "[memory_tree::jobs] worker {idx} hit transient I/O error, \ + "[memory::jobs] worker {idx} hit transient I/O error, \ backing off 30s: {err:#}" ); tokio::time::sleep(Duration::from_secs(30)).await; @@ -165,7 +165,7 @@ pub async fn run_once(config: &Config) -> Result { // itself, sized to `WORKER_COUNT`, is the upstream bound). let memory_uses_local = config.workload_uses_local("memory"); log::trace!( - "[memory_tree::jobs] llm permit routing job_id={} kind={} memory_uses_local={}", + "[memory::jobs] llm permit routing job_id={} kind={} memory_uses_local={}", job.id, job.kind.as_str(), memory_uses_local @@ -195,7 +195,7 @@ pub async fn run_once(config: &Config) -> Result { match result { Ok(JobOutcome::Done) => { log::debug!( - "[memory_tree::jobs] done id={} kind={}", + "[memory::jobs] done id={} kind={}", job.id, job.kind.as_str() ); @@ -212,7 +212,7 @@ pub async fn run_once(config: &Config) -> Result { // include upstream provider responses; scrub for log // emission while keeping the original in DB state. log::info!( - "[memory_tree::jobs] deferred id={} kind={} until_ms={} reason={}", + "[memory::jobs] deferred id={} kind={} until_ms={} reason={}", job.id, job.kind.as_str(), until_ms, @@ -228,7 +228,7 @@ pub async fn run_once(config: &Config) -> Result { // commonly embed upstream HTTP bodies / auth headers. let message = format!("{err:#}"); log::warn!( - "[memory_tree::jobs] job failed id={} kind={} err={}", + "[memory::jobs] job failed id={} kind={} err={}", job.id, job.kind.as_str(), scrub_for_log(&message) @@ -304,6 +304,29 @@ fn is_sqlite_busy(err: &anyhow::Error) -> bool { #[cfg(test)] mod tests { use super::*; + use crate::openhuman::memory::jobs::store::{count_by_status, enqueue, get_job}; + use crate::openhuman::memory::jobs::types::{ + FlushStalePayload, JobKind, JobStatus, NewJob, ReembedBackfillPayload, + }; + use crate::openhuman::memory_store::chunks::store::{ + tree_active_signature, upsert_chunks, upsert_staged_chunks_tx, with_connection, + }; + use crate::openhuman::memory_store::chunks::types::{ + chunk_id, Chunk, Metadata, SourceKind, SourceRef, + }; + use crate::openhuman::memory_store::content as content_store; + use chrono::{TimeZone, Utc}; + use tempfile::TempDir; + + fn test_config() -> (TempDir, Config) { + let tmp = TempDir::new().unwrap(); + let mut cfg = Config::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + cfg.memory_tree.embedding_endpoint = None; + cfg.memory_tree.embedding_model = None; + cfg.memory_tree.embedding_strict = false; + (tmp, cfg) + } /// Raw `rusqlite::Error::SqliteFailure` with the `DatabaseBusy` code /// is what surfaces when the `busy_timeout` is exhausted on a write. @@ -457,4 +480,102 @@ mod tests { ); assert!(!is_sqlite_io_transient(&anyhow::Error::from(raw))); } + + #[tokio::test] + async fn wake_workers_is_noop_before_start() { + wake_workers(); + } + + #[tokio::test] + async fn run_once_returns_false_when_queue_is_empty() { + let (_tmp, cfg) = test_config(); + let processed = run_once(&cfg).await.unwrap(); + assert!(!processed); + } + + #[tokio::test] + async fn run_once_claims_and_completes_a_flush_stale_job() { + let (_tmp, cfg) = test_config(); + let new_job = NewJob::flush_stale(&FlushStalePayload::default(), "2026-05-24", 3).unwrap(); + let id = enqueue(&cfg, &new_job).unwrap().expect("enqueue job"); + + let processed = run_once(&cfg).await.unwrap(); + assert!(processed); + + let job = get_job(&cfg, &id).unwrap().expect("job should still exist"); + assert_eq!(job.kind.as_str(), "flush_stale"); + assert_eq!(job.status, JobStatus::Done); + assert_eq!(count_by_status(&cfg, JobStatus::Done).unwrap(), 1); + assert!(job.completed_at_ms.is_some()); + assert!(job.locked_until_ms.is_none()); + } + + #[tokio::test] + async fn run_once_reschedules_reembed_backfill_jobs_that_defer() { + let (_tmp, cfg) = test_config(); + let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(); + let chunk = Chunk { + id: chunk_id(SourceKind::Chat, "slack:#eng", 0, "reembed-worker-seed"), + content: "memory content about the phoenix migration project".into(), + metadata: Metadata { + source_kind: SourceKind::Chat, + source_id: "slack:#eng".into(), + owner: "alice".into(), + timestamp: ts, + time_range: (ts, ts), + tags: vec![], + source_ref: Some(SourceRef::new("slack://x")), + }, + token_count: 12, + seq_in_source: 0, + created_at: ts, + partial_message: false, + }; + upsert_chunks(&cfg, &[chunk.clone()]).unwrap(); + let content_root = cfg.memory_tree_content_root(); + std::fs::create_dir_all(&content_root).unwrap(); + let staged = content_store::stage_chunks(&content_root, &[chunk]).unwrap(); + with_connection(&cfg, |conn| { + let tx = conn.unchecked_transaction()?; + upsert_staged_chunks_tx(&tx, &staged)?; + tx.commit()?; + Ok(()) + }) + .unwrap(); + + let signature = tree_active_signature(&cfg); + let new_job = NewJob::reembed_backfill(&ReembedBackfillPayload { + signature: signature.clone(), + }) + .unwrap(); + let id = enqueue(&cfg, &new_job) + .unwrap() + .expect("enqueue backfill job"); + + let processed = run_once(&cfg).await.unwrap(); + assert!(processed); + + let job = get_job(&cfg, &id).unwrap().expect("job should still exist"); + assert_eq!(job.kind, JobKind::ReembedBackfill); + assert_eq!(job.status, JobStatus::Ready); + assert_eq!( + job.attempts, 0, + "defer should revert the claim attempt bump" + ); + assert!(job.started_at_ms.is_none()); + assert!(job.locked_until_ms.is_none()); + assert!(job.completed_at_ms.is_none()); + assert!( + job.available_at_ms > Utc::now().timestamp_millis(), + "deferred job should be rescheduled into the future" + ); + assert!( + job.last_error + .as_deref() + .unwrap_or("") + .contains("re-embed backfill"), + "defer reason should be recorded for visibility" + ); + assert_eq!(count_by_status(&cfg, JobStatus::Ready).unwrap(), 1); + } } diff --git a/src/openhuman/memory_store/README.md b/src/openhuman/memory_store/README.md new file mode 100644 index 000000000..7388c0ebf --- /dev/null +++ b/src/openhuman/memory_store/README.md @@ -0,0 +1,60 @@ +# memory_store + +Single home for every persisted memory shape. Owns the storage primitives — +nothing above this module touches SQLite or the on-disk vault directly. + +```text +content/ on-disk .md files — SOURCE OF TRUTH for every body +chunks/ SQLite chunk rows (metadata + tags + md path pointer + + lifecycle status) + the two chunkers that produce them +entities/ mem_tree_entity_index — every entity occurrence per node +trees/ summary tree persistence (one table, kind-parameterized) +vectors/ local vector DB (cosine, brute-force) +kv/ global + namespace key-value (kv_global, kv_namespace) +contacts/ facade over people::store (Person/Handle/Interaction) +unified/ [staging for removal] UnifiedMemory's remaining SQLite surface + (documents/query/segments/events/profile) — replaced as the + Memory trait callers migrate to per-kind backends +``` + +## Cross-cutting modules + +| Path | Role | +| --- | --- | +| [`mod.rs`](mod.rs) | Module root + public re-exports. | +| [`README.md`](README.md) | You are here. | +| [`kinds.rs`](kinds.rs) | `MemoryKind` enum — the authoritative catalog: Raw / Chunk / Entity / Tree / Vector / Kv / Contact — plus per-kind type aliases. | +| [`traits.rs`](traits.rs) | `VectorEmbeddable` + `ObsidianRepresentable` + `ObsidianFile`. Every stored kind implements both — the compiler enforces "everything in memory_store is vector and obsidian compatible". | +| [`types.rs`](types.rs) | Shared serde types used across submodules: `NamespaceDocumentInput`, `NamespaceMemoryHit`, `NamespaceQueryResult`, `NamespaceRetrievalContext`, `RetrievalScoreBreakdown`, `MemoryItemKind`, `MemoryKvRecord`. | +| [`memory_trait.rs`](memory_trait.rs) | `impl Memory for UnifiedMemory` — bridges the generic `Memory` trait surface onto the unified store. | +| [`client.rs`](client.rs) | `MemoryClient` / `MemoryClientRef` / `MemoryState`. Async wrapper over `UnifiedMemory` used by RPC controllers; owns the singleton ingestion-queue handle. | +| [`factories.rs`](factories.rs) | `create_memory*` constructors. Selects the embedding provider per the `MemoryConfig`, probes Ollama health, and builds a `Box` over `UnifiedMemory`. | +| [`retrieval/`](retrieval/) | `RetrievalFacade` — single import surface over the four retrieval modes (tree-walk, vector, keyword, param/tag). | +| [`tools/`](tools/) | Agent tools that read directly from memory_store: `memory_store_raw_search`, `memory_store_raw_chunks`, `memory_store_kinds`. | + +## Storage submodules + +| Path | Owns | +| --- | --- | +| [`content/`](content/) | **Source of truth** for chunk + summary bodies as on-disk `.md` files. Atomic writes, path layout, YAML front-matter compose/parse, tag rewrites, Obsidian vault defaults. See [`content/README.md`](content/README.md). | +| [`chunks/`](chunks/) | Full chunk lifecycle. `types.rs` (`Chunk`, `Metadata`, `SourceKind`, `RawRef`, `ListChunksQuery`) + `store.rs` (SQLite persistence + connection cache) + `produce.rs` (source-kind dispatch chunker used by the ingest pipeline) + `semantic.rs` (heading/paragraph-aware chunker). | +| [`entities/`](entities/) | Thin re-export of `memory::score::store` — `index_entity`, `index_entities`, `lookup_entity`, `list_entity_ids_for_node`, `clear_entity_index_for_node`, `count_entity_index`, `EntityHit`. Reads/writes the `mem_tree_entity_index` table. | +| [`trees/`](trees/) | `store.rs` (`mem_tree_trees` / `mem_tree_summaries` / `mem_tree_buffers`), `types.rs` (Tree / SummaryNode / TreeKind / TreeStatus / Buffer + topic hotness types), `registry.rs` (kind-parameterized helpers), `hotness.rs` (entity hotness side-table). | +| [`vectors/`](vectors/) | Standalone vector store. `VectorStore` over SQLite, byte-codec for f32 vectors, cosine similarity. | +| [`kv.rs`](kv.rs) | Global + namespace key-value (`kv_global`, `kv_namespace` tables). | +| [`contacts/`](contacts/) | Re-export of `people::store` + async fail-soft helpers (`get_contact`, `list_contacts`, `lookup_contact`). | +| [`unified/`](unified/) | **Staging for removal.** Shrinking SQLite surface that still backs the `Memory` trait while callers migrate to per-kind modules. Active pieces today: `documents`, `query`, `segments`, `events`, `profile`. The `fts5` episodic surface is replaced by [`memory_archivist`](../memory_archivist/) and `graph` by [`memory_graph`](../memory_graph/). See [`unified/README.md`](unified/README.md). | + +## Layer rules + +- **Content bytes are immutable.** The `.md` file written by `content/` is + the source of truth; SQLite stores a `(content_path, content_sha256)` + pointer. The body never changes after the first write — only YAML + front-matter (`tags:`) is rewritable. +- **SQLite is for indexing and vectors.** Anything keyword/param-searchable + on the body itself should be served by grepping the `.md` files. +- **No upward dependencies.** memory_store does not depend on + `memory_tree`, `memory_tools`, or `memory`. The one documented exception + is `retrieval::RetrievalFacade::tree_walk`, which delegates to + `memory::retrieval::drill_down`; revisit when drill_down's policy bits + can be cleanly separated from its pure traversal. diff --git a/src/openhuman/memory_store/chunks/mod.rs b/src/openhuman/memory_store/chunks/mod.rs new file mode 100644 index 000000000..c52cabb96 --- /dev/null +++ b/src/openhuman/memory_store/chunks/mod.rs @@ -0,0 +1,28 @@ +//! Chunks — the unit of memory_store persistence. +//! +//! One module for the full chunk lifecycle: +//! +//! - [`types`] — `Chunk`, `Metadata`, `SourceKind`, `RawRef`, +//! `ListChunksQuery`. The persisted shape. +//! - [`store`] — SQLite persistence (`chunks` table + connection cache). +//! - [`produce`] — source-kind-dispatch chunker (chat / email / document). +//! Used by the memory ingest pipeline; produces stable +//! per-source sequence numbers and bounded segments. +//! - [`semantic`] — heading- and paragraph-aware chunker used by the +//! unified memory writer to split large documents into +//! LLM-context-sized pieces while preserving heading +//! context. +//! +//! `produce::chunk_markdown` (the default) and `semantic::chunk_markdown` +//! both yield string-shaped chunks; the store side decides what to do with +//! them. + +pub mod produce; +pub mod semantic; +pub mod store; +pub mod types; + +pub use produce::{chunk_markdown, ChunkerInput, ChunkerOptions}; +pub use semantic::chunk_markdown as chunk_semantic; +pub use store::*; +pub use types::*; diff --git a/src/openhuman/memory_tree/chunker.rs b/src/openhuman/memory_store/chunks/produce.rs similarity index 98% rename from src/openhuman/memory_tree/chunker.rs rename to src/openhuman/memory_store/chunks/produce.rs index de6a1367b..ac244bc90 100644 --- a/src/openhuman/memory_tree/chunker.rs +++ b/src/openhuman/memory_store/chunks/produce.rs @@ -16,8 +16,10 @@ //! becomes one chunk. Same oversize fallback as Chat. //! - **Document**: original paragraph-based greedy packing (unchanged). -use crate::openhuman::memory_tree::types::{approx_token_count, Chunk, Metadata, SourceKind}; -use crate::openhuman::memory_tree::util::redact::redact; +use crate::openhuman::memory::util::redact::redact; +use crate::openhuman::memory_store::chunks::types::{ + approx_token_count, chunk_id, Chunk, Metadata, SourceKind, +}; /// Default upper bound on per-chunk tokens. /// @@ -98,7 +100,7 @@ pub fn chunk_markdown(input: &ChunkerInput, opts: &ChunkerOptions) -> Vec .map(|(idx, content)| { let seq = idx as u32; let token_count = approx_token_count(&content); - let id = super::types::chunk_id(input.source_kind, &input.source_id, seq, &content); + let id = chunk_id(input.source_kind, &input.source_id, seq, &content); Chunk { id, content, @@ -138,7 +140,7 @@ pub fn chunk_markdown(input: &ChunkerInput, opts: &ChunkerOptions) -> Vec let content = acc.join(unit_separator); let seq = out.len() as u32; let tc = approx_token_count(&content); - let id = super::types::chunk_id(input.source_kind, &input.source_id, seq, &content); + let id = chunk_id(input.source_kind, &input.source_id, seq, &content); out.push(Chunk { id, content, @@ -162,7 +164,7 @@ pub fn chunk_markdown(input: &ChunkerInput, opts: &ChunkerOptions) -> Vec for piece in sub_pieces { let seq = out.len() as u32; let tc = approx_token_count(&piece); - let id = super::types::chunk_id(input.source_kind, &input.source_id, seq, &piece); + let id = chunk_id(input.source_kind, &input.source_id, seq, &piece); out.push(Chunk { id, content: piece, @@ -200,7 +202,7 @@ pub fn chunk_markdown(input: &ChunkerInput, opts: &ChunkerOptions) -> Vec if out.is_empty() { // Degenerate: empty input → one empty chunk, matching original behaviour. - let id = super::types::chunk_id(input.source_kind, &input.source_id, 0, ""); + let id = chunk_id(input.source_kind, &input.source_id, 0, ""); out.push(Chunk { id, content: String::new(), diff --git a/src/openhuman/memory/chunker.rs b/src/openhuman/memory_store/chunks/semantic.rs similarity index 100% rename from src/openhuman/memory/chunker.rs rename to src/openhuman/memory_store/chunks/semantic.rs diff --git a/src/openhuman/memory_tree/store.rs b/src/openhuman/memory_store/chunks/store.rs similarity index 97% rename from src/openhuman/memory_tree/store.rs rename to src/openhuman/memory_store/chunks/store.rs index 1902b0b6d..39c77c39c 100644 --- a/src/openhuman/memory_tree/store.rs +++ b/src/openhuman/memory_store/chunks/store.rs @@ -35,8 +35,8 @@ use std::sync::{Arc, OnceLock}; use std::time::{Duration, Instant}; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::content_store::StagedChunk; -use crate::openhuman::memory_tree::types::{Chunk, Metadata, SourceKind, SourceRef}; +use crate::openhuman::memory_store::chunks::types::{Chunk, Metadata, SourceKind, SourceRef}; +use crate::openhuman::memory_store::content::StagedChunk; const DB_DIR: &str = "memory_tree"; const DB_FILE: &str = "chunks.db"; @@ -354,7 +354,7 @@ pub fn upsert_chunks(config: &Config, chunks: &[Chunk]) -> Result { return Ok(0); } log::debug!( - "[memory_tree::store] upsert_chunks: n={} first_id={}", + "[memory::chunk_store] upsert_chunks: n={} first_id={}", chunks.len(), chunks[0].id ); @@ -651,7 +651,7 @@ fn set_chunk_lifecycle_status_conn(conn: &Connection, chunk_id: &str, status: &s )?; if changed == 0 { log::warn!( - "[memory_tree::store] lifecycle update affected 0 rows chunk_id={} status={}", + "[memory::chunk_store] lifecycle update affected 0 rows chunk_id={} status={}", chunk_id, status ); @@ -1290,7 +1290,7 @@ fn migrate_legacy_embeddings_to_sidecar(conn: &Connection, config: &Config) -> R return Ok(()); } - let (provider, model, dims) = crate::openhuman::memory::store::effective_embedding_settings( + let (provider, model, dims) = crate::openhuman::memory_store::effective_embedding_settings( &config.memory, config.workload_local_model("embeddings").as_deref(), ); @@ -1334,7 +1334,7 @@ fn migrate_legacy_embeddings_to_sidecar(conn: &Connection, config: &Config) -> R set_chunk_embedding_for_signature_tx(&tx, &id, &sig, &vec)?; copied_chunks += 1; } else { - crate::openhuman::memory_tree::tree_source::store::set_summary_embedding_for_signature_tx( + crate::openhuman::memory_store::trees::store::set_summary_embedding_for_signature_tx( &tx, &id, &sig, &vec, )?; copied_summaries += 1; @@ -1350,18 +1350,20 @@ fn migrate_legacy_embeddings_to_sidecar(conn: &Connection, config: &Config) -> R // migration; dedupe key = signature, so exactly one chain per space. let has_uncovered = has_uncovered_reembed_work(&*tx, &sig)?; if has_uncovered { - let backfill_job = crate::openhuman::memory_tree::jobs::types::NewJob::reembed_backfill( - &crate::openhuman::memory_tree::jobs::types::ReembedBackfillPayload { + let backfill_job = crate::openhuman::memory::jobs::types::NewJob::reembed_backfill( + &crate::openhuman::memory::jobs::types::ReembedBackfillPayload { signature: sig.clone(), }, )?; - crate::openhuman::memory_tree::jobs::enqueue_tx(&tx, &backfill_job)?; - crate::openhuman::memory_tree::jobs::set_backfill_in_progress(true); + crate::openhuman::memory::jobs::enqueue_tx(&tx, &backfill_job)?; } tx.commit()?; conn.pragma_update(None, "user_version", TREE_EMBEDDING_MIGRATION_VERSION) .context("set PRAGMA user_version after #1574 migration")?; + if has_uncovered { + crate::openhuman::memory::jobs::set_backfill_in_progress(true); + } log::info!( "[memory_tree::migrate] #1574 §7 done: copied chunks={copied_chunks} summaries={copied_summaries} \ skipped_dim_mismatch={skipped_dim_mismatch} (left for §6 re-embed); user_version={TREE_EMBEDDING_MIGRATION_VERSION}" @@ -1525,7 +1527,9 @@ fn add_column_if_missing(conn: &Connection, table: &str, name: &str, sql_type: & [], ) { Ok(_) => { - log::debug!("[memory_tree::store] migration: added column {table}.{name} ({sql_type})"); + log::debug!( + "[memory::chunk_store] migration: added column {table}.{name} ({sql_type})" + ); Ok(()) } Err(err) if err.to_string().contains("duplicate column name") => Ok(()), @@ -1540,11 +1544,11 @@ fn add_column_if_missing(conn: &Connection, table: &str, name: &str, sql_type: & /// by (#1574). Reuses the established local-AI workload derivation /// ([`Config::workload_local_model`]) and the probe-stable /// `active_embedding_signature`; introduces no parallel resolution path. -/// `pub(crate)` so the sibling `tree_source` summary store shares the exact +/// `pub(crate)` so the sibling `tree` summary store shares the exact /// same resolution. pub(crate) fn tree_active_signature(config: &Config) -> String { let local_model = config.workload_local_model("embeddings"); - crate::openhuman::memory::store::active_embedding_signature( + crate::openhuman::memory_store::active_embedding_signature( &config.memory, local_model.as_deref(), ) @@ -1560,7 +1564,7 @@ pub(crate) fn tree_active_signature(config: &Config) -> String { pub fn set_chunk_embedding(config: &Config, chunk_id: &str, embedding: &[f32]) -> Result<()> { let signature = tree_active_signature(config); log::debug!( - "[memory_tree::store] set_chunk_embedding: chunk_id={chunk_id} sig={signature} dims={}", + "[memory::chunk_store] set_chunk_embedding: chunk_id={chunk_id} sig={signature} dims={}", embedding.len() ); set_chunk_embedding_for_signature(config, chunk_id, &signature, embedding) @@ -1667,7 +1671,7 @@ pub fn mark_chunk_reembed_skipped( rusqlite::params![chunk_id, model_signature, reason, now_ms], )?; log::debug!( - "[memory_tree::store] mark_chunk_reembed_skipped chunk_id={chunk_id} sig={model_signature} reason={reason}" + "[memory::chunk_store] mark_chunk_reembed_skipped chunk_id={chunk_id} sig={model_signature} reason={reason}" ); Ok(()) }) @@ -1692,7 +1696,7 @@ pub fn clear_chunk_reembed_skipped( rusqlite::params![chunk_id, model_signature], )?; log::debug!( - "[memory_tree::store] clear_chunk_reembed_skipped chunk_id={chunk_id} sig={model_signature}" + "[memory::chunk_store] clear_chunk_reembed_skipped chunk_id={chunk_id} sig={model_signature}" ); Ok(()) }) @@ -1717,7 +1721,7 @@ pub fn clear_reembed_skipped_for_signature( rusqlite::params![model_signature], )?; log::debug!( - "[memory_tree::store] clear_reembed_skipped_for_signature sig={model_signature} chunk_rows={chunk_deleted} summary_rows={summary_deleted}" + "[memory::chunk_store] clear_reembed_skipped_for_signature sig={model_signature} chunk_rows={chunk_deleted} summary_rows={summary_deleted}" ); Ok(chunk_deleted + summary_deleted) }) diff --git a/src/openhuman/memory_tree/store_tests.rs b/src/openhuman/memory_store/chunks/store_tests.rs similarity index 99% rename from src/openhuman/memory_tree/store_tests.rs rename to src/openhuman/memory_store/chunks/store_tests.rs index f76419c92..b7f41f511 100644 --- a/src/openhuman/memory_tree/store_tests.rs +++ b/src/openhuman/memory_store/chunks/store_tests.rs @@ -11,7 +11,7 @@ //! that don't need it. use super::*; -use crate::openhuman::memory_tree::types::chunk_id; +use crate::openhuman::memory_store::chunks::types::chunk_id; use chrono::TimeZone; use rusqlite::params; use tempfile::TempDir; @@ -414,7 +414,7 @@ fn legacy_embeddings_migrate_to_sidecar_once() { // Resolve the active signature/dims exactly as the migration does — // base-independent, never hard-coded (see the brittle-literal lesson). - let (p, m, dims) = crate::openhuman::memory::store::effective_embedding_settings( + let (p, m, dims) = crate::openhuman::memory_store::effective_embedding_settings( &cfg.memory, cfg.workload_local_model("embeddings").as_deref(), ); @@ -714,7 +714,7 @@ fn clear_reembed_skipped_for_signature_removes_all_tombstones_for_sig() { Ok(()) }) .unwrap(); - crate::openhuman::memory_tree::tree_source::store::mark_summary_reembed_skipped( + crate::openhuman::memory_store::trees::store::mark_summary_reembed_skipped( &cfg, summary_id, &sig, diff --git a/src/openhuman/memory_tree/types.rs b/src/openhuman/memory_store/chunks/types.rs similarity index 100% rename from src/openhuman/memory_tree/types.rs rename to src/openhuman/memory_store/chunks/types.rs diff --git a/src/openhuman/memory/store/client.rs b/src/openhuman/memory_store/client.rs similarity index 97% rename from src/openhuman/memory/store/client.rs rename to src/openhuman/memory_store/client.rs index fac8e108c..3e0f2e086 100644 --- a/src/openhuman/memory/store/client.rs +++ b/src/openhuman/memory_store/client.rs @@ -17,10 +17,10 @@ use crate::openhuman::memory::ingestion::{ IngestionJob, IngestionQueue, IngestionState, MemoryIngestionConfig, MemoryIngestionRequest, MemoryIngestionResult, }; -use crate::openhuman::memory::store::types::{ +use crate::openhuman::memory_store::types::{ NamespaceDocumentInput, NamespaceMemoryHit, NamespaceRetrievalContext, }; -use crate::openhuman::memory::store::unified::UnifiedMemory; +use crate::openhuman::memory_store::unified::UnifiedMemory; /// Reference-counted handle to a `MemoryClient`. pub type MemoryClientRef = Arc; @@ -43,8 +43,7 @@ pub struct MemoryState(pub std::sync::Mutex>); /// first `embed` call rather than at client construction. /// /// Callers that need a non-default embedder should construct the underlying -/// store via [`crate::openhuman::memory::create_memory_with_storage_and_routes`] -/// (or [`crate::openhuman::memory::create_memory_with_local_ai`]) with the +/// store via [`crate::openhuman::memory::create_memory_with_local_ai`] with the /// appropriate `MemoryConfig.embedding_provider`. #[derive(Clone)] pub struct MemoryClient { @@ -57,7 +56,7 @@ pub struct MemoryClient { impl MemoryClient { /// Returns a handle to the underlying SQLite connection for direct /// profile-facet writes via - /// [`crate::openhuman::memory::store::unified::profile::profile_upsert`]. + /// [`crate::openhuman::memory_store::unified::profile::profile_upsert`]. /// /// Intentionally `pub(crate)` — external consumers should use the /// higher-level `MemoryClient` API; this escape hatch exists so @@ -109,7 +108,7 @@ impl MemoryClient { // unauthenticated session produces a clear error on first embed rather // than blocking client construction. Callers that need the local // Ollama path should build their memory store via - // `create_memory_with_storage_and_routes` with the appropriate + // `create_memory_with_local_ai` with the appropriate // `MemoryConfig.embedding_provider`. let embedder: Arc = embeddings::default_embedding_provider(); diff --git a/src/openhuman/memory/store/client_tests.rs b/src/openhuman/memory_store/client_tests.rs similarity index 100% rename from src/openhuman/memory/store/client_tests.rs rename to src/openhuman/memory_store/client_tests.rs diff --git a/src/openhuman/memory_store/contacts/mod.rs b/src/openhuman/memory_store/contacts/mod.rs new file mode 100644 index 000000000..aaa8ec0fa --- /dev/null +++ b/src/openhuman/memory_store/contacts/mod.rs @@ -0,0 +1,65 @@ +//! Contacts storage facade. +//! +//! Contacts (the `people` domain — resolver, scorer, address book) are owned +//! by `src/openhuman/people/`. This module re-exports the storage surface so +//! `memory_store` is a single import point for ALL stored memory kinds: raw +//! content, chunks, summary trees, vectors, AND contacts. +//! +//! No data is duplicated: `PeopleStore` is the source of truth and its +//! singleton (`people::store::get`) backs every call routed through here. +//! When the user asks "what does memory_store store?", the answer is the +//! union of: chunks, content (md files), trees (Source/Global/Topic), +//! vectors, AND contacts. + +pub use crate::openhuman::people::store::{get as get_store, init as init_store, PeopleStore}; +pub use crate::openhuman::people::types::{ + AddressBookContact, Handle, Interaction, Person, PersonId, ScoreComponents, +}; + +/// Async helper: load a contact by id from the global PeopleStore. Returns +/// `Ok(None)` when no PersonId matches (and when the store hasn't been +/// initialized — callers in early-boot paths shouldn't crash). +pub async fn get_contact(person_id: PersonId) -> Option { + match get_store() { + Ok(s) => s.get(person_id).await.ok().flatten(), + Err(_) => None, + } +} + +/// Async helper: list every stored contact. Returns an empty vec if the store +/// is uninitialized — matches the read-side fail-soft contract used by the +/// rest of the memory_store retrieval surface. +pub async fn list_contacts() -> Vec { + match get_store() { + Ok(s) => s.list().await.unwrap_or_default(), + Err(_) => Vec::new(), + } +} + +/// Async helper: resolve a handle to its canonical `PersonId` without +/// inserting. Used by retrieval paths that need a person id but don't want +/// to mint a new contact for an unknown handle. +pub async fn lookup_contact(handle: &Handle) -> Option { + match get_store() { + Ok(s) => s.lookup(handle).await.ok().flatten(), + Err(_) => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use uuid::Uuid; + + #[tokio::test] + async fn contact_facade_fail_softs_when_store_is_uninitialized() { + let missing = get_contact(PersonId(Uuid::nil())).await; + assert!(missing.is_none()); + + let listed = list_contacts().await; + assert!(listed.is_empty()); + + let looked_up = lookup_contact(&Handle::Email("nobody@example.com".into())).await; + assert!(looked_up.is_none()); + } +} diff --git a/src/openhuman/memory_tree/content_store/README.md b/src/openhuman/memory_store/content/README.md similarity index 85% rename from src/openhuman/memory_tree/content_store/README.md rename to src/openhuman/memory_store/content/README.md index 0aba3d865..f33e81cbb 100644 --- a/src/openhuman/memory_tree/content_store/README.md +++ b/src/openhuman/memory_store/content/README.md @@ -11,6 +11,8 @@ The body is **immutable** once written — only the YAML front-matter `tags:` bl - [`compose.rs`](compose.rs) — YAML front-matter + body composition. `compose_chunk_file` for chunks (with email-only `participants:` / `aliases:` fields parsed from `gmail:{addr1|addr2|…}` source ids), `compose_summary_md` for summary nodes. `rewrite_tags` / `rewrite_summary_tags` swap the `tags:` block in place. `split_front_matter` parses `---\n…\n---\n`. - [`paths.rs`](paths.rs) — path generators. `chunk_rel_path` (`email//.md`, `chat//.md`, `document//.md`); `summary_rel_path` (`summaries/{source,global,topic}/…`). `slugify_source_id` is the canonical filesystem-safe slug. - [`read.rs`](read.rs) — `read_chunk_file` / `read_summary_file` parse front-matter and return body+SHA. `verify_*` compares against an expected SHA. `read_chunk_body` / `read_summary_body` resolve the path via SQLite and verify the integrity hash; this is the authoritative entry-point for callers that need the **full** body (LLM extractor, summariser, embedder, retrieval API). +- [`raw.rs`](raw.rs) — verbatim source-byte mirror under `/raw/`. Writes the unmodified upstream payload (eml, slack json, raw markdown) so downstream callers can re-canonicalise without re-fetching. +- [`obsidian.rs`](obsidian.rs) + [`obsidian_defaults/`](obsidian_defaults/) — bootstrap an `.obsidian/` config (workspace, graph, app) into the content root on first write so a user opening the vault gets a usable view. - [`tags.rs`](tags.rs) — post-extraction tag rewrites. `update_chunk_tags` (atomic tempfile rewrite of the `tags:` block) and `update_summary_tags` (fetches entities from `mem_tree_entity_index`, builds Obsidian `kind/Value` tags, rewrites, verifies body SHA is unchanged). `slugify_tag_kind`, `slugify_tag_value`, `entity_tag` build the tag strings. ## Integrity contract diff --git a/src/openhuman/memory_tree/content_store/atomic.rs b/src/openhuman/memory_store/content/atomic.rs similarity index 98% rename from src/openhuman/memory_tree/content_store/atomic.rs rename to src/openhuman/memory_store/content/atomic.rs index b1a3c4276..c8d780392 100644 --- a/src/openhuman/memory_tree/content_store/atomic.rs +++ b/src/openhuman/memory_store/content/atomic.rs @@ -230,8 +230,8 @@ fn uuid_v4_hex() -> String { #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory_tree::content_store::compose::SummaryComposeInput; - use crate::openhuman::memory_tree::content_store::paths::SummaryTreeKind; + use crate::openhuman::memory_store::content::compose::SummaryComposeInput; + use crate::openhuman::memory_store::content::paths::SummaryTreeKind; use tempfile::TempDir; #[test] diff --git a/src/openhuman/memory_tree/content_store/compose.rs b/src/openhuman/memory_store/content/compose.rs similarity index 99% rename from src/openhuman/memory_tree/content_store/compose.rs rename to src/openhuman/memory_store/content/compose.rs index 67216080c..9ff02a4eb 100644 --- a/src/openhuman/memory_tree/content_store/compose.rs +++ b/src/openhuman/memory_store/content/compose.rs @@ -38,10 +38,10 @@ use chrono::{DateTime, Utc}; -use crate::openhuman::memory_tree::content_store::paths::{ +use crate::openhuman::memory_store::chunks::types::{Chunk, SourceKind}; +use crate::openhuman::memory_store::content::paths::{ slugify_source_id, summary_filename, SummaryTreeKind, }; -use crate::openhuman::memory_tree::types::{Chunk, SourceKind}; pub const MEMORY_ARTIFACT_FORMAT: u32 = 2; pub const OPENHUMAN_CORE_VERSION: &str = env!("CARGO_PKG_VERSION"); @@ -624,8 +624,8 @@ fn yaml_scalar(s: &str) -> String { #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory_tree::content_store::paths::SummaryTreeKind; - use crate::openhuman::memory_tree::types::{Metadata, SourceKind, SourceRef}; + use crate::openhuman::memory_store::chunks::types::{Metadata, SourceKind, SourceRef}; + use crate::openhuman::memory_store::content::paths::SummaryTreeKind; use chrono::TimeZone; fn sample_chunk() -> Chunk { diff --git a/src/openhuman/memory_tree/content_store/mod.rs b/src/openhuman/memory_store/content/mod.rs similarity index 97% rename from src/openhuman/memory_tree/content_store/mod.rs rename to src/openhuman/memory_store/content/mod.rs index cd2c7bdd4..134a634ee 100644 --- a/src/openhuman/memory_tree/content_store/mod.rs +++ b/src/openhuman/memory_store/content/mod.rs @@ -22,7 +22,7 @@ pub mod tags; use std::path::Path; -use crate::openhuman::memory_tree::types::Chunk; +use crate::openhuman::memory_store::chunks::types::Chunk; pub use atomic::StagedSummary; pub use compose::SummaryComposeInput; @@ -70,7 +70,7 @@ pub fn update_summary_tags( /// /// `content_root` — absolute path to the root of the content store. pub fn stage_chunks(content_root: &Path, chunks: &[Chunk]) -> anyhow::Result> { - use crate::openhuman::memory_tree::types::SourceKind; + use crate::openhuman::memory_store::chunks::types::SourceKind; let mut staged = Vec::with_capacity(chunks.len()); for chunk in chunks { @@ -128,7 +128,7 @@ pub fn stage_chunks(content_root: &Path, chunks: &[Chunk]) -> anyhow::Result/wiki/summaries/` — flattened diff --git a/src/openhuman/memory_tree/content_store/raw.rs b/src/openhuman/memory_store/content/raw.rs similarity index 99% rename from src/openhuman/memory_tree/content_store/raw.rs rename to src/openhuman/memory_store/content/raw.rs index e9d5fee0c..caa842ccc 100644 --- a/src/openhuman/memory_tree/content_store/raw.rs +++ b/src/openhuman/memory_store/content/raw.rs @@ -132,7 +132,7 @@ pub fn raw_kind_dir(content_root: &Path, source_id: &str, kind: RawKind) -> Path /// Forward-slash relative path of a raw file under `/`, /// e.g. `"raw/gmail-acct/emails/1700000000000_msg-1.md"`. Used by -/// callers that record a [`crate::openhuman::memory_tree::store::RawRef`] +/// callers that record a [`crate::openhuman::memory_store::chunks::store::RawRef`] /// so reads can resolve the file later without re-deriving the layout. pub fn raw_rel_path(source_id: &str, kind: RawKind, created_at_ms: i64, uid: &str) -> String { let slug = slugify_source_id(source_id); diff --git a/src/openhuman/memory_tree/content_store/read.rs b/src/openhuman/memory_store/content/read.rs similarity index 65% rename from src/openhuman/memory_tree/content_store/read.rs rename to src/openhuman/memory_store/content/read.rs index 4bf755cf7..1f2650800 100644 --- a/src/openhuman/memory_tree/content_store/read.rs +++ b/src/openhuman/memory_store/content/read.rs @@ -4,7 +4,7 @@ use std::path::Path; use super::atomic::sha256_hex; use super::compose::split_front_matter; -use crate::openhuman::memory_tree::util::redact::redact; +use crate::openhuman::memory::util::redact::redact; /// The result of reading a chunk file from disk. pub struct ChunkFileContents { @@ -143,7 +143,9 @@ pub fn read_chunk_body( config: &crate::openhuman::config::Config, chunk_id: &str, ) -> anyhow::Result { - use crate::openhuman::memory_tree::store::{get_chunk_content_pointers, get_chunk_raw_refs}; + use crate::openhuman::memory_store::chunks::store::{ + get_chunk_content_pointers, get_chunk_raw_refs, + }; // Path 1: chunk has raw-archive pointers (today: email). Read each // referenced file, slice by byte range, join with `\n\n` (the @@ -230,7 +232,7 @@ use anyhow::Context as _; /// missing raw file doesn't take the whole chunk down. fn read_chunk_body_from_raw( config: &crate::openhuman::config::Config, - refs: &[crate::openhuman::memory_tree::store::RawRef], + refs: &[crate::openhuman::memory_store::chunks::store::RawRef], ) -> anyhow::Result { let content_root = config.memory_tree_content_root(); let mut parts: Vec = Vec::with_capacity(refs.len()); @@ -287,7 +289,7 @@ pub fn read_summary_body( config: &crate::openhuman::config::Config, summary_id: &str, ) -> anyhow::Result { - use crate::openhuman::memory_tree::store::get_summary_content_pointers; + use crate::openhuman::memory_store::chunks::store::get_summary_content_pointers; let pointers = get_summary_content_pointers(config, summary_id)?.ok_or_else(|| { anyhow::anyhow!( @@ -339,9 +341,17 @@ pub fn read_summary_body( #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory_tree::content_store::atomic::{sha256_hex, write_if_new}; - use crate::openhuman::memory_tree::content_store::compose::compose_chunk_file; - use crate::openhuman::memory_tree::types::{Chunk, Metadata, SourceKind}; + use crate::openhuman::config::Config; + use crate::openhuman::memory_store::chunks::store::{upsert_chunks, with_connection}; + use crate::openhuman::memory_store::chunks::types::{Chunk, Metadata, SourceKind}; + use crate::openhuman::memory_store::content::atomic::{sha256_hex, write_if_new}; + use crate::openhuman::memory_store::content::compose::{ + compose_chunk_file, SummaryComposeInput, + }; + use crate::openhuman::memory_store::content::paths::SummaryTreeKind; + use crate::openhuman::memory_store::content::{atomic::stage_summary, stage_chunks}; + use crate::openhuman::memory_store::trees::store::{insert_summary_tx, insert_tree}; + use crate::openhuman::memory_store::trees::types::{SummaryNode, Tree, TreeKind, TreeStatus}; use chrono::TimeZone; use tempfile::TempDir; @@ -366,6 +376,47 @@ mod tests { } } + fn test_config(tmp: &TempDir) -> Config { + let mut cfg = Config::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + cfg + } + + fn sample_tree() -> Tree { + Tree { + id: "tree-1".into(), + kind: TreeKind::Source, + scope: "slack:#eng".into(), + root_id: None, + max_level: 0, + status: TreeStatus::Active, + created_at: chrono::Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(), + last_sealed_at: None, + } + } + + fn sample_summary_node() -> SummaryNode { + let ts = chrono::Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(); + SummaryNode { + id: "summary-1".into(), + tree_id: "tree-1".into(), + tree_kind: TreeKind::Source, + level: 1, + parent_id: None, + child_ids: vec!["leaf-a".into()], + content: "summary full body".into(), + token_count: 4, + entities: vec![], + topics: vec![], + time_range_start: ts, + time_range_end: ts, + score: 0.5, + sealed_at: ts, + deleted: false, + embedding: None, + } + } + #[test] fn read_returns_body_and_correct_sha256() { let dir = TempDir::new().unwrap(); @@ -412,11 +463,11 @@ mod tests { // ─── summary read / verify tests ───────────────────────────────────────── fn write_summary_file(dir: &TempDir, body: &str) -> (std::path::PathBuf, String) { - use crate::openhuman::memory_tree::content_store::atomic::{sha256_hex, write_if_new}; - use crate::openhuman::memory_tree::content_store::compose::{ + use crate::openhuman::memory_store::content::atomic::{sha256_hex, write_if_new}; + use crate::openhuman::memory_store::content::compose::{ compose_summary_md, SummaryComposeInput, }; - use crate::openhuman::memory_tree::content_store::paths::SummaryTreeKind; + use crate::openhuman::memory_store::content::paths::SummaryTreeKind; use chrono::TimeZone; let ts = chrono::Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(); let input = SummaryComposeInput { @@ -474,4 +525,191 @@ mod tests { VerifyResult::Missing ); } + + #[test] + fn read_chunk_file_rejects_invalid_utf8() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("bad.md"); + std::fs::write(&path, [0xff, 0xfe, 0xfd]).unwrap(); + let err = match read_chunk_file(&path) { + Ok(_) => panic!("invalid UTF-8 should fail"), + Err(err) => err, + }; + assert!(err.to_string().contains("invalid UTF-8")); + } + + #[test] + fn read_chunk_file_rejects_missing_front_matter() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("plain.md"); + std::fs::write(&path, "no front matter here").unwrap(); + let err = match read_chunk_file(&path) { + Ok(_) => panic!("missing front matter should fail"), + Err(err) => err, + }; + assert!(err.to_string().contains("no front-matter")); + } + + #[test] + fn verify_summary_file_mismatch_returns_actual_sha() { + let dir = TempDir::new().unwrap(); + let (path, expected_sha) = write_summary_file(&dir, "body text\n"); + let actual = match verify_summary_file(&path, "deadbeef").unwrap() { + VerifyResult::Mismatch { actual } => actual, + other => panic!("expected mismatch, got {other:?}"), + }; + assert_eq!(actual, expected_sha); + } + + #[test] + fn read_chunk_body_from_raw_clamps_ranges_and_skips_bad_refs() { + use crate::openhuman::memory_store::chunks::store::RawRef; + + let dir = TempDir::new().unwrap(); + let mut cfg = crate::openhuman::config::Config::default(); + cfg.workspace_dir = dir.path().to_path_buf(); + + let content_root = cfg.memory_tree_content_root(); + std::fs::create_dir_all(&content_root).unwrap(); + + std::fs::write(content_root.join("one.txt"), "abcdef").unwrap(); + std::fs::write(content_root.join("two.txt"), [0xff, 0xfe]).unwrap(); + + let refs = vec![ + RawRef { + path: "one.txt".into(), + start: 1, + end: Some(4), + }, + RawRef { + path: "missing.txt".into(), + start: 0, + end: None, + }, + RawRef { + path: "two.txt".into(), + start: 0, + end: None, + }, + RawRef { + path: "one.txt".into(), + start: 99, + end: None, + }, + ]; + + let body = read_chunk_body_from_raw(&cfg, &refs).unwrap(); + assert_eq!(body, "bcd"); + } + + #[test] + fn read_chunk_body_roundtrips_from_staged_content_pointer() { + let tmp = TempDir::new().unwrap(); + let cfg = test_config(&tmp); + let chunk = sample_chunk(); + upsert_chunks(&cfg, std::slice::from_ref(&chunk)).unwrap(); + let staged = stage_chunks( + &cfg.memory_tree_content_root(), + std::slice::from_ref(&chunk), + ) + .unwrap(); + with_connection(&cfg, |conn| { + let tx = conn.unchecked_transaction()?; + crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(&tx, &staged)?; + tx.commit()?; + Ok(()) + }) + .unwrap(); + + let body = read_chunk_body(&cfg, &chunk.id).unwrap(); + assert_eq!(body, chunk.content); + } + + #[test] + fn read_chunk_body_errors_when_pointers_are_missing() { + let tmp = TempDir::new().unwrap(); + let cfg = test_config(&tmp); + let err = read_chunk_body(&cfg, "missing-chunk").unwrap_err(); + assert!(err.to_string().contains("no content_path or raw_refs")); + } + + #[test] + fn read_chunk_body_errors_on_sha_mismatch() { + let tmp = TempDir::new().unwrap(); + let cfg = test_config(&tmp); + let chunk = sample_chunk(); + upsert_chunks(&cfg, std::slice::from_ref(&chunk)).unwrap(); + let staged = stage_chunks( + &cfg.memory_tree_content_root(), + std::slice::from_ref(&chunk), + ) + .unwrap(); + with_connection(&cfg, |conn| { + let tx = conn.unchecked_transaction()?; + crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(&tx, &staged)?; + tx.commit()?; + Ok(()) + }) + .unwrap(); + + let rel = + crate::openhuman::memory_store::chunks::store::get_chunk_content_path(&cfg, &chunk.id) + .unwrap() + .unwrap(); + let mut abs = cfg.memory_tree_content_root(); + for part in rel.split('/') { + abs.push(part); + } + std::fs::write(&abs, b"---\nsource_kind: chat\n---\nmutated body").unwrap(); + + let err = read_chunk_body(&cfg, &chunk.id).unwrap_err(); + assert!(err.to_string().contains("sha256 mismatch")); + } + + #[test] + fn read_summary_body_roundtrips_from_staged_content_pointer() { + let tmp = TempDir::new().unwrap(); + let cfg = test_config(&tmp); + let tree = sample_tree(); + let node = sample_summary_node(); + insert_tree(&cfg, &tree).unwrap(); + let staged = stage_summary( + &cfg.memory_tree_content_root(), + &SummaryComposeInput { + summary_id: &node.id, + tree_kind: SummaryTreeKind::Source, + tree_id: &tree.id, + tree_scope: &tree.scope, + level: node.level, + child_ids: &node.child_ids, + child_basenames: None, + child_count: node.child_ids.len(), + time_range_start: node.time_range_start, + time_range_end: node.time_range_end, + sealed_at: node.sealed_at, + body: &node.content, + }, + "slack-eng", + None, + ) + .unwrap(); + with_connection(&cfg, |conn| { + let tx = conn.unchecked_transaction()?; + insert_summary_tx(&tx, &node, Some(&staged), "test")?; + tx.commit()?; + Ok(()) + }) + .unwrap(); + + let body = read_summary_body(&cfg, &node.id).unwrap(); + assert_eq!(body, node.content); + } + + #[test] + fn read_summary_body_errors_when_pointers_are_missing() { + let tmp = TempDir::new().unwrap(); + let cfg = test_config(&tmp); + let err = read_summary_body(&cfg, "missing-summary").unwrap_err(); + assert!(err.to_string().contains("no content_path for summary_id")); + } } diff --git a/src/openhuman/memory_tree/content_store/tags.rs b/src/openhuman/memory_store/content/tags.rs similarity index 96% rename from src/openhuman/memory_tree/content_store/tags.rs rename to src/openhuman/memory_store/content/tags.rs index 50e02dfff..eb776640e 100644 --- a/src/openhuman/memory_tree/content_store/tags.rs +++ b/src/openhuman/memory_store/content/tags.rs @@ -14,8 +14,8 @@ use super::compose::{ split_front_matter, }; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::score::store::list_entity_ids_for_node; -use crate::openhuman::memory_tree::store::get_summary_content_pointers; +use crate::openhuman::memory::score::store::list_entity_ids_for_node; +use crate::openhuman::memory_store::chunks::store::get_summary_content_pointers; /// Rewrite the `tags:` block in a chunk's on-disk `.md` file. /// @@ -334,9 +334,9 @@ fn crate_temp_id() -> String { #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory_tree::content_store::atomic::{sha256_hex, write_if_new}; - use crate::openhuman::memory_tree::content_store::compose::compose_chunk_file; - use crate::openhuman::memory_tree::types::{Chunk, Metadata, SourceKind}; + use crate::openhuman::memory_store::chunks::types::{Chunk, Metadata, SourceKind}; + use crate::openhuman::memory_store::content::atomic::{sha256_hex, write_if_new}; + use crate::openhuman::memory_store::content::compose::compose_chunk_file; use chrono::TimeZone; use tempfile::TempDir; @@ -430,10 +430,10 @@ mod tests { /// Write a summary .md file to disk with empty tags and verify rewriting works. #[test] fn rewrite_summary_tags_preserves_body_and_replaces_tags() { - use crate::openhuman::memory_tree::content_store::compose::{ + use crate::openhuman::memory_store::content::compose::{ compose_summary_md, SummaryComposeInput, }; - use crate::openhuman::memory_tree::content_store::paths::SummaryTreeKind; + use crate::openhuman::memory_store::content::paths::SummaryTreeKind; let dir = TempDir::new().unwrap(); let ts = chrono::Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(); @@ -483,7 +483,7 @@ mod tests { assert!(updated.ends_with(body)); // Body sha unchanged - use crate::openhuman::memory_tree::content_store::compose::split_front_matter; + use crate::openhuman::memory_store::content::compose::split_front_matter; let (_, body_after) = split_front_matter(&updated).unwrap(); let sha = sha256_hex(body_after.as_bytes()); let expected_sha = sha256_hex(body.as_bytes()); diff --git a/src/openhuman/memory_store/entities/mod.rs b/src/openhuman/memory_store/entities/mod.rs new file mode 100644 index 000000000..9e6f2b915 --- /dev/null +++ b/src/openhuman/memory_store/entities/mod.rs @@ -0,0 +1,55 @@ +//! Entities — the `mem_tree_entity_index` table surfaced as a first-class +//! memory_store submodule. +//! +//! The entity index is one of the four primitives memory_store owns +//! (raw / entities / tree / vector + kv). Today its persistence lives in +//! `memory::score::store` because the scorer was the first writer; this +//! module re-exports the read/write surface under the canonical +//! `memory_store::entities::*` path so callers don't have to know about +//! the implementation location. +//! +//! Once the score module finishes splitting (entity persistence vs +//! scoring math), the table-owning code moves here and `memory::score` +//! becomes a pure consumer. +//! +//! ## API +//! +//! | Re-export | Source | +//! | --- | --- | +//! | [`EntityHit`] | `memory::score::store::EntityHit` | +//! | [`index_entity`] | `memory::score::store::index_entity` | +//! | [`index_entities`] | `memory::score::store::index_entities` | +//! | [`lookup_entity`] | `memory::score::store::lookup_entity` | +//! | [`list_entity_ids_for_node`] | `memory::score::store::list_entity_ids_for_node` | +//! | [`clear_entity_index_for_node`] | `memory::score::store::clear_entity_index_for_node` | +//! | [`count_entity_index`] | `memory::score::store::count_entity_index` | +//! +//! See [`crate::openhuman::memory_graph`] for the derived co-occurrence +//! query layer built on top of these primitives. + +pub use crate::openhuman::memory::score::store::{ + clear_entity_index_for_node, count_entity_index, index_entities, index_entity, + list_entity_ids_for_node, lookup_entity, EntityHit, +}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn entity_hit_reexport_is_constructible() { + let hit = EntityHit { + entity_id: "person:alice".into(), + node_id: "chunk-1".into(), + node_kind: "leaf".into(), + entity_kind: crate::openhuman::memory::score::extract::EntityKind::Person, + surface: "Alice".into(), + score: 1.0, + timestamp_ms: 123, + tree_id: Some("tree-1".into()), + is_user: false, + }; + assert_eq!(hit.entity_id, "person:alice"); + assert_eq!(hit.node_kind, "leaf"); + } +} diff --git a/src/openhuman/memory/store/factories.rs b/src/openhuman/memory_store/factories.rs similarity index 91% rename from src/openhuman/memory/store/factories.rs rename to src/openhuman/memory_store/factories.rs index 19d29bc80..a40bcd6ba 100644 --- a/src/openhuman/memory/store/factories.rs +++ b/src/openhuman/memory_store/factories.rs @@ -17,23 +17,8 @@ use crate::openhuman::embeddings::{ self, format_embedding_signature, EmbeddingProvider, DEFAULT_CLOUD_EMBEDDING_DIMENSIONS, DEFAULT_CLOUD_EMBEDDING_MODEL, DEFAULT_OLLAMA_DIMENSIONS, DEFAULT_OLLAMA_MODEL, }; -use crate::openhuman::memory::store::agentmemory::AgentMemoryBackend; -use crate::openhuman::memory::store::unified::UnifiedMemory; use crate::openhuman::memory::traits::Memory; - -/// Stable wire string for the agentmemory backend selector. -/// -/// When `MemoryConfig.backend` matches this (ASCII case-insensitive), the -/// memory factory short-circuits the SQLite + embedder path and returns an -/// [`AgentMemoryBackend`] that proxies trait calls through agentmemory's -/// REST surface. OpenHuman's `embedding_provider` / `embedding_model` / -/// `embedding_dimensions` are ignored on this path — agentmemory owns its -/// embedding stack via `~/.agentmemory/.env`. -pub const AGENTMEMORY_BACKEND: &str = "agentmemory"; - -fn is_agentmemory_backend(name: &str) -> bool { - name.eq_ignore_ascii_case(AGENTMEMORY_BACKEND) -} +use crate::openhuman::memory_store::unified::UnifiedMemory; /// One-shot guard so the Ollama health-gate fallback only reports to Sentry /// once per process lifetime. Memory is constructed many times per session @@ -281,16 +266,7 @@ pub fn create_memory( config: &MemoryConfig, workspace_dir: &Path, ) -> anyhow::Result> { - create_memory_with_storage_and_routes(config, &[], None, workspace_dir) -} - -/// Create a memory instance with an optional storage provider configuration. -pub fn create_memory_with_storage( - config: &MemoryConfig, - storage_provider: Option<&StorageProviderConfig>, - workspace_dir: &Path, -) -> anyhow::Result> { - create_memory_full(config, &[], storage_provider, None, workspace_dir) + create_memory_full(config, &[], None, None, workspace_dir) } /// Create a memory instance honouring the unified per-workload embedding @@ -317,24 +293,6 @@ pub fn create_memory_with_local_ai( ) } -/// Back-compat wrapper preserved for existing call sites that don't have a -/// `LocalAiConfig` to pass. The local-AI opt-in is not honored on this path — -/// use [`create_memory_with_local_ai`] when both sections are available. -pub fn create_memory_with_storage_and_routes( - config: &MemoryConfig, - embedding_routes: &[EmbeddingRouteConfig], - storage_provider: Option<&StorageProviderConfig>, - workspace_dir: &Path, -) -> anyhow::Result> { - create_memory_full( - config, - embedding_routes, - storage_provider, - None, - workspace_dir, - ) -} - /// Synchronous health-check shim around [`probe_ollama_reachable`]. /// /// Production call sites (`create_memory_with_local_ai` and friends) live in @@ -382,25 +340,6 @@ fn create_memory_full( local_embedding_model: Option<&str>, workspace_dir: &Path, ) -> anyhow::Result> { - // 0. Short-circuit the unified path when the user has explicitly - // selected the agentmemory backend. agentmemory owns its own - // embedding stack, persistence, and graph layer — wiring a local - // embedder + SQLite store on top of it would duplicate the - // embedding pipeline and create a divergence between OpenHuman's - // cached vectors and agentmemory's. Fail loud at boot if the daemon - // isn't reachable (per the issue #1664 fallback decision). - if is_agentmemory_backend(&config.backend) { - log::info!( - "[memory::factory] using agentmemory backend at {}", - config - .agentmemory_url - .as_deref() - .unwrap_or(crate::openhuman::memory::store::agentmemory_default_url()), - ); - let backend = AgentMemoryBackend::from_config(config)?; - return Ok(Box::new(backend)); - } - // 1. Resolve the intended provider from config. let intended = effective_embedding_settings(config, local_embedding_model); let local_ai_opt_in = local_embedding_model diff --git a/src/openhuman/memory_store/kinds.rs b/src/openhuman/memory_store/kinds.rs new file mode 100644 index 000000000..1187d484a --- /dev/null +++ b/src/openhuman/memory_store/kinds.rs @@ -0,0 +1,112 @@ +//! Catalog of every kind of data the memory_store persists. +//! +//! Every stored object falls into exactly one of these kinds. The enum is the +//! authoritative answer to "what can memory_store store?" and is used by: +//! - The retrieval facade, to fan out a query to the right backends. +//! - The vector/obsidian compatibility traits, to dispatch by kind. +//! - Agent tools, to surface a kind filter to LLM callers. +//! +//! Adding a new storage kind = adding a variant here, an impl of the +//! [`VectorEmbeddable`] / [`ObsidianRepresentable`] traits +//! ([`crate::openhuman::memory_store::traits`]), and a delegation in +//! [`crate::openhuman::memory_store::retrieval`]. + +use serde::{Deserialize, Serialize}; + +/// Every persisted data shape in memory_store, named once. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MemoryKind { + /// On-disk raw markdown file (the content store). One file per + /// canonicalized source chunk OR per summary node. Source of truth + /// for all content bodies. + Raw, + /// SQLite chunk row — metadata + tags + raw-md pointer + lifecycle. + /// Bodies live in `Raw`; the chunk row is the index entry. + Chunk, + /// Canonical entity row in `mem_tree_entity_index` — every entity + /// occurrence per tree node. The substrate `memory_graph` derives + /// co-occurrence edges from. + Entity, + /// Sealed summary tree node — Source, Global, or Topic flavor. + Tree, + /// Dense vector embedding row in the local vector DB. + Vector, + /// Key-value record (global or namespace-scoped). Lives in the + /// `kv_global` / `kv_namespace` tables. + Kv, + /// Address-book contact (`people::Person`) routed through the contacts + /// facade. + Contact, +} + +impl MemoryKind { + /// Snake-case discriminant used in RPC payloads, logs, and tool args. + pub fn as_str(self) -> &'static str { + match self { + MemoryKind::Raw => "raw", + MemoryKind::Chunk => "chunk", + MemoryKind::Entity => "entity", + MemoryKind::Tree => "tree", + MemoryKind::Vector => "vector", + MemoryKind::Kv => "kv", + MemoryKind::Contact => "contact", + } + } + + /// Every variant, in stable declaration order. Useful for fan-out + /// retrieval and for surfacing the kind catalog to LLM tools. + pub const ALL: &'static [MemoryKind] = &[ + MemoryKind::Raw, + MemoryKind::Chunk, + MemoryKind::Entity, + MemoryKind::Tree, + MemoryKind::Vector, + MemoryKind::Kv, + MemoryKind::Contact, + ]; +} + +/// Per-kind canonical Rust type aliases — one stop to find "what struct +/// represents a Tree row?", "what struct represents a Contact?", etc. +/// Aliases (not re-exports) so the documentation lives here and the +/// source-of-truth types stay in their owning modules. +pub mod types { + pub use crate::openhuman::memory_store::chunks::types::Chunk; + pub use crate::openhuman::memory_store::contacts::Person as Contact; + pub use crate::openhuman::memory_store::entities::EntityHit as Entity; + pub use crate::openhuman::memory_store::trees::{SummaryNode as TreeNode, Tree, TreeKind}; + pub use crate::openhuman::memory_store::types::MemoryKvRecord as Kv; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn memory_kind_as_str_matches_all_catalog_entries() { + let kinds = [ + MemoryKind::Raw, + MemoryKind::Chunk, + MemoryKind::Entity, + MemoryKind::Tree, + MemoryKind::Vector, + MemoryKind::Kv, + MemoryKind::Contact, + ]; + let labels: Vec<&str> = kinds.iter().map(|k| k.as_str()).collect(); + let all: Vec<&str> = MemoryKind::ALL.iter().map(|k| k.as_str()).collect(); + assert_eq!(labels, all); + } + + #[test] + fn memory_kind_serde_uses_snake_case() { + let raw = serde_json::to_string(&MemoryKind::Raw).unwrap(); + let tree = serde_json::to_string(&MemoryKind::Tree).unwrap(); + assert_eq!(raw, "\"raw\""); + assert_eq!(tree, "\"tree\""); + + let decoded: MemoryKind = serde_json::from_str("\"contact\"").unwrap(); + assert_eq!(decoded, MemoryKind::Contact); + } +} diff --git a/src/openhuman/memory/store/unified/kv.rs b/src/openhuman/memory_store/kv.rs similarity index 75% rename from src/openhuman/memory/store/unified/kv.rs rename to src/openhuman/memory_store/kv.rs index f30d0ddb2..1b2455b25 100644 --- a/src/openhuman/memory/store/unified/kv.rs +++ b/src/openhuman/memory_store/kv.rs @@ -1,15 +1,18 @@ -//! Key-value storage backed by the `kv_global` and `kv_namespace` tables. +//! Key-value storage — `kv_global` + `kv_namespace` tables. //! -//! Provides global and namespace-scoped get/set/delete/list, plus internal -//! record loaders used by the retrieval pipeline. +//! Lifted out of `unified/` so KV is a peer of `trees/`, `vectors/`, and +//! the other first-class memory_store submodules. The `impl UnifiedMemory` +//! block stays here because the methods still operate on the unified +//! SQLite connection; once `Memory` trait callers migrate to a per-kind +//! backend, the `UnifiedMemory` impl shrinks to a thin shim and the bulk +//! of this file moves to free functions. use rusqlite::{params, OptionalExtension}; use serde_json::json; -use crate::openhuman::memory::safety; -use crate::openhuman::memory::store::types::MemoryKvRecord; - -use super::UnifiedMemory; +use crate::openhuman::memory_store::safety; +use crate::openhuman::memory_store::types::MemoryKvRecord; +use crate::openhuman::memory_store::unified::UnifiedMemory; impl UnifiedMemory { /// Insert or update a global key-value pair. @@ -264,3 +267,85 @@ impl UnifiedMemory { Ok(out) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::embeddings::NoopEmbedding; + use tempfile::TempDir; + + fn test_memory() -> (TempDir, UnifiedMemory) { + let tmp = TempDir::new().unwrap(); + let memory = + UnifiedMemory::new(tmp.path(), std::sync::Arc::new(NoopEmbedding), None).unwrap(); + (tmp, memory) + } + + #[tokio::test] + async fn global_kv_roundtrips_and_deletes() { + let (_tmp, memory) = test_memory(); + memory.kv_set_global("theme", &json!("dark")).await.unwrap(); + assert_eq!( + memory.kv_get_global("theme").await.unwrap(), + Some(json!("dark")) + ); + + assert!(memory.kv_delete_global("theme").await.unwrap()); + assert_eq!(memory.kv_get_global("theme").await.unwrap(), None); + } + + #[tokio::test] + async fn namespace_kv_roundtrips_lists_and_combines_scope_records() { + let (_tmp, memory) = test_memory(); + memory + .kv_set_global("global-setting", &json!(true)) + .await + .unwrap(); + memory + .kv_set_namespace("team alpha/#1", "state", &json!({"open": true})) + .await + .unwrap(); + + assert_eq!( + memory + .kv_get_namespace("team alpha/#1", "state") + .await + .unwrap(), + Some(json!({"open": true})) + ); + + let listed = memory.kv_list_namespace("team alpha/#1").await.unwrap(); + assert_eq!(listed.len(), 1); + assert_eq!(listed[0]["key"], "state"); + assert_eq!(listed[0]["value"], json!({"open": true})); + + let scoped = memory.kv_records_for_scope("team alpha/#1").await.unwrap(); + assert_eq!(scoped.len(), 2); + assert!(scoped + .iter() + .any(|r| r.namespace.is_none() && r.key == "global-setting")); + assert!(scoped + .iter() + .any(|r| { r.namespace.as_deref() == Some("team_alpha/_1") && r.key == "state" })); + } + + #[tokio::test] + async fn kv_rejects_secret_like_keys() { + let (_tmp, memory) = test_memory(); + let err = memory + .kv_set_global("sk-proj-abcdefghijklmnop", &json!("secret")) + .await + .unwrap_err(); + assert!(err.contains("cannot contain secrets")); + + let err = memory + .kv_set_namespace( + "project", + "ghp_abcdefghijklmnopqrstuvwx123456", + &json!("secret"), + ) + .await + .unwrap_err(); + assert!(err.contains("cannot contain secrets")); + } +} diff --git a/src/openhuman/memory/store/memory_trait.rs b/src/openhuman/memory_store/memory_trait.rs similarity index 99% rename from src/openhuman/memory/store/memory_trait.rs rename to src/openhuman/memory_store/memory_trait.rs index d0ca3336e..dd0c41901 100644 --- a/src/openhuman/memory/store/memory_trait.rs +++ b/src/openhuman/memory_store/memory_trait.rs @@ -14,11 +14,11 @@ use chrono::{TimeZone, Utc}; use rusqlite::{params, OptionalExtension}; use serde_json::json; -use crate::openhuman::memory::store::types::{NamespaceDocumentInput, GLOBAL_NAMESPACE}; -use crate::openhuman::memory::store::unified::fts5; use crate::openhuman::memory::traits::{ Memory, MemoryCategory, MemoryEntry, NamespaceSummary, RecallOpts, }; +use crate::openhuman::memory_store::types::{NamespaceDocumentInput, GLOBAL_NAMESPACE}; +use crate::openhuman::memory_store::unified::fts5; use anyhow::Context; use super::unified::UnifiedMemory; diff --git a/src/openhuman/memory/store/mod.rs b/src/openhuman/memory_store/mod.rs similarity index 66% rename from src/openhuman/memory/store/mod.rs rename to src/openhuman/memory_store/mod.rs index ba0366f88..ba6869383 100644 --- a/src/openhuman/memory/store/mod.rs +++ b/src/openhuman/memory_store/mod.rs @@ -16,22 +16,32 @@ //! - `factories`: Factory functions for creating and initializing memory instances. //! - `memory_trait`: Defines the `Memory` trait that all implementations must satisfy. +pub mod chunks; +pub mod contacts; +pub mod content; +pub mod entities; +pub mod kinds; +pub mod kv; +pub mod retrieval; +pub mod safety; +pub mod tools; +pub mod traits; +pub mod trees; pub mod types; -mod unified; +pub mod unified; +pub mod vectors; -mod agentmemory; mod client; -mod factories; +pub mod factories; mod memory_trait; -pub use agentmemory::{agentmemory_default_url, AgentMemoryBackend, DEFAULT_AGENTMEMORY_URL}; +pub use kinds::MemoryKind; +pub use traits::{ObsidianFile, ObsidianRepresentable, VectorEmbeddable}; pub use client::{MemoryClient, MemoryClientRef, MemoryState}; pub use factories::{ active_embedding_signature, create_memory, create_memory_for_migration, - create_memory_with_local_ai, create_memory_with_storage, create_memory_with_storage_and_routes, - effective_embedding_settings, effective_embedding_settings_probed, - effective_memory_backend_name, + create_memory_with_local_ai, effective_embedding_settings, effective_memory_backend_name, }; pub use types::{ GraphRelationRecord, MemoryItemKind, MemoryKvRecord, NamespaceDocumentInput, @@ -43,3 +53,15 @@ pub use unified::fts5; pub use unified::profile; pub use unified::segments; pub use unified::UnifiedMemory; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn memory_store_reexports_expected_memory_kind_catalog() { + assert!(MemoryKind::ALL.contains(&MemoryKind::Chunk)); + assert!(MemoryKind::ALL.contains(&MemoryKind::Tree)); + assert!(MemoryKind::ALL.contains(&MemoryKind::Contact)); + } +} diff --git a/src/openhuman/memory_store/retrieval/mod.rs b/src/openhuman/memory_store/retrieval/mod.rs new file mode 100644 index 000000000..e19b845cb --- /dev/null +++ b/src/openhuman/memory_store/retrieval/mod.rs @@ -0,0 +1,397 @@ +//! Unified retrieval facade over the memory_store backends. +//! +//! `memory_store` owns four distinct retrieval modalities, each implemented in +//! a different submodule today: +//! +//! 1. **tree-walk** — BFS over sealed summary nodes (delegates to the +//! existing drill_down logic in `memory::retrieval::drill_down`). +//! 2. **vector search** — embedding-similarity ranking over namespace docs +//! (delegates to `UnifiedMemory::query_namespace_hits`). +//! 3. **keyword search** — FTS5/keyword overlap, same hybrid entry point as +//! vector (the hybrid scorer already blends both signals). +//! 4. **param/tag search** — structured filters over chunk metadata + content +//! store tags (delegates to `chunks::store::list_chunks` and +//! `content::tags`). +//! +//! The facade is a thin aggregation layer: it does NOT reimplement any +//! scoring or storage logic. It exists so callers have a single import surface +//! (`memory_store::retrieval::RetrievalFacade`) instead of reaching into four +//! different submodules. +//! +//! Layering note: `tree_walk` calls `memory::retrieval::drill_down`, which is +//! a reverse dependency from `memory_store` up into `memory`. This is +//! intentional and bounded — `drill_down` is "tree walk over stored trees" and +//! conceptually belongs in `memory_store`, but moving it is out of scope for +//! the storage-extraction refactor. Revisit when drill_down's policy bits +//! (entity hits, source-vs-summary precedence) can be cleanly split from the +//! pure tree traversal. + +use anyhow::Result; +use std::sync::Arc; + +use crate::openhuman::config::Config; +use crate::openhuman::memory::retrieval::types::RetrievalHit; +use crate::openhuman::memory_store::chunks::store::list_chunks; +use crate::openhuman::memory_store::chunks::types::{Chunk, SourceKind}; +use crate::openhuman::memory_store::types::NamespaceMemoryHit; +use crate::openhuman::memory_store::UnifiedMemory; + +/// Optional filter set for `param_tag_search`. All `Some` fields are AND-ed +/// together; `None` fields are unconstrained. +#[derive(Debug, Default, Clone)] +pub struct ParamTagFilters { + pub source_kind: Option, + pub source_id: Option, + pub owner: Option, + /// Inclusive lower bound on chunk `timestamp_ms`. + pub since_ms: Option, + /// Inclusive upper bound on chunk `timestamp_ms`. + pub until_ms: Option, + /// If `Some`, post-filter to chunks whose `tags` contains every listed tag. + pub tags_all_of: Option>, + /// Max rows to return (default 100 when `None`). + pub limit: Option, +} + +/// Unified retrieval entry point. Construct with an `Arc` for +/// vector/keyword ops; tree-walk and param/tag ops only need `&Config`. +#[derive(Clone)] +pub struct RetrievalFacade { + unified: Arc, +} + +impl RetrievalFacade { + pub fn new(unified: Arc) -> Self { + Self { unified } + } + + /// BFS walk from `node_id` down to `max_depth`. When `query` is `Some`, + /// hits are reranked by cosine similarity to the query embedding. + /// + /// See `memory::retrieval::drill_down::drill_down` for the full contract. + pub async fn tree_walk( + &self, + config: &Config, + node_id: &str, + max_depth: u32, + query: Option<&str>, + limit: Option, + ) -> Result> { + crate::openhuman::memory::retrieval::drill_down::drill_down( + config, node_id, max_depth, query, limit, + ) + .await + } + + /// Hybrid vector + graph + freshness retrieval. Same underlying scorer as + /// `keyword_search`; the difference is purely semantic intent at the call + /// site (callers using this entry point are saying "I have an embeddable + /// query"). Returns the full ranked hit list. + pub async fn vector_search( + &self, + namespace: &str, + query: &str, + limit: u32, + ) -> Result, String> { + self.unified + .query_namespace_hits(namespace, query, limit) + .await + } + + /// Same hybrid scorer as `vector_search` — the underlying retrieval plan + /// blends keyword overlap and vector similarity in one pass. Exposed as a + /// separate method so callers that only want lexical matching have an + /// honest name; the result set is identical for any given query. + pub async fn keyword_search( + &self, + namespace: &str, + query: &str, + limit: u32, + ) -> Result, String> { + self.unified + .query_namespace_hits(namespace, query, limit) + .await + } + + /// Structured chunk search by source/owner/time/tag filters. Bypasses the + /// ranking pipeline entirely — results are timestamp-DESC ordered. Use + /// when the caller knows the exact subset of chunks it wants. + pub fn param_tag_search( + &self, + config: &Config, + filters: &ParamTagFilters, + ) -> Result> { + let query = crate::openhuman::memory_store::chunks::store::ListChunksQuery { + source_kind: filters.source_kind, + source_id: filters.source_id.clone(), + owner: filters.owner.clone(), + since_ms: filters.since_ms, + until_ms: filters.until_ms, + limit: filters.limit, + }; + let rows = list_chunks(config, &query)?; + let Some(required) = filters.tags_all_of.as_ref() else { + return Ok(rows); + }; + if required.is_empty() { + return Ok(rows); + } + Ok(rows + .into_iter() + .filter(|c| { + required + .iter() + .all(|t| c.metadata.tags.iter().any(|ct| ct == t)) + }) + .collect()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::embeddings::NoopEmbedding; + use crate::openhuman::memory_store::chunks::store::upsert_chunks; + use crate::openhuman::memory_store::chunks::types::{Chunk, Metadata}; + use chrono::{TimeZone, Utc}; + use tempfile::TempDir; + + fn test_config() -> (TempDir, Config) { + let tmp = TempDir::new().unwrap(); + let mut cfg = Config::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + (tmp, cfg) + } + + fn test_facade(tmp: &TempDir) -> RetrievalFacade { + let unified = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + RetrievalFacade::new(Arc::new(unified)) + } + + fn chunk( + id: &str, + source_kind: SourceKind, + source_id: &str, + owner: &str, + tags: &[&str], + ) -> Chunk { + chunk_at(id, source_kind, source_id, owner, tags, Utc::now()) + } + + fn chunk_at( + id: &str, + source_kind: SourceKind, + source_id: &str, + owner: &str, + tags: &[&str], + ts: chrono::DateTime, + ) -> Chunk { + Chunk { + id: id.into(), + content: format!("content for {id}"), + metadata: Metadata { + source_kind, + source_id: source_id.into(), + owner: owner.into(), + timestamp: ts, + time_range: (ts, ts), + tags: tags.iter().map(|s| (*s).to_string()).collect(), + source_ref: None, + }, + token_count: 3, + seq_in_source: 0, + created_at: ts, + partial_message: false, + } + } + + #[test] + fn param_tag_filters_default_to_no_constraints() { + let filters = ParamTagFilters::default(); + assert!(filters.source_kind.is_none()); + assert!(filters.source_id.is_none()); + assert!(filters.owner.is_none()); + assert!(filters.since_ms.is_none()); + assert!(filters.until_ms.is_none()); + assert!(filters.tags_all_of.is_none()); + assert!(filters.limit.is_none()); + } + + #[test] + fn param_tag_search_filters_by_tags_all_of() { + let (tmp, cfg) = test_config(); + let facade = test_facade(&tmp); + upsert_chunks( + &cfg, + &[ + chunk( + "c1", + SourceKind::Chat, + "slack:#eng", + "alice", + &["person:alice", "deploy"], + ), + chunk( + "c2", + SourceKind::Chat, + "slack:#eng", + "alice", + &["person:alice"], + ), + chunk( + "c3", + SourceKind::Email, + "gmail:thread-1", + "bob", + &["deploy"], + ), + ], + ) + .unwrap(); + + let filters = ParamTagFilters { + tags_all_of: Some(vec!["person:alice".into(), "deploy".into()]), + ..ParamTagFilters::default() + }; + let hits = facade.param_tag_search(&cfg, &filters).unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].id, "c1"); + } + + #[test] + fn param_tag_search_respects_source_kind_filter() { + let (tmp, cfg) = test_config(); + let facade = test_facade(&tmp); + upsert_chunks( + &cfg, + &[ + chunk("c1", SourceKind::Chat, "slack:#eng", "alice", &[]), + chunk("c2", SourceKind::Email, "gmail:thread-1", "alice", &[]), + ], + ) + .unwrap(); + + let filters = ParamTagFilters { + source_kind: Some(SourceKind::Email), + ..ParamTagFilters::default() + }; + let hits = facade.param_tag_search(&cfg, &filters).unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].id, "c2"); + } + + #[test] + fn param_tag_search_respects_source_id_owner_and_limit() { + let (tmp, cfg) = test_config(); + let facade = test_facade(&tmp); + upsert_chunks( + &cfg, + &[ + chunk("c1", SourceKind::Chat, "slack:#eng", "alice", &[]), + chunk("c2", SourceKind::Chat, "slack:#eng", "bob", &[]), + chunk("c3", SourceKind::Chat, "slack:#ops", "alice", &[]), + ], + ) + .unwrap(); + + let filters = ParamTagFilters { + source_id: Some("slack:#eng".into()), + owner: Some("alice".into()), + limit: Some(1), + ..ParamTagFilters::default() + }; + let hits = facade.param_tag_search(&cfg, &filters).unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].id, "c1"); + assert_eq!(hits[0].metadata.source_id, "slack:#eng"); + assert_eq!(hits[0].metadata.owner, "alice"); + } + + #[test] + fn param_tag_search_empty_required_tags_is_noop() { + let (tmp, cfg) = test_config(); + let facade = test_facade(&tmp); + upsert_chunks( + &cfg, + &[ + chunk("c1", SourceKind::Chat, "slack:#eng", "alice", &["deploy"]), + chunk( + "c2", + SourceKind::Email, + "gmail:thread-1", + "bob", + &["person:bob"], + ), + ], + ) + .unwrap(); + + let hits = facade + .param_tag_search( + &cfg, + &ParamTagFilters { + tags_all_of: Some(vec![]), + ..ParamTagFilters::default() + }, + ) + .unwrap(); + assert_eq!(hits.len(), 2); + } + + #[test] + fn param_tag_search_respects_since_and_until_bounds() { + let (tmp, cfg) = test_config(); + let facade = test_facade(&tmp); + let older = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(); + let newer = Utc.timestamp_millis_opt(1_700_100_000_000).unwrap(); + upsert_chunks( + &cfg, + &[ + chunk_at("c1", SourceKind::Chat, "slack:#eng", "alice", &[], older), + chunk_at("c2", SourceKind::Chat, "slack:#eng", "alice", &[], newer), + ], + ) + .unwrap(); + + let hits = facade + .param_tag_search( + &cfg, + &ParamTagFilters { + since_ms: Some(newer.timestamp_millis()), + until_ms: Some(newer.timestamp_millis()), + ..ParamTagFilters::default() + }, + ) + .unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].id, "c2"); + } + + #[test] + fn param_tag_search_returns_empty_when_required_tag_is_missing() { + let (tmp, cfg) = test_config(); + let facade = test_facade(&tmp); + upsert_chunks( + &cfg, + &[chunk( + "c1", + SourceKind::Chat, + "slack:#eng", + "alice", + &["deploy"], + )], + ) + .unwrap(); + + let hits = facade + .param_tag_search( + &cfg, + &ParamTagFilters { + tags_all_of: Some(vec!["person:bob".into()]), + ..ParamTagFilters::default() + }, + ) + .unwrap(); + assert!(hits.is_empty()); + } +} diff --git a/src/openhuman/memory/safety/mod.rs b/src/openhuman/memory_store/safety/mod.rs similarity index 99% rename from src/openhuman/memory/safety/mod.rs rename to src/openhuman/memory_store/safety/mod.rs index 44a061e0f..b94c1c718 100644 --- a/src/openhuman/memory/safety/mod.rs +++ b/src/openhuman/memory_store/safety/mod.rs @@ -13,7 +13,7 @@ use once_cell::sync::Lazy; use regex::Regex; use serde_json::Value; -use crate::openhuman::memory::store::types::NamespaceDocumentInput; +use crate::openhuman::memory_store::types::NamespaceDocumentInput; const REDACTED_SECRET: &str = "[REDACTED_SECRET]"; const REDACTED_PRIVATE_KEY: &str = "[REDACTED_PRIVATE_KEY]"; diff --git a/src/openhuman/memory/safety/pii.rs b/src/openhuman/memory_store/safety/pii.rs similarity index 100% rename from src/openhuman/memory/safety/pii.rs rename to src/openhuman/memory_store/safety/pii.rs diff --git a/src/openhuman/memory_store/tools/kinds.rs b/src/openhuman/memory_store/tools/kinds.rs new file mode 100644 index 000000000..06af2bfb9 --- /dev/null +++ b/src/openhuman/memory_store/tools/kinds.rs @@ -0,0 +1,60 @@ +//! `memory_store_kinds` — introspection. Enumerate every supported +//! [`MemoryKind`] so an agent can plan a fan-out without hard-coding. + +use async_trait::async_trait; +use serde_json::{json, Value}; + +use crate::openhuman::memory_store::MemoryKind; +use crate::openhuman::tools::traits::{Tool, ToolResult}; + +pub struct MemoryStoreKindsTool; + +#[async_trait] +impl Tool for MemoryStoreKindsTool { + fn name(&self) -> &str { + "memory_store_kinds" + } + + fn description(&self) -> &str { + "Return the catalog of memory_store storage kinds (content, chunk, \ + tree, vector, document, kv, graph, contact). No arguments. Use \ + when planning a multi-kind retrieval fan-out." + } + + fn parameters_schema(&self) -> serde_json::Value { + json!({ "type": "object", "properties": {} }) + } + + async fn execute(&self, _args: Value) -> anyhow::Result { + log::debug!("[tool][memory_store] kinds start"); + let kinds: Vec<&'static str> = MemoryKind::ALL.iter().map(|k| k.as_str()).collect(); + let json = serde_json::to_string(&json!({ "kinds": kinds }))?; + log::debug!( + "[tool][memory_store] kinds success count={}", + MemoryKind::ALL.len() + ); + Ok(ToolResult::success(json)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parameters_schema_is_empty_object() { + let tool = MemoryStoreKindsTool; + let schema = tool.parameters_schema(); + assert_eq!(schema["type"], "object"); + assert_eq!(schema["properties"], json!({})); + } + + #[tokio::test] + async fn execute_returns_all_memory_kinds() { + let tool = MemoryStoreKindsTool; + let result = tool.execute(Value::Null).await.unwrap(); + let parsed: serde_json::Value = serde_json::from_str(&result.output()).unwrap(); + let expected: Vec<&str> = MemoryKind::ALL.iter().map(|k| k.as_str()).collect(); + assert_eq!(parsed["kinds"], json!(expected)); + } +} diff --git a/src/openhuman/memory_store/tools/mod.rs b/src/openhuman/memory_store/tools/mod.rs new file mode 100644 index 000000000..faadd4f9b --- /dev/null +++ b/src/openhuman/memory_store/tools/mod.rs @@ -0,0 +1,36 @@ +//! Raw search/retrieve tools surfaced to the agent harness. +//! +//! These tools expose the storage layer directly — no policy, no scoring +//! beyond what the underlying backend already applies. They exist so an agent +//! can drop one layer below the curated `memory_tree_*` tools when it needs +//! to inspect or operate on raw memory_store rows. +//! +//! Three tools, one per major access pattern: +//! - [`MemoryStoreRawSearchTool`] — hybrid (vector+keyword) namespace query. +//! - [`MemoryStoreRawChunksTool`] — structured chunk filter by source/owner/ +//! time/tags. +//! - [`MemoryStoreKindsTool`] — introspection: enumerate every +//! [`MemoryKind`] the store supports. +//! +//! All three are async, return JSON, and follow the project Tool trait. + +mod kinds; +mod raw_chunks; +mod raw_search; + +pub use kinds::MemoryStoreKindsTool; +pub use raw_chunks::MemoryStoreRawChunksTool; +pub use raw_search::MemoryStoreRawSearchTool; + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::tools::traits::Tool; + + #[test] + fn exports_memory_store_tools_with_stable_names() { + assert_eq!(MemoryStoreKindsTool.name(), "memory_store_kinds"); + assert_eq!(MemoryStoreRawChunksTool.name(), "memory_store_raw_chunks"); + assert_eq!(MemoryStoreRawSearchTool.name(), "memory_store_raw_search"); + } +} diff --git a/src/openhuman/memory_store/tools/raw_chunks.rs b/src/openhuman/memory_store/tools/raw_chunks.rs new file mode 100644 index 000000000..5601a146b --- /dev/null +++ b/src/openhuman/memory_store/tools/raw_chunks.rs @@ -0,0 +1,206 @@ +//! `memory_store_raw_chunks` — structured chunk filter. +//! +//! Bypasses ranking entirely. Returns chunks (timestamp DESC) matching the +//! supplied source/owner/time/tag filters. Use when the agent knows the +//! exact subset of memory it wants to inspect. + +use async_trait::async_trait; +use serde::Deserialize; +use serde_json::json; + +use crate::openhuman::config::rpc as config_rpc; +use crate::openhuman::memory_store::chunks::store::{list_chunks, ListChunksQuery}; +use crate::openhuman::memory_store::chunks::types::SourceKind; +use crate::openhuman::tools::traits::{Tool, ToolResult}; + +pub struct MemoryStoreRawChunksTool; + +#[derive(Debug, Deserialize)] +struct Args { + #[serde(default)] + source_kind: Option, + #[serde(default)] + source_id: Option, + #[serde(default)] + owner: Option, + #[serde(default)] + since_ms: Option, + #[serde(default)] + until_ms: Option, + #[serde(default)] + tags_all_of: Option>, + #[serde(default)] + limit: Option, +} + +#[async_trait] +impl Tool for MemoryStoreRawChunksTool { + fn name(&self) -> &str { + "memory_store_raw_chunks" + } + + fn description(&self) -> &str { + "List raw memory_store chunks (timestamp DESC) matching structured \ + filters: source kind, source id, owner, time range, required tags. \ + No scoring or rerank — use for exact-subset inspection, not search. \ + Returns full Chunk rows with metadata and content." + } + + fn parameters_schema(&self) -> serde_json::Value { + json!({ + "type": "object", + "properties": { + "source_kind": { "type": "string", "enum": ["chat", "email", "document"] }, + "source_id": { "type": "string", "description": "Exact source id." }, + "owner": { "type": "string", "description": "Owner / account filter." }, + "since_ms": { "type": "integer", "description": "Inclusive lower bound on timestamp_ms." }, + "until_ms": { "type": "integer", "description": "Inclusive upper bound on timestamp_ms." }, + "tags_all_of": { + "type": "array", + "items": { "type": "string" }, + "description": "Post-filter: chunk.metadata.tags must contain every tag listed." + }, + "limit": { "type": "integer", "minimum": 1, "maximum": 1000, "description": "Default 100." } + } + }) + } + + async fn execute(&self, args: serde_json::Value) -> anyhow::Result { + let parsed: Args = serde_json::from_value(args) + .map_err(|e| anyhow::anyhow!("invalid arguments for memory_store_raw_chunks: {e}"))?; + log::debug!( + "[tool][memory_store] raw_chunks source_kind={:?} owner={:?} tags={:?} limit={:?}", + parsed.source_kind, + parsed.owner, + parsed.tags_all_of, + parsed.limit + ); + let cfg = config_rpc::load_config_with_timeout() + .await + .map_err(|e| anyhow::anyhow!("memory_store_raw_chunks: load config failed: {e}"))?; + let source_kind = match parsed.source_kind.as_deref() { + Some(s) => Some( + SourceKind::parse(s) + .map_err(|e| anyhow::anyhow!("memory_store_raw_chunks: {e}"))?, + ), + None => None, + }; + if let Some(limit) = parsed.limit { + if !(1..=1000).contains(&limit) { + return Err(anyhow::anyhow!( + "memory_store_raw_chunks: limit must be between 1 and 1000" + )); + } + } + let query = ListChunksQuery { + source_kind, + source_id: parsed.source_id, + owner: parsed.owner, + since_ms: parsed.since_ms, + until_ms: parsed.until_ms, + limit: parsed.limit, + }; + let mut rows = list_chunks(&cfg, &query)?; + if let Some(required) = parsed.tags_all_of.as_ref() { + if !required.is_empty() { + rows.retain(|c| { + required + .iter() + .all(|t| c.metadata.tags.iter().any(|ct| ct == t)) + }); + } + } + log::debug!( + "[tool][memory_store] raw_chunks returning rows={}", + rows.len() + ); + let json = serde_json::to_string(&rows)?; + Ok(ToolResult::success(json)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::tools::traits::Tool; + use serde_json::json; + + #[test] + fn args_deserialize_optional_filters() { + let args: Args = serde_json::from_value(json!({ + "source_kind": "chat", + "source_id": "slack:#eng", + "owner": "alice", + "since_ms": 10, + "until_ms": 20, + "tags_all_of": ["person:alice"], + "limit": 25 + })) + .unwrap(); + + assert_eq!(args.source_kind.as_deref(), Some("chat")); + assert_eq!(args.source_id.as_deref(), Some("slack:#eng")); + assert_eq!(args.owner.as_deref(), Some("alice")); + assert_eq!(args.since_ms, Some(10)); + assert_eq!(args.until_ms, Some(20)); + assert_eq!(args.tags_all_of, Some(vec!["person:alice".to_string()])); + assert_eq!(args.limit, Some(25)); + } + + #[test] + fn parameters_schema_exposes_supported_source_kinds() { + let tool = MemoryStoreRawChunksTool; + let schema = tool.parameters_schema(); + assert_eq!(schema["type"], "object"); + assert_eq!( + schema["properties"]["source_kind"]["enum"], + json!(["chat", "email", "document"]) + ); + assert_eq!(schema["properties"]["limit"]["maximum"], 1000); + } + + #[tokio::test] + async fn execute_rejects_invalid_source_kind() { + let tool = MemoryStoreRawChunksTool; + let err = tool + .execute(json!({ + "source_kind": "not-real" + })) + .await + .expect_err("invalid source kind should fail"); + assert!(err.to_string().contains("memory_store_raw_chunks:")); + } + + #[tokio::test] + async fn execute_rejects_wrong_type_for_limit() { + let tool = MemoryStoreRawChunksTool; + let err = tool + .execute(json!({ + "limit": "ten" + })) + .await + .expect_err("wrong limit type should fail"); + assert!(err + .to_string() + .contains("invalid arguments for memory_store_raw_chunks")); + } + + #[tokio::test] + async fn execute_success_path_returns_json_array() { + let tool = MemoryStoreRawChunksTool; + let result = tool + .execute(json!({ + "source_kind": "document", + "limit": 2 + })) + .await + .expect("valid raw_chunks request should succeed"); + assert!(!result.is_error); + let parsed: serde_json::Value = + serde_json::from_str(&result.text()).expect("tool result should be json"); + assert!( + parsed.is_array(), + "raw_chunks should serialize a JSON array" + ); + } +} diff --git a/src/openhuman/memory_store/tools/raw_search.rs b/src/openhuman/memory_store/tools/raw_search.rs new file mode 100644 index 000000000..e75bcbc9b --- /dev/null +++ b/src/openhuman/memory_store/tools/raw_search.rs @@ -0,0 +1,177 @@ +//! `memory_store_raw_search` — free-text search over the entity index. +//! +//! Thin wrapper around `memory::retrieval::search_entities`. Returns canonical +//! entity ids ranked by mention count. This is the rawest of the raw search +//! paths: no narrative, no scoring beyond aggregate occurrence, no rerank. +//! Use it when an agent needs to discover what entities exist in the store +//! before drilling into trees. + +use async_trait::async_trait; +use serde::Deserialize; +use serde_json::json; + +use crate::openhuman::config::rpc as config_rpc; +use crate::openhuman::memory::retrieval::search::search_entities; +use crate::openhuman::memory::score::extract::EntityKind; +use crate::openhuman::tools::traits::{Tool, ToolResult}; + +pub struct MemoryStoreRawSearchTool; + +#[derive(Debug, Deserialize)] +struct Args { + query: String, + #[serde(default)] + kinds: Option>, + #[serde(default = "default_limit")] + limit: usize, +} + +fn default_limit() -> usize { + 5 +} + +#[async_trait] +impl Tool for MemoryStoreRawSearchTool { + fn name(&self) -> &str { + "memory_store_raw_search" + } + + fn description(&self) -> &str { + "Free-text LIKE search over the canonical entity index. Returns \ + entity ids ranked by total mention count across every tree. Use to \ + discover what entities (people, channels, threads) exist in the \ + memory store before drilling into a tree with the memory_tree_* \ + tools. Pass `kinds` to narrow the result set (e.g. only people)." + } + + fn parameters_schema(&self) -> serde_json::Value { + json!({ + "type": "object", + "required": ["query"], + "properties": { + "query": { + "type": "string", + "description": "Substring matched against canonical entity id and surface form (case-insensitive)." + }, + "kinds": { + "type": "array", + "items": { "type": "string" }, + "description": "Optional entity kind filter (e.g. [\"person\", \"channel\"]). Empty/absent = all kinds." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "description": "Max matches to return (default 5, clamped 100)." + } + } + }) + } + + async fn execute(&self, args: serde_json::Value) -> anyhow::Result { + let parsed: Args = serde_json::from_value(args) + .map_err(|e| anyhow::anyhow!("invalid arguments for memory_store_raw_search: {e}"))?; + log::debug!( + "[tool][memory_store] raw_search q_len={} kinds={:?} limit={}", + parsed.query.len(), + parsed.kinds, + parsed.limit + ); + let cfg = config_rpc::load_config_with_timeout() + .await + .map_err(|e| anyhow::anyhow!("memory_store_raw_search: load config failed: {e}"))?; + let kinds = match parsed.kinds { + Some(ks) if !ks.is_empty() => { + let mut out = Vec::with_capacity(ks.len()); + for k in ks { + out.push( + EntityKind::parse(&k) + .map_err(|e| anyhow::anyhow!("memory_store_raw_search: {e}"))?, + ); + } + Some(out) + } + _ => None, + }; + let hits = search_entities(&cfg, &parsed.query, kinds, parsed.limit).await?; + log::debug!( + "[tool][memory_store] raw_search returning hits={}", + hits.len() + ); + let json = serde_json::to_string(&hits)?; + Ok(ToolResult::success(json)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::tools::traits::Tool; + use serde_json::json; + + #[test] + fn default_limit_is_five() { + assert_eq!(default_limit(), 5); + } + + #[test] + fn args_deserialize_with_default_limit() { + let args: Args = serde_json::from_value(json!({ "query": "alice" })).unwrap(); + assert_eq!(args.query, "alice"); + assert_eq!(args.limit, 5); + assert!(args.kinds.is_none()); + } + + #[test] + fn parameters_schema_describes_required_query() { + let tool = MemoryStoreRawSearchTool; + let schema = tool.parameters_schema(); + assert_eq!(schema["type"], "object"); + assert_eq!(schema["required"], json!(["query"])); + assert_eq!(schema["properties"]["limit"]["maximum"], 100); + } + + #[tokio::test] + async fn execute_rejects_missing_query() { + let tool = MemoryStoreRawSearchTool; + let err = tool + .execute(json!({})) + .await + .expect_err("missing query should fail"); + assert!(err + .to_string() + .contains("invalid arguments for memory_store_raw_search")); + } + + #[tokio::test] + async fn execute_rejects_invalid_kind() { + let tool = MemoryStoreRawSearchTool; + let err = tool + .execute(json!({ + "query": "alice", + "kinds": ["not-a-kind"] + })) + .await + .expect_err("invalid kind should fail"); + assert!(err.to_string().contains("memory_store_raw_search:")); + } + + #[tokio::test] + async fn execute_success_path_returns_json_array() { + let tool = MemoryStoreRawSearchTool; + let result = tool + .execute(json!({ + "query": "alice", + "limit": 3 + })) + .await + .expect("valid raw_search request should succeed"); + assert!(!result.is_error); + let parsed: serde_json::Value = + serde_json::from_str(&result.text()).expect("tool result should be json"); + assert!( + parsed.is_array(), + "raw_search should serialize a JSON array" + ); + } +} diff --git a/src/openhuman/memory_store/traits.rs b/src/openhuman/memory_store/traits.rs new file mode 100644 index 000000000..395d7edc9 --- /dev/null +++ b/src/openhuman/memory_store/traits.rs @@ -0,0 +1,308 @@ +//! Storage compatibility traits. +//! +//! Every stored memory kind must answer two questions: +//! +//! 1. **Can it be embedded into a vector?** — yes, via [`VectorEmbeddable`]. +//! The trait provides the canonical embeddable string for the object so a +//! single embedding pipeline can index any kind uniformly. +//! 2. **Can it be represented as an Obsidian-compatible markdown file?** — +//! yes, via [`ObsidianRepresentable`]. The trait yields a relative vault +//! path and a fully-formed markdown body (YAML front-matter + content) +//! that can be written into the content store and opened by Obsidian +//! without further processing. +//! +//! Together these two traits are the contract that makes "everything in +//! memory_store is vector and obsidian compatible" a checkable property +//! rather than a slogan — the compiler enforces it for every new storage +//! kind that gets added. + +use std::path::PathBuf; + +use crate::openhuman::memory_store::chunks::types::Chunk; +use crate::openhuman::memory_store::contacts::Person; +use crate::openhuman::memory_store::kinds::MemoryKind; +use crate::openhuman::memory_store::trees::{SummaryNode, Tree}; + +/// A rendered Obsidian markdown file: where it lives in the vault and what +/// bytes to write. Vault path is relative to the content-store root. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ObsidianFile { + pub relative_path: PathBuf, + pub markdown: String, +} + +/// Objects that can produce a canonical string for embedding into the +/// vector store. The returned text should be deterministic and stable across +/// calls so re-embedding produces consistent vectors. +pub trait VectorEmbeddable { + /// The MemoryKind this value belongs to. Used by the embedding pipeline + /// to route vectors into per-kind namespaces. + fn memory_kind(&self) -> MemoryKind; + + /// Canonical UTF-8 text fed to the embedding model. Strip front-matter, + /// markdown formatting noise, and anything not semantically meaningful. + fn embeddable_text(&self) -> String; +} + +/// Objects that can be rendered as an Obsidian-compatible markdown file. +/// The file should round-trip through the content store unchanged so vault +/// edits stay idempotent. +pub trait ObsidianRepresentable { + fn to_obsidian(&self) -> ObsidianFile; +} + +// ---- impls: Chunk ---------------------------------------------------------- + +impl VectorEmbeddable for Chunk { + fn memory_kind(&self) -> MemoryKind { + MemoryKind::Chunk + } + + fn embeddable_text(&self) -> String { + self.content.clone() + } +} + +impl ObsidianRepresentable for Chunk { + fn to_obsidian(&self) -> ObsidianFile { + let tags_yaml = if self.metadata.tags.is_empty() { + String::new() + } else { + let lines: Vec = self + .metadata + .tags + .iter() + .map(|t| format!(" - {}", t)) + .collect(); + format!("tags:\n{}\n", lines.join("\n")) + }; + let markdown = format!( + "---\nid: {}\nsource_kind: {}\nsource_id: {}\nseq: {}\n{}---\n\n{}\n", + self.id, + self.metadata.source_kind.as_str(), + self.metadata.source_id, + self.seq_in_source, + tags_yaml, + self.content + ); + ObsidianFile { + relative_path: PathBuf::from("chunks").join(format!("{}.md", self.id)), + markdown, + } + } +} + +// ---- impls: Tree + SummaryNode -------------------------------------------- + +impl VectorEmbeddable for SummaryNode { + fn memory_kind(&self) -> MemoryKind { + MemoryKind::Tree + } + + fn embeddable_text(&self) -> String { + self.content.clone() + } +} + +impl ObsidianRepresentable for SummaryNode { + fn to_obsidian(&self) -> ObsidianFile { + let markdown = format!( + "---\nid: {}\ntree_id: {}\nlevel: {}\n---\n\n{}\n", + self.id, self.tree_id, self.level, self.content + ); + ObsidianFile { + relative_path: PathBuf::from("summaries").join(format!("{}.md", self.id)), + markdown, + } + } +} + +impl ObsidianRepresentable for Tree { + fn to_obsidian(&self) -> ObsidianFile { + let markdown = format!( + "---\nid: {}\nkind: {:?}\nstatus: {:?}\n---\n\nTree {} ({:?})\n", + self.id, self.kind, self.status, self.id, self.kind + ); + ObsidianFile { + relative_path: PathBuf::from("trees").join(format!("{}.md", self.id)), + markdown, + } + } +} + +// ---- impls: Contact (Person) ---------------------------------------------- + +impl VectorEmbeddable for Person { + fn memory_kind(&self) -> MemoryKind { + MemoryKind::Contact + } + + fn embeddable_text(&self) -> String { + // Embed the display name plus primary email — both carry useful + // disambiguation signal. Handles are routing keys, not semantic + // content, and intentionally excluded. + let mut parts: Vec = Vec::new(); + if let Some(name) = self.display_name.as_deref() { + parts.push(name.to_string()); + } + if let Some(email) = self.primary_email.as_deref() { + parts.push(email.to_string()); + } + parts.join("\n") + } +} + +impl ObsidianRepresentable for Person { + fn to_obsidian(&self) -> ObsidianFile { + let display = self.display_name.as_deref().unwrap_or("Unknown"); + let email = self.primary_email.as_deref().unwrap_or(""); + let markdown = format!( + "---\nperson_id: {}\n---\n\n# {}\n\nEmail: {}\n", + self.id, display, email + ); + ObsidianFile { + relative_path: PathBuf::from("contacts").join(format!("{}.md", self.id)), + markdown, + } + } +} + +// Documents are no longer a first-class MemoryKind — the md backend +// (`content/`) is the canonical persistence for any document body. Anything +// that historically used `StoredMemoryDocument` should land its body as a +// raw md file and reference it via path. + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::memory_store::chunks::types::{Metadata, SourceKind}; + use chrono::Utc; + + fn sample_chunk() -> Chunk { + let ts = Utc::now(); + Chunk { + id: "chunk-1".into(), + content: "hello world".into(), + metadata: Metadata { + source_kind: SourceKind::Chat, + source_id: "slack:#eng".into(), + timestamp: ts, + time_range: (ts, ts), + owner: "alice".into(), + source_ref: None, + tags: vec!["person:alice".into()], + }, + seq_in_source: 7, + token_count: 2, + created_at: ts, + partial_message: false, + } + } + + #[test] + fn chunk_traits_render_expected_kind_and_obsidian_path() { + let chunk = sample_chunk(); + assert_eq!(chunk.memory_kind(), MemoryKind::Chunk); + assert_eq!(chunk.embeddable_text(), "hello world"); + + let obsidian = chunk.to_obsidian(); + assert_eq!(obsidian.relative_path, PathBuf::from("chunks/chunk-1.md")); + assert!(obsidian.markdown.contains("source_kind: chat")); + assert!(obsidian.markdown.contains("source_id: slack:#eng")); + assert!(obsidian.markdown.contains("hello world")); + } + + #[test] + fn summary_node_traits_render_expected_kind_and_path() { + let node = SummaryNode { + id: "summary-1".into(), + tree_id: "tree-1".into(), + tree_kind: crate::openhuman::memory_store::trees::TreeKind::Source, + level: 1, + parent_id: None, + child_ids: vec!["chunk-1".into()], + content: "summary body".into(), + token_count: 3, + entities: vec![], + topics: vec![], + time_range_start: Utc::now(), + time_range_end: Utc::now(), + score: 0.5, + sealed_at: Utc::now(), + deleted: false, + embedding: None, + }; + assert_eq!(node.memory_kind(), MemoryKind::Tree); + assert_eq!(node.embeddable_text(), "summary body"); + let obsidian = node.to_obsidian(); + assert_eq!( + obsidian.relative_path, + PathBuf::from("summaries/summary-1.md") + ); + assert!(obsidian.markdown.contains("tree_id: tree-1")); + assert!(obsidian.markdown.contains("summary body")); + } + + #[test] + fn tree_traits_render_obsidian_metadata() { + let tree = Tree { + id: "tree-1".into(), + kind: crate::openhuman::memory_store::trees::TreeKind::Topic, + scope: "topic:phoenix".into(), + root_id: Some("summary-root".into()), + max_level: 2, + status: crate::openhuman::memory_store::trees::TreeStatus::Active, + created_at: Utc::now(), + last_sealed_at: None, + }; + let obsidian = tree.to_obsidian(); + assert_eq!(obsidian.relative_path, PathBuf::from("trees/tree-1.md")); + assert!(obsidian.markdown.contains("id: tree-1")); + assert!(obsidian.markdown.contains("Tree tree-1")); + assert!(obsidian.markdown.contains("Topic")); + } + + #[test] + fn person_traits_render_name_and_email_when_present() { + let now = Utc::now(); + let person = Person { + id: crate::openhuman::people::types::PersonId::new(), + display_name: Some("Alice Example".into()), + primary_email: Some("alice@example.com".into()), + primary_phone: Some("+1 555 0100".into()), + handles: vec![ + crate::openhuman::people::types::Handle::DisplayName("Alice Example".into()), + crate::openhuman::people::types::Handle::Email("alice@example.com".into()), + ], + created_at: now, + updated_at: now, + }; + assert_eq!(person.memory_kind(), MemoryKind::Contact); + assert_eq!(person.embeddable_text(), "Alice Example\nalice@example.com"); + let obsidian = person.to_obsidian(); + assert_eq!( + obsidian.relative_path, + PathBuf::from("contacts").join(format!("{}.md", person.id)) + ); + assert!(obsidian.markdown.contains("# Alice Example")); + assert!(obsidian.markdown.contains("Email: alice@example.com")); + } + + #[test] + fn person_traits_fall_back_when_fields_are_missing() { + let now = Utc::now(); + let person = Person { + id: crate::openhuman::people::types::PersonId::new(), + display_name: None, + primary_email: None, + primary_phone: None, + handles: vec![], + created_at: now, + updated_at: now, + }; + assert_eq!(person.embeddable_text(), ""); + let obsidian = person.to_obsidian(); + assert!(obsidian.markdown.contains("# Unknown")); + assert!(obsidian.markdown.contains("Email: ")); + } +} diff --git a/src/openhuman/memory_tree/tree_topic/store.rs b/src/openhuman/memory_store/trees/hotness.rs similarity index 63% rename from src/openhuman/memory_tree/tree_topic/store.rs rename to src/openhuman/memory_store/trees/hotness.rs index 1eab93748..d3d2cb2a3 100644 --- a/src/openhuman/memory_tree/tree_topic/store.rs +++ b/src/openhuman/memory_store/trees/hotness.rs @@ -1,24 +1,18 @@ -//! SQLite persistence for topic-tree-specific state (#709 Phase 3c). +//! Entity-hotness counter persistence (`mem_tree_entity_hotness` table). //! -//! The only new table owned here is `mem_tree_entity_hotness` — the -//! per-entity counter block driving lazy materialisation. Tree rows and -//! summary nodes are reused from [`super::super::tree_source::store`] via -//! the shared `mem_tree_trees` / `mem_tree_summaries` / `mem_tree_buffers` -//! tables, which already carry a `kind` column that discriminates -//! `source` from `topic`. No schema additions for those tables in Phase -//! 3c — only the new hotness table. -//! -//! Schema for `mem_tree_entity_hotness` is declared in -//! [`super::super::store::SCHEMA`] (the sibling Phase 1 store file) so -//! migrations all run through the same `with_connection` entry point. +//! Previously at `memory_store::trees_topic::store`. Folded into `trees/` +//! because hotness is the only state that differentiates topic trees from +//! source/global trees, and even then it's a side-table — not a tree row. +//! Keeping it next to tree storage makes "what does memory_store know about +//! trees?" a single-directory answer. use anyhow::{Context, Result}; use chrono::Utc; use rusqlite::{params, OptionalExtension}; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::store::with_connection; -use crate::openhuman::memory_tree::tree_topic::types::HotnessCounters; +use crate::openhuman::memory_store::chunks::store::with_connection; +use crate::openhuman::memory_store::trees::types::HotnessCounters; /// Fetch the hotness row for `entity_id`, or `None` if the entity has /// never been seen. Callers usually want [`get_or_fresh`] instead. @@ -92,8 +86,6 @@ pub fn upsert(config: &Config, counters: &HotnessCounters) -> Result<()> { } /// Count `(node_id) → DISTINCT tree_id` in the entity index for `entity_id`. -/// Used by the curator to refresh `distinct_sources` during the periodic -/// hotness recompute without rescanning every chunk. pub fn distinct_sources_for(config: &Config, entity_id: &str) -> Result { with_connection(config, |conn| { let n: i64 = conn @@ -161,7 +153,6 @@ mod tests { assert_eq!(c.mention_count_30d, 0); assert_eq!(c.distinct_sources, 0); assert!(c.last_hotness.is_none()); - // Not persisted — still zero rows in the table. assert_eq!(count(&cfg).unwrap(), 0); } @@ -184,62 +175,4 @@ mod tests { assert_eq!(got, c); assert_eq!(count(&cfg).unwrap(), 1); } - - #[test] - fn upsert_is_idempotent_and_updates_fields() { - let (_tmp, cfg) = test_config(); - let mut c = HotnessCounters::fresh("email:alice@example.com", 0); - c.mention_count_30d = 1; - upsert(&cfg, &c).unwrap(); - c.mention_count_30d = 99; - c.last_updated_ms = 500; - upsert(&cfg, &c).unwrap(); - assert_eq!(count(&cfg).unwrap(), 1); - let got = get(&cfg, "email:alice@example.com").unwrap().unwrap(); - assert_eq!(got.mention_count_30d, 99); - assert_eq!(got.last_updated_ms, 500); - } - - #[test] - fn distinct_sources_counts_trees() { - use crate::openhuman::memory_tree::score::extract::EntityKind; - use crate::openhuman::memory_tree::score::resolver::CanonicalEntity; - use crate::openhuman::memory_tree::score::store::index_entity; - let (_tmp, cfg) = test_config(); - let e = CanonicalEntity { - canonical_id: "email:alice@example.com".into(), - kind: EntityKind::Email, - surface: "alice@example.com".into(), - span_start: 0, - span_end: 17, - score: 1.0, - }; - index_entity(&cfg, &e, "chunk-1", "leaf", 1000, Some("source:slack")).unwrap(); - index_entity(&cfg, &e, "chunk-2", "leaf", 2000, Some("source:gmail")).unwrap(); - index_entity(&cfg, &e, "chunk-3", "leaf", 3000, Some("source:slack")).unwrap(); - // 3 rows but only 2 distinct tree_ids. - let n = distinct_sources_for(&cfg, "email:alice@example.com").unwrap(); - assert_eq!(n, 2); - } - - #[test] - fn distinct_sources_ignores_null_tree_id() { - use crate::openhuman::memory_tree::score::extract::EntityKind; - use crate::openhuman::memory_tree::score::resolver::CanonicalEntity; - use crate::openhuman::memory_tree::score::store::index_entity; - let (_tmp, cfg) = test_config(); - let e = CanonicalEntity { - canonical_id: "email:alice@example.com".into(), - kind: EntityKind::Email, - surface: "alice@example.com".into(), - span_start: 0, - span_end: 17, - score: 1.0, - }; - // tree_id = None — should not count toward distinct_sources. - index_entity(&cfg, &e, "chunk-1", "leaf", 1000, None).unwrap(); - index_entity(&cfg, &e, "chunk-2", "leaf", 2000, Some("source:slack")).unwrap(); - let n = distinct_sources_for(&cfg, "email:alice@example.com").unwrap(); - assert_eq!(n, 1); - } } diff --git a/src/openhuman/memory_store/trees/mod.rs b/src/openhuman/memory_store/trees/mod.rs new file mode 100644 index 000000000..766a9c363 --- /dev/null +++ b/src/openhuman/memory_store/trees/mod.rs @@ -0,0 +1,42 @@ +//! Tree persistence — shared across Source, Global, and Topic kinds. +//! +//! All three flavors live in `mem_tree_trees` keyed by [`TreeKind`]. This +//! module hosts: +//! - `store` — generic CRUD over the trees + summaries + buffers tables. +//! - `types` — Tree, SummaryNode, TreeKind, TreeStatus, Buffer, and the +//! topic-hotness types ([`HotnessCounters`], thresholds). +//! - `registry` — kind-parameterized get-or-create / list / archive helpers. +//! - `hotness` — entity-hotness side-table that gates topic-tree spawn. +//! +//! Tree _logic_ (bucket_seal, flush, generic registry, sources/global/topic +//! policy) stays in `memory_tree`. + +pub mod hotness; +pub mod registry; +pub mod store; +pub mod types; + +pub use registry::{ + archive_topic_tree, archive_tree, force_create_topic_tree, get_or_create_global_tree, + get_or_create_topic_tree, list_topic_trees, list_trees_by_kind, +}; +pub use store::{get_summary_embedding, set_summary_embedding}; +pub use types::{ + Buffer, EntityIndexStats, HotnessCounters, SummaryNode, Tree, TreeKind, TreeStatus, + INPUT_TOKEN_BUDGET, OUTPUT_TOKEN_BUDGET, SUMMARY_FANOUT, TOPIC_ARCHIVE_THRESHOLD, + TOPIC_CREATION_THRESHOLD, TOPIC_RECHECK_EVERY, +}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tree_module_reexports_expected_constants() { + assert_eq!(INPUT_TOKEN_BUDGET, 50_000); + assert_eq!(OUTPUT_TOKEN_BUDGET, 5_000); + assert_eq!(SUMMARY_FANOUT, 10); + assert!(TOPIC_CREATION_THRESHOLD > TOPIC_ARCHIVE_THRESHOLD); + assert!(TOPIC_RECHECK_EVERY > 0); + } +} diff --git a/src/openhuman/memory_store/trees/registry.rs b/src/openhuman/memory_store/trees/registry.rs new file mode 100644 index 000000000..ce104d5a9 --- /dev/null +++ b/src/openhuman/memory_store/trees/registry.rs @@ -0,0 +1,214 @@ +//! Kind-parameterized tree registry helpers. +//! +//! All three tree flavors (Source, Global, Topic) live in the same +//! `mem_tree_trees` table keyed by [`TreeKind`]. The generic get-or-create +//! dance with UNIQUE-race recovery lives in +//! [`memory_tree::tree::registry::get_or_create_tree`] (logic file — stays +//! in memory_tree). This module hosts the thin per-kind convenience wrappers +//! (formerly spread across `trees_global::registry` and +//! `trees_topic::registry`) so callers have one place to land. + +use anyhow::{Context, Result}; +use chrono::Utc; +use rusqlite::params; + +use crate::openhuman::config::Config; +use crate::openhuman::memory_store::chunks::store::with_connection; +use crate::openhuman::memory_store::trees::types::{Tree, TreeKind}; +use crate::openhuman::memory_tree::global::GLOBAL_SCOPE; +use crate::openhuman::memory_tree::tree::registry::get_or_create_tree; + +/// Return the workspace's singleton global tree, creating it lazily on first +/// call. Safe to call on every ingest; subsequent calls short-circuit to the +/// existing row. +pub fn get_or_create_global_tree(config: &Config) -> Result { + log::debug!("[trees::registry] get_or_create_global_tree"); + get_or_create_tree(config, TreeKind::Global, GLOBAL_SCOPE) +} + +/// Look up the topic tree for `entity_id`, or create a new one. +/// +/// Callers should NOT use this directly to materialise topic trees eagerly — +/// go through `memory_tree::topic::curator::maybe_spawn_topic_tree` so +/// creation is gated on hotness. Admin / forced-materialisation flows can +/// call this directly (or its alias [`force_create_topic_tree`]). +pub fn get_or_create_topic_tree(config: &Config, entity_id: &str) -> Result { + let entity_kind = entity_id + .split_once(':') + .map(|(k, _)| k) + .unwrap_or("unknown"); + log::debug!( + "[trees::registry] get_or_create_topic_tree entity_kind={}", + entity_kind + ); + get_or_create_tree(config, TreeKind::Topic, entity_id) +} + +/// Semantic alias used by the admin "force materialise" path. +pub fn force_create_topic_tree(config: &Config, entity_id: &str) -> Result { + get_or_create_topic_tree(config, entity_id) +} + +/// List every tree of the given kind, ordered by creation time ascending. +pub fn list_trees_by_kind(config: &Config, kind: TreeKind) -> Result> { + with_connection(config, |conn| { + let mut stmt = conn.prepare( + "SELECT id, kind, scope, root_id, max_level, status, + created_at_ms, last_sealed_at_ms + FROM mem_tree_trees + WHERE kind = ?1 + ORDER BY created_at_ms ASC", + )?; + let rows = stmt + .query_map(params![kind.as_str()], row_to_tree_loose)? + .collect::>>() + .with_context(|| format!("failed to list trees of kind {}", kind.as_str()))?; + Ok(rows) + }) +} + +/// Topic-specific alias preserved for call-site readability. +pub fn list_topic_trees(config: &Config) -> Result> { + list_trees_by_kind(config, TreeKind::Topic) +} + +/// Flip a tree's status to `archived`. Idempotent. Existing rows remain +/// queryable; new leaves will not be routed to this tree until it is +/// manually unarchived (which is not currently a primitive). +pub fn archive_tree(config: &Config, tree_id: &str) -> Result<()> { + use crate::openhuman::memory_store::trees::types::TreeStatus; + with_connection(config, |conn| { + let n = conn + .execute( + "UPDATE mem_tree_trees + SET status = ?1 + WHERE id = ?2", + params![TreeStatus::Archived.as_str(), tree_id], + ) + .with_context(|| format!("failed to archive tree {}", tree_id))?; + log::debug!("[trees::registry] archive_tree id={} rows={}", tree_id, n); + Ok(()) + }) +} + +/// Topic-specific alias preserved for call-site readability. +pub fn archive_topic_tree(config: &Config, tree_id: &str) -> Result<()> { + archive_tree(config, tree_id) +} + +fn row_to_tree_loose(row: &rusqlite::Row<'_>) -> rusqlite::Result { + use crate::openhuman::memory_store::trees::types::TreeStatus; + use chrono::TimeZone; + let id: String = row.get(0)?; + let kind_s: String = row.get(1)?; + let scope: String = row.get(2)?; + let root_id: Option = row.get(3)?; + let max_level: i64 = row.get(4)?; + let status_s: String = row.get(5)?; + let created_ms: i64 = row.get(6)?; + let last_sealed_ms: Option = row.get(7)?; + let kind = TreeKind::parse(&kind_s).map_err(|e| { + rusqlite::Error::FromSqlConversionFailure(0, rusqlite::types::Type::Text, e.into()) + })?; + let status = TreeStatus::parse(&status_s).map_err(|e| { + rusqlite::Error::FromSqlConversionFailure(0, rusqlite::types::Type::Text, e.into()) + })?; + Ok(Tree { + id, + kind, + scope, + root_id, + max_level: max_level.max(0) as u32, + status, + created_at: Utc.timestamp_millis_opt(created_ms).unwrap(), + last_sealed_at: last_sealed_ms.and_then(|ms| Utc.timestamp_millis_opt(ms).single()), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::memory_store::trees::store::insert_tree; + use crate::openhuman::memory_store::trees::types::TreeStatus; + use tempfile::TempDir; + + fn test_config() -> (TempDir, Config) { + let tmp = TempDir::new().unwrap(); + let mut cfg = Config::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + (tmp, cfg) + } + + fn sample_tree(id: &str, kind: TreeKind, scope: &str) -> Tree { + Tree { + id: id.into(), + kind, + scope: scope.into(), + root_id: Some("root-1".into()), + max_level: 2, + status: TreeStatus::Active, + created_at: Utc::now(), + last_sealed_at: None, + } + } + + #[test] + fn list_trees_by_kind_returns_only_requested_kind() { + let (_tmp, cfg) = test_config(); + insert_tree( + &cfg, + &sample_tree("source-1", TreeKind::Source, "chat:slack:#eng"), + ) + .unwrap(); + insert_tree( + &cfg, + &sample_tree("topic-1", TreeKind::Topic, "person:alice"), + ) + .unwrap(); + insert_tree( + &cfg, + &sample_tree("source-2", TreeKind::Source, "chat:discord:#ops"), + ) + .unwrap(); + + let source_ids: Vec = list_trees_by_kind(&cfg, TreeKind::Source) + .unwrap() + .into_iter() + .map(|tree| tree.id) + .collect(); + assert_eq!( + source_ids, + vec!["source-1".to_string(), "source-2".to_string()] + ); + + let topic_ids: Vec = list_topic_trees(&cfg) + .unwrap() + .into_iter() + .map(|tree| tree.id) + .collect(); + assert_eq!(topic_ids, vec!["topic-1".to_string()]); + } + + #[test] + fn archive_tree_flips_status_to_archived() { + let (_tmp, cfg) = test_config(); + let tree = sample_tree("topic-1", TreeKind::Topic, "person:alice"); + insert_tree(&cfg, &tree).unwrap(); + + archive_tree(&cfg, "topic-1").unwrap(); + + let archived = list_topic_trees(&cfg).unwrap(); + assert_eq!(archived.len(), 1); + assert_eq!(archived[0].status, TreeStatus::Archived); + } + + #[test] + fn get_or_create_global_tree_is_idempotent() { + let (_tmp, cfg) = test_config(); + let first = get_or_create_global_tree(&cfg).unwrap(); + let second = get_or_create_global_tree(&cfg).unwrap(); + assert_eq!(first.id, second.id); + assert_eq!(first.kind, TreeKind::Global); + assert_eq!(first.scope, GLOBAL_SCOPE); + } +} diff --git a/src/openhuman/memory_tree/tree_source/store.rs b/src/openhuman/memory_store/trees/store.rs similarity index 94% rename from src/openhuman/memory_tree/tree_source/store.rs rename to src/openhuman/memory_store/trees/store.rs index b801721fd..86a00ec31 100644 --- a/src/openhuman/memory_tree/tree_source/store.rs +++ b/src/openhuman/memory_store/trees/store.rs @@ -12,7 +12,7 @@ //! //! Phase 4 (#710) adds a nullable `embedding` blob on //! `mem_tree_summaries` — packed little-endian `f32` vectors via -//! [`crate::openhuman::memory_tree::score::embed::pack_embedding`]. New +//! [`crate::openhuman::memory::score::embed::pack_embedding`]. New //! writes populate it via [`insert_summary_tx`]; reads decode it when //! present. @@ -21,10 +21,10 @@ use chrono::{DateTime, TimeZone, Utc}; use rusqlite::{params, Connection, OptionalExtension, Transaction}; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::content_store::StagedSummary; -use crate::openhuman::memory_tree::score::embed::{decode_optional_blob, pack_checked}; -use crate::openhuman::memory_tree::store::with_connection; -use crate::openhuman::memory_tree::tree_source::types::{ +use crate::openhuman::memory::score::embed::{decode_optional_blob, pack_checked}; +use crate::openhuman::memory_store::chunks::store::with_connection; +use crate::openhuman::memory_store::content::StagedSummary; +use crate::openhuman::memory_store::trees::types::{ Buffer, SummaryNode, Tree, TreeKind, TreeStatus, }; @@ -263,7 +263,7 @@ pub(crate) fn insert_summary_tx( /// at the active signature (via [`set_summary_embedding_for_signature`]) /// instead of the legacy `mem_tree_summaries.embedding` column. The signature /// is resolved internally from `config` via the shared -/// [`crate::openhuman::memory_tree::store::tree_active_signature`] — same +/// [`crate::openhuman::memory_store::chunks::store::tree_active_signature`] — same /// resolution as the chunk path. Returns `1` on success (one sidecar row /// written/updated); the legacy "0 if id unknown" count no longer applies /// since the sidecar upsert does not join the parent summary row. @@ -272,9 +272,9 @@ pub fn set_summary_embedding( summary_id: &str, embedding: &[f32], ) -> Result { - let signature = crate::openhuman::memory_tree::store::tree_active_signature(config); + let signature = crate::openhuman::memory_store::chunks::store::tree_active_signature(config); log::debug!( - "[tree_source::store] set_summary_embedding: summary_id={summary_id} sig={signature} dims={}", + "[tree::store] set_summary_embedding: summary_id={summary_id} sig={signature} dims={}", embedding.len() ); set_summary_embedding_for_signature(config, summary_id, &signature, embedding)?; @@ -289,7 +289,7 @@ pub fn set_summary_embedding( /// vector exists under the active signature — graceful absence during the §7 /// backfill window, never a cross-space read. pub fn get_summary_embedding(config: &Config, summary_id: &str) -> Result>> { - let signature = crate::openhuman::memory_tree::store::tree_active_signature(config); + let signature = crate::openhuman::memory_store::chunks::store::tree_active_signature(config); get_summary_embedding_for_signature(config, summary_id, &signature) } @@ -348,9 +348,11 @@ pub fn mark_summary_reembed_skipped( model_signature: &str, reason: &str, ) -> Result<()> { - let summary_id = - crate::openhuman::memory_tree::store::validate_reembed_skip_key("summary_id", summary_id)?; - let model_signature = crate::openhuman::memory_tree::store::validate_reembed_skip_key( + let summary_id = crate::openhuman::memory_store::chunks::store::validate_reembed_skip_key( + "summary_id", + summary_id, + )?; + let model_signature = crate::openhuman::memory_store::chunks::store::validate_reembed_skip_key( "model_signature", model_signature, )?; @@ -366,7 +368,7 @@ pub fn mark_summary_reembed_skipped( params![summary_id, model_signature, reason, now_ms], )?; log::debug!( - "[memory_tree::store] mark_summary_reembed_skipped summary_id={summary_id} sig={model_signature} reason={reason}" + "[memory::chunk_store] mark_summary_reembed_skipped summary_id={summary_id} sig={model_signature} reason={reason}" ); Ok(()) }) @@ -374,15 +376,17 @@ pub fn mark_summary_reembed_skipped( /// Remove a single summary tombstone so re-embed backfill can retry the row. /// -/// Idempotent — see [`crate::openhuman::memory_tree::store::clear_chunk_reembed_skipped`]. +/// Idempotent — see [`crate::openhuman::memory_store::chunks::store::clear_chunk_reembed_skipped`]. pub fn clear_summary_reembed_skipped( config: &Config, summary_id: &str, model_signature: &str, ) -> Result<()> { - let summary_id = - crate::openhuman::memory_tree::store::validate_reembed_skip_key("summary_id", summary_id)?; - let model_signature = crate::openhuman::memory_tree::store::validate_reembed_skip_key( + let summary_id = crate::openhuman::memory_store::chunks::store::validate_reembed_skip_key( + "summary_id", + summary_id, + )?; + let model_signature = crate::openhuman::memory_store::chunks::store::validate_reembed_skip_key( "model_signature", model_signature, )?; @@ -393,7 +397,7 @@ pub fn clear_summary_reembed_skipped( params![summary_id, model_signature], )?; log::debug!( - "[memory_tree::store] clear_summary_reembed_skipped summary_id={summary_id} sig={model_signature}" + "[memory::chunk_store] clear_summary_reembed_skipped summary_id={summary_id} sig={model_signature}" ); Ok(()) }) diff --git a/src/openhuman/memory_tree/tree_source/store_tests.rs b/src/openhuman/memory_store/trees/store_tests.rs similarity index 100% rename from src/openhuman/memory_tree/tree_source/store_tests.rs rename to src/openhuman/memory_store/trees/store_tests.rs diff --git a/src/openhuman/memory_tree/tree_source/types.rs b/src/openhuman/memory_store/trees/types.rs similarity index 69% rename from src/openhuman/memory_tree/tree_source/types.rs rename to src/openhuman/memory_store/trees/types.rs index 4e7c652b7..86df08332 100644 --- a/src/openhuman/memory_tree/tree_source/types.rs +++ b/src/openhuman/memory_store/trees/types.rs @@ -250,3 +250,118 @@ mod tests { assert!(!b.is_stale(Utc::now(), chrono::Duration::hours(20))); } } + +// ============================================================================ +// Topic-tree hotness (Phase 3c) — formerly memory_store::trees_topic::types +// ============================================================================ +// +// Folded in here because topic and global trees are not structurally distinct +// from source trees — they all live in the same `mem_tree_trees` table keyed +// by `TreeKind`. The only topic-specific extra state is the entity hotness +// counters in `mem_tree_entity_hotness`, which gate materialisation of a +// topic tree but are themselves not trees. + +/// Hotness threshold above which a topic tree is materialised for an entity. +pub const TOPIC_CREATION_THRESHOLD: f32 = 10.0; + +/// Hotness threshold below which a topic tree becomes an archive candidate. +pub const TOPIC_ARCHIVE_THRESHOLD: f32 = 2.0; + +/// How often (in ingests touching the entity) to recompute hotness from the +/// full [`EntityIndexStats`]. Between recomputes only the cheap counters bump. +pub const TOPIC_RECHECK_EVERY: u32 = 100; + +/// Input record fed to the hotness math (see `memory_tree::topic::hotness`). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct EntityIndexStats { + pub mention_count_30d: u32, + pub distinct_sources: u32, + pub last_seen_ms: Option, + pub query_hits_30d: u32, + pub graph_centrality: Option, +} + +/// Row persisted in `mem_tree_entity_hotness`. Persistence helpers live in +/// [`super::hotness`]. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct HotnessCounters { + pub entity_id: String, + pub mention_count_30d: u32, + pub distinct_sources: u32, + pub last_seen_ms: Option, + pub query_hits_30d: u32, + pub graph_centrality: Option, + pub ingests_since_check: u32, + pub last_hotness: Option, + pub last_updated_ms: i64, +} + +impl HotnessCounters { + pub fn fresh(entity_id: &str, now_ms: i64) -> Self { + Self { + entity_id: entity_id.to_string(), + mention_count_30d: 0, + distinct_sources: 0, + last_seen_ms: None, + query_hits_30d: 0, + graph_centrality: None, + ingests_since_check: 0, + last_hotness: None, + last_updated_ms: now_ms, + } + } + + pub fn stats(&self) -> EntityIndexStats { + EntityIndexStats { + mention_count_30d: self.mention_count_30d, + distinct_sources: self.distinct_sources, + last_seen_ms: self.last_seen_ms, + query_hits_30d: self.query_hits_30d, + graph_centrality: self.graph_centrality, + } + } +} + +#[cfg(test)] +mod hotness_type_tests { + use super::*; + + #[test] + fn fresh_counters_are_zero() { + let c = HotnessCounters::fresh("email:alice@example.com", 1_700_000_000_000); + assert_eq!(c.entity_id, "email:alice@example.com"); + assert_eq!(c.mention_count_30d, 0); + assert_eq!(c.distinct_sources, 0); + assert_eq!(c.ingests_since_check, 0); + assert!(c.last_hotness.is_none()); + assert!(c.last_seen_ms.is_none()); + assert_eq!(c.last_updated_ms, 1_700_000_000_000); + } + + #[test] + fn stats_projection_mirrors_row() { + let c = HotnessCounters { + entity_id: "e".into(), + mention_count_30d: 5, + distinct_sources: 2, + last_seen_ms: Some(42), + query_hits_30d: 1, + graph_centrality: Some(0.3), + ingests_since_check: 4, + last_hotness: Some(9.9), + last_updated_ms: 100, + }; + let s = c.stats(); + assert_eq!(s.mention_count_30d, 5); + assert_eq!(s.distinct_sources, 2); + assert_eq!(s.last_seen_ms, Some(42)); + assert_eq!(s.query_hits_30d, 1); + assert_eq!(s.graph_centrality, Some(0.3)); + } + + #[test] + fn thresholds_make_creation_strictly_above_archive() { + assert!(TOPIC_CREATION_THRESHOLD > TOPIC_ARCHIVE_THRESHOLD); + assert!(TOPIC_RECHECK_EVERY > 0); + } +} diff --git a/src/openhuman/memory/store/types.rs b/src/openhuman/memory_store/types.rs similarity index 55% rename from src/openhuman/memory/store/types.rs rename to src/openhuman/memory_store/types.rs index dc65e8677..655fde8bb 100644 --- a/src/openhuman/memory/store/types.rs +++ b/src/openhuman/memory_store/types.rs @@ -139,3 +139,106 @@ pub struct NamespaceRetrievalContext { pub context_text: String, pub hits: Vec, } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn global_namespace_constant_is_stable() { + assert_eq!(GLOBAL_NAMESPACE, "global"); + } + + #[test] + fn memory_item_kind_serde_uses_snake_case() { + let json_value = serde_json::to_string(&MemoryItemKind::Document).unwrap(); + assert_eq!(json_value, "\"document\""); + let decoded: MemoryItemKind = serde_json::from_str("\"episodic\"").unwrap(); + assert_eq!(decoded, MemoryItemKind::Episodic); + } + + #[test] + fn namespace_document_input_defaults_optional_fields() { + let value = json!({ + "namespace": "global", + "key": "note-1", + "title": "Title", + "content": "Body", + "source_type": "manual", + "priority": "normal", + "metadata": {}, + "category": "core" + }); + let parsed: NamespaceDocumentInput = serde_json::from_value(value).unwrap(); + assert!(parsed.tags.is_empty()); + assert_eq!(parsed.metadata, json!({})); + assert!(parsed.session_id.is_none()); + assert!(parsed.document_id.is_none()); + } + + #[test] + fn retrieval_score_breakdown_default_is_zeroed() { + let breakdown = RetrievalScoreBreakdown::default(); + assert_eq!(breakdown.keyword_relevance, 0.0); + assert_eq!(breakdown.vector_similarity, 0.0); + assert_eq!(breakdown.graph_relevance, 0.0); + assert_eq!(breakdown.episodic_relevance, 0.0); + assert_eq!(breakdown.freshness, 0.0); + assert_eq!(breakdown.final_score, 0.0); + } + + #[test] + fn memory_kv_record_roundtrips_with_optional_namespace() { + let global = MemoryKvRecord { + namespace: None, + key: "theme".into(), + value: json!("dark"), + updated_at: 1.5, + }; + let namespaced = MemoryKvRecord { + namespace: Some("project".into()), + key: "state".into(), + value: json!({"open": true}), + updated_at: 2.5, + }; + for record in [global, namespaced] { + let value = serde_json::to_value(&record).unwrap(); + let decoded: MemoryKvRecord = serde_json::from_value(value).unwrap(); + assert_eq!(decoded.namespace, record.namespace); + assert_eq!(decoded.key, record.key); + assert_eq!(decoded.value, record.value); + assert_eq!(decoded.updated_at, record.updated_at); + } + } + + #[test] + fn namespace_memory_hit_defaults_optional_fields() { + let hit: NamespaceMemoryHit = serde_json::from_value(json!({ + "id": "hit-1", + "kind": "document", + "namespace": "global", + "key": "note-1", + "title": "Title", + "content": "Body", + "category": "core", + "source_type": "manual", + "updated_at": 3.5, + "score": 0.8, + "score_breakdown": { + "keyword_relevance": 0.5, + "vector_similarity": 0.2, + "graph_relevance": 0.0, + "episodic_relevance": 0.0, + "freshness": 0.1, + "final_score": 0.8 + } + })) + .unwrap(); + + assert!(hit.document_id.is_none()); + assert!(hit.chunk_id.is_none()); + assert!(hit.supporting_relations.is_empty()); + assert_eq!(hit.kind, MemoryItemKind::Document); + } +} diff --git a/src/openhuman/memory/store/unified/README.md b/src/openhuman/memory_store/unified/README.md similarity index 100% rename from src/openhuman/memory/store/unified/README.md rename to src/openhuman/memory_store/unified/README.md diff --git a/src/openhuman/memory/store/unified/documents.rs b/src/openhuman/memory_store/unified/documents.rs similarity index 99% rename from src/openhuman/memory/store/unified/documents.rs rename to src/openhuman/memory_store/unified/documents.rs index ced96762f..38f5226a0 100644 --- a/src/openhuman/memory/store/unified/documents.rs +++ b/src/openhuman/memory_store/unified/documents.rs @@ -9,8 +9,8 @@ use serde_json::{json, Value}; use std::collections::BTreeSet; use uuid::Uuid; -use crate::openhuman::memory::safety; -use crate::openhuman::memory::store::types::{NamespaceDocumentInput, StoredMemoryDocument}; +use crate::openhuman::memory_store::safety; +use crate::openhuman::memory_store::types::{NamespaceDocumentInput, StoredMemoryDocument}; use super::UnifiedMemory; diff --git a/src/openhuman/memory/store/unified/documents_tests.rs b/src/openhuman/memory_store/unified/documents_tests.rs similarity index 50% rename from src/openhuman/memory/store/unified/documents_tests.rs rename to src/openhuman/memory_store/unified/documents_tests.rs index 21dd18c2e..bd265968d 100644 --- a/src/openhuman/memory/store/unified/documents_tests.rs +++ b/src/openhuman/memory_store/unified/documents_tests.rs @@ -29,6 +29,530 @@ fn make_doc_input( } } +fn count_vector_chunks(memory: &UnifiedMemory, namespace: &str, document_id: &str) -> i64 { + let conn = memory.conn.lock(); + conn.query_row( + "SELECT COUNT(*) FROM vector_chunks WHERE namespace = ?1 AND document_id = ?2", + rusqlite::params![UnifiedMemory::sanitize_namespace(namespace), document_id], + |row| row.get(0), + ) + .unwrap() +} + +#[tokio::test] +async fn list_documents_without_namespace_returns_all_docs_across_namespaces() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + memory + .upsert_document(make_doc_input("test:one", "doc-a", "Doc A", "A body")) + .await + .unwrap(); + memory + .upsert_document(make_doc_input("test:two", "doc-b", "Doc B", "B body")) + .await + .unwrap(); + + let docs = memory.list_documents(None).await.unwrap(); + assert_eq!(docs["count"].as_u64().unwrap(), 2); + let namespaces: std::collections::BTreeSet<_> = docs["documents"] + .as_array() + .unwrap() + .iter() + .filter_map(|doc| doc["namespace"].as_str()) + .collect(); + assert!(namespaces.contains("test_one")); + assert!(namespaces.contains("test_two")); +} + +#[tokio::test] +async fn list_namespaces_returns_distinct_sorted_sanitized_namespaces() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + memory + .upsert_document(make_doc_input("team alpha/#1", "doc-a", "Doc A", "A body")) + .await + .unwrap(); + memory + .upsert_document(make_doc_input("team alpha/#1", "doc-b", "Doc B", "B body")) + .await + .unwrap(); + memory + .upsert_document(make_doc_input("zeta", "doc-c", "Doc C", "C body")) + .await + .unwrap(); + + let namespaces = memory.list_namespaces().await.unwrap(); + assert_eq!( + namespaces, + vec!["team_alpha/_1".to_string(), "zeta".to_string()] + ); +} + +#[tokio::test] +async fn list_documents_with_namespace_filters_by_sanitized_namespace_and_orders_newest_first() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + memory + .upsert_document(make_doc_input( + "team alpha/#1", + "doc-a", + "Older Doc", + "A body", + )) + .await + .unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + memory + .upsert_document(make_doc_input( + "team alpha/#1", + "doc-b", + "Newer Doc", + "B body", + )) + .await + .unwrap(); + memory + .upsert_document(make_doc_input("other", "doc-c", "Other Doc", "C body")) + .await + .unwrap(); + + let docs = memory.list_documents(Some("team alpha/#1")).await.unwrap(); + let documents = docs["documents"].as_array().unwrap(); + + assert_eq!(docs["count"].as_u64().unwrap(), 2); + assert_eq!(documents[0]["namespace"], json!("team_alpha/_1")); + assert_eq!(documents[0]["key"], json!("doc-b")); + assert_eq!(documents[1]["key"], json!("doc-a")); +} + +#[tokio::test] +async fn load_documents_for_scope_defaults_invalid_json_fields_from_persisted_rows() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + let namespace = UnifiedMemory::sanitize_namespace("broken/json"); + + { + let conn = memory.conn.lock(); + conn.execute( + "INSERT INTO memory_docs + (document_id, namespace, key, title, content, source_type, priority, tags_json, metadata_json, category, session_id, created_at, updated_at, markdown_rel_path) + VALUES + (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)", + rusqlite::params![ + "doc-invalid-json", + namespace, + "doc-a", + "Doc A", + "Body", + "doc", + "medium", + "{not json", + "also not json", + "core", + Option::::None, + 10.0_f64, + 20.0_f64, + "memory/namespaces/broken_json/docs/doc-invalid-json.md" + ], + ) + .unwrap(); + } + + let docs = memory + .load_documents_for_scope("broken/json") + .await + .unwrap(); + assert_eq!(docs.len(), 1); + assert!( + docs[0].tags.is_empty(), + "invalid tags_json should fall back to []" + ); + assert_eq!( + docs[0].metadata, + json!({}), + "invalid metadata_json should fall back to an empty object" + ); +} + +#[tokio::test] +async fn upsert_document_metadata_only_reuses_document_id_for_same_namespace_and_key() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let first_id = memory + .upsert_document_metadata_only(make_doc_input( + "test:meta", + "doc-a", + "Doc A", + "Initial body", + )) + .await + .unwrap(); + let second_id = memory + .upsert_document_metadata_only(make_doc_input( + "test:meta", + "doc-a", + "Doc A v2", + "Updated body", + )) + .await + .unwrap(); + + assert_eq!( + first_id, second_id, + "metadata-only upsert should reuse the document id" + ); + let docs = memory.load_documents_for_scope("test:meta").await.unwrap(); + assert_eq!(docs.len(), 1); + assert_eq!(docs[0].document_id, first_id); + assert_eq!(docs[0].title, "Doc A v2"); + assert_eq!(docs[0].content, "Updated body"); + assert_eq!( + count_vector_chunks(&memory, "test:meta", &first_id), + 0, + "metadata-only writes must not enqueue vector chunks" + ); +} + +#[tokio::test] +async fn upsert_document_metadata_only_preserves_created_at_and_rewrites_sidecar() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let first_id = memory + .upsert_document_metadata_only(make_doc_input( + "test:meta-sidecar", + "doc-a", + "Doc A", + "Initial body", + )) + .await + .unwrap(); + let first_doc = memory + .load_documents_for_scope("test:meta-sidecar") + .await + .unwrap()[0] + .clone(); + let sidecar = tmp.path().join(&first_doc.markdown_rel_path); + let first_markdown = std::fs::read_to_string(&sidecar).unwrap(); + assert!(first_markdown.contains("Initial body")); + + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + + let second_id = memory + .upsert_document_metadata_only(make_doc_input( + "test:meta-sidecar", + "doc-a", + "Doc A v2", + "Updated body", + )) + .await + .unwrap(); + + assert_eq!(first_id, second_id); + let updated_doc = memory + .load_documents_for_scope("test:meta-sidecar") + .await + .unwrap()[0] + .clone(); + assert_eq!(updated_doc.created_at, first_doc.created_at); + assert!(updated_doc.updated_at >= first_doc.updated_at); + let updated_markdown = std::fs::read_to_string(sidecar).unwrap(); + assert!(updated_markdown.contains("Updated body")); + assert!(updated_markdown.contains("Doc A v2")); +} + +#[tokio::test] +async fn upsert_document_metadata_only_over_existing_document_preserves_vector_chunks() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let document_id = memory + .upsert_document(make_doc_input( + "test:meta-preserve-chunks", + "doc-a", + "Doc A", + &"alpha ".repeat(400), + )) + .await + .unwrap(); + let original_chunk_count = + count_vector_chunks(&memory, "test:meta-preserve-chunks", &document_id); + assert!(original_chunk_count > 0); + + let updated_id = memory + .upsert_document_metadata_only(make_doc_input( + "test:meta-preserve-chunks", + "doc-a", + "Doc A v2", + "Updated body without re-embedding", + )) + .await + .unwrap(); + + assert_eq!(updated_id, document_id); + let docs = memory + .load_documents_for_scope("test:meta-preserve-chunks") + .await + .unwrap(); + assert_eq!(docs.len(), 1); + assert_eq!(docs[0].content, "Updated body without re-embedding"); + assert_eq!( + count_vector_chunks(&memory, "test:meta-preserve-chunks", &document_id), + original_chunk_count, + "metadata-only writes should not delete existing semantic chunks" + ); +} + +#[tokio::test] +async fn upsert_document_after_metadata_only_reuses_document_id_and_adds_vector_chunks() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let metadata_only_id = memory + .upsert_document_metadata_only(make_doc_input( + "test:meta-then-full", + "doc-a", + "Doc A", + "Short body", + )) + .await + .unwrap(); + assert_eq!( + count_vector_chunks(&memory, "test:meta-then-full", &metadata_only_id), + 0 + ); + + let full_id = memory + .upsert_document(make_doc_input( + "test:meta-then-full", + "doc-a", + "Doc A Embedded", + &"beta ".repeat(400), + )) + .await + .unwrap(); + + assert_eq!(full_id, metadata_only_id); + let docs = memory + .load_documents_for_scope("test:meta-then-full") + .await + .unwrap(); + assert_eq!(docs.len(), 1); + assert_eq!(docs[0].title, "Doc A Embedded"); + assert!( + count_vector_chunks(&memory, "test:meta-then-full", &full_id) > 0, + "full upsert should backfill chunks for a metadata-only document" + ); +} + +#[tokio::test] +async fn upsert_document_writes_vector_chunks_for_chunked_content() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let long_body = "alpha ".repeat(400); + let document_id = memory + .upsert_document(make_doc_input("test:vector", "doc-a", "Doc A", &long_body)) + .await + .unwrap(); + + assert!( + count_vector_chunks(&memory, "test:vector", &document_id) > 0, + "full document upsert should replace vector chunks for semantic retrieval" + ); +} + +#[tokio::test] +async fn upsert_document_reuses_document_id_preserves_created_at_and_replaces_vector_chunks() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let first_id = memory + .upsert_document(make_doc_input( + "test:update", + "doc-a", + "Doc A", + &"alpha ".repeat(400), + )) + .await + .unwrap(); + let first_doc = memory + .load_documents_for_scope("test:update") + .await + .unwrap()[0] + .clone(); + let first_chunk_count = count_vector_chunks(&memory, "test:update", &first_id); + assert!(first_chunk_count > 0); + + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + + let second_id = memory + .upsert_document(make_doc_input( + "test:update", + "doc-a", + "Doc A v2", + &"beta ".repeat(40), + )) + .await + .unwrap(); + + assert_eq!( + first_id, second_id, + "upsert should reuse the existing document id" + ); + let updated_doc = memory + .load_documents_for_scope("test:update") + .await + .unwrap()[0] + .clone(); + assert_eq!(updated_doc.document_id, first_id); + assert_eq!(updated_doc.created_at, first_doc.created_at); + assert!(updated_doc.updated_at >= first_doc.updated_at); + assert_eq!(updated_doc.title, "Doc A v2"); + assert_eq!(updated_doc.content, "beta ".repeat(40)); + let second_chunk_count = count_vector_chunks(&memory, "test:update", &second_id); + assert!(second_chunk_count > 0); + assert!( + second_chunk_count <= first_chunk_count, + "replacing with shorter content should not leave stale vector chunks behind" + ); +} + +#[tokio::test] +async fn delete_document_removes_doc_sidecar_and_is_idempotent() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let document_id = memory + .upsert_document(make_doc_input("test:delete", "doc-a", "Doc A", "Delete me")) + .await + .unwrap(); + + let docs = memory + .load_documents_for_scope("test:delete") + .await + .unwrap(); + assert_eq!(docs.len(), 1); + let sidecar = tmp.path().join(&docs[0].markdown_rel_path); + assert!(sidecar.exists(), "sidecar should exist before delete"); + + memory + .graph_upsert_namespace( + "test:delete", + "Alice", + "OWNS", + "Phoenix", + &json!({ + "document_id": document_id.clone(), + "chunk_id": format!("{document_id}:0") + }), + ) + .await + .unwrap(); + + let deleted = memory + .delete_document("test:delete", &document_id) + .await + .unwrap(); + assert_eq!(deleted["deleted"], json!(true)); + assert_eq!(deleted["documentId"], json!(document_id.clone())); + assert!(!sidecar.exists(), "sidecar should be removed on delete"); + assert!(memory + .load_documents_for_scope("test:delete") + .await + .unwrap() + .is_empty()); + assert!( + memory + .graph_relations_namespace("test:delete", None, None) + .await + .unwrap() + .is_empty(), + "document-linked graph relations should be pruned" + ); + + let second = memory + .delete_document("test:delete", &document_id) + .await + .unwrap(); + assert_eq!(second["deleted"], json!(false)); + assert_eq!(second["documentId"], json!(document_id)); +} + +#[tokio::test] +async fn delete_document_succeeds_when_sidecar_is_already_missing() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let document_id = memory + .upsert_document(make_doc_input( + "test:delete-missing-sidecar", + "doc-a", + "Doc A", + "Delete me", + )) + .await + .unwrap(); + + let docs = memory + .load_documents_for_scope("test:delete-missing-sidecar") + .await + .unwrap(); + assert_eq!(docs.len(), 1); + let sidecar = tmp.path().join(&docs[0].markdown_rel_path); + assert!(sidecar.exists()); + std::fs::remove_file(&sidecar).unwrap(); + assert!(!sidecar.exists()); + + let deleted = memory + .delete_document("test:delete-missing-sidecar", &document_id) + .await + .unwrap(); + assert_eq!(deleted["deleted"], json!(true)); + assert!(memory + .load_documents_for_scope("test:delete-missing-sidecar") + .await + .unwrap() + .is_empty()); +} + +#[tokio::test] +async fn delete_document_accepts_unsanitized_namespace_and_removes_chunks() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let document_id = memory + .upsert_document(make_doc_input( + "Team Alpha/#1", + "doc-a", + "Doc A", + &"delete ".repeat(300), + )) + .await + .unwrap(); + assert!(count_vector_chunks(&memory, "Team Alpha/#1", &document_id) > 0); + + let deleted = memory + .delete_document("Team Alpha/#1", &document_id) + .await + .unwrap(); + assert_eq!(deleted["deleted"], json!(true)); + assert_eq!(deleted["namespace"], json!("Team_Alpha/_1")); + assert_eq!( + count_vector_chunks(&memory, "Team Alpha/#1", &document_id), + 0 + ); + assert!(memory + .load_documents_for_scope("Team Alpha/#1") + .await + .unwrap() + .is_empty()); +} + #[tokio::test] async fn clear_namespace_removes_all_data_and_preserves_other_namespaces() { let tmp = TempDir::new().unwrap(); @@ -257,6 +781,88 @@ async fn clear_namespace_removes_on_disk_markdown_files() { ); } +#[tokio::test] +async fn clear_namespace_accepts_unsanitized_namespace_and_removes_sanitized_docs_dir() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + memory + .upsert_document(make_doc_input( + "Team Alpha/#1", + "doc-a", + "Doc A", + "Namespace cleanup body", + )) + .await + .unwrap(); + memory + .kv_set_namespace("Team Alpha/#1", "pref-1", &json!({"theme": "dark"})) + .await + .unwrap(); + + let docs_dir = tmp + .path() + .join("memory") + .join("namespaces") + .join("Team_Alpha/_1") + .join("docs"); + assert!(docs_dir.exists()); + + memory.clear_namespace("Team Alpha/#1").await.unwrap(); + + assert!(memory + .load_documents_for_scope("Team Alpha/#1") + .await + .unwrap() + .is_empty()); + assert!(memory + .kv_list_namespace("Team Alpha/#1") + .await + .unwrap() + .is_empty()); + assert!(!docs_dir.exists()); +} + +#[tokio::test] +async fn list_namespaces_skips_blank_rows_inserted_outside_normal_writes() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + { + let conn = memory.conn.lock(); + conn.execute( + "INSERT INTO memory_docs + (document_id, namespace, key, title, content, source_type, priority, tags_json, metadata_json, category, session_id, created_at, updated_at, markdown_rel_path) + VALUES + (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)", + rusqlite::params![ + "doc-blank-ns", + " ", + "doc-a", + "Doc A", + "Body", + "doc", + "medium", + "[]", + "{}", + "core", + Option::::None, + 10.0_f64, + 20.0_f64, + "memory/namespaces/blank/docs/doc-blank-ns.md" + ], + ) + .unwrap(); + } + memory + .upsert_document(make_doc_input("valid/ns", "doc-b", "Doc B", "Body")) + .await + .unwrap(); + + let namespaces = memory.list_namespaces().await.unwrap(); + assert_eq!(namespaces, vec!["valid/ns".to_string()]); +} + #[tokio::test] async fn upsert_document_redacts_secret_like_content_before_persisting() { let tmp = TempDir::new().unwrap(); diff --git a/src/openhuman/memory/store/unified/events.rs b/src/openhuman/memory_store/unified/events.rs similarity index 100% rename from src/openhuman/memory/store/unified/events.rs rename to src/openhuman/memory_store/unified/events.rs diff --git a/src/openhuman/memory/store/unified/events_tests.rs b/src/openhuman/memory_store/unified/events_tests.rs similarity index 100% rename from src/openhuman/memory/store/unified/events_tests.rs rename to src/openhuman/memory_store/unified/events_tests.rs diff --git a/src/openhuman/memory/store/unified/fts5.rs b/src/openhuman/memory_store/unified/fts5.rs similarity index 99% rename from src/openhuman/memory/store/unified/fts5.rs rename to src/openhuman/memory_store/unified/fts5.rs index 08aaafc7e..09242fdb5 100644 --- a/src/openhuman/memory/store/unified/fts5.rs +++ b/src/openhuman/memory_store/unified/fts5.rs @@ -10,7 +10,7 @@ use rusqlite::Connection; use serde::{Deserialize, Serialize}; use std::sync::Arc; -use crate::openhuman::memory::safety; +use crate::openhuman::memory_store::safety; /// A single episodic record (one turn or event). #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/src/openhuman/memory/store/unified/graph.rs b/src/openhuman/memory_store/unified/graph.rs similarity index 64% rename from src/openhuman/memory/store/unified/graph.rs rename to src/openhuman/memory_store/unified/graph.rs index ed4737cf4..f1a00db00 100644 --- a/src/openhuman/memory/store/unified/graph.rs +++ b/src/openhuman/memory_store/unified/graph.rs @@ -7,7 +7,7 @@ use rusqlite::{params, OptionalExtension}; use serde_json::{json, Map, Value}; -use crate::openhuman::memory::store::types::GraphRelationRecord; +use crate::openhuman::memory_store::types::GraphRelationRecord; use super::UnifiedMemory; @@ -510,3 +510,333 @@ impl UnifiedMemory { }) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::embeddings::NoopEmbedding; + use std::sync::Arc; + use tempfile::TempDir; + + #[test] + fn merge_graph_attrs_accumulates_evidence_and_dedupes_ids() { + let existing = json!({ + "evidence_count": 2, + "document_ids": ["doc-1"], + "chunk_ids": ["doc-1:chunk-1"], + "order_index": 7, + "created_at": 1.0 + }); + let incoming = json!({ + "evidence_count": 3, + "document_ids": ["doc-1", "doc-2"], + "chunk_ids": ["doc-2:chunk-9"], + "order_index": 3, + "attrs_only": true + }); + + let merged = UnifiedMemory::merge_graph_attrs(Some(&existing.to_string()), &incoming, 9.0); + assert_eq!(merged["evidence_count"], json!(5)); + assert_eq!(merged["document_ids"], json!(["doc-1", "doc-2"])); + assert_eq!( + merged["chunk_ids"], + json!(["doc-1:chunk-1", "doc-2:chunk-9"]) + ); + assert_eq!(merged["order_index"], json!(3)); + assert_eq!(merged["created_at"], json!(1.0)); + assert_eq!(merged["updated_at"], json!(9.0)); + assert_eq!(merged["attrs_only"], json!(true)); + } + + #[test] + fn graph_relation_from_parts_extracts_counts_and_ids() { + let record = UnifiedMemory::graph_relation_from_parts( + Some("global".into()), + "Alice".into(), + "OWNS".into(), + "OpenHuman".into(), + r#"{"evidence_count":2,"order_index":4,"document_ids":["doc-1"],"chunk_ids":["doc-1:chunk-1"]}"#, + 5.0, + ); + assert_eq!(record.namespace.as_deref(), Some("global")); + assert_eq!(record.evidence_count, 2); + assert_eq!(record.order_index, Some(4)); + assert_eq!(record.document_ids, vec!["doc-1".to_string()]); + assert_eq!(record.chunk_ids, vec!["doc-1:chunk-1".to_string()]); + } + + #[test] + fn merge_graph_attrs_recovers_from_invalid_existing_json_and_negative_evidence() { + let incoming = json!({ + "evidence_count": -4, + "document_id": "doc-2", + "chunk_id": "doc-2:chunk-9", + "order_index": 8 + }); + + let merged = UnifiedMemory::merge_graph_attrs(Some("not-json"), &incoming, 11.0); + assert_eq!( + merged["evidence_count"], + json!(1), + "negative evidence should clamp to the minimum count" + ); + assert_eq!(merged["document_ids"], json!(["doc-2"])); + assert_eq!(merged["chunk_ids"], json!(["doc-2:chunk-9"])); + assert_eq!(merged["order_index"], json!(8)); + assert_eq!(merged["created_at"], json!(11.0)); + assert_eq!(merged["updated_at"], json!(11.0)); + } + + #[test] + fn graph_relation_from_parts_defaults_invalid_attrs_payload() { + let record = UnifiedMemory::graph_relation_from_parts( + None, + "Alice".into(), + "OWNS".into(), + "Phoenix".into(), + "not-json", + 7.5, + ); + assert_eq!(record.evidence_count, 1); + assert_eq!(record.order_index, None); + assert!(record.document_ids.is_empty()); + assert!(record.chunk_ids.is_empty()); + assert_eq!(record.attrs, json!({})); + } + + #[test] + fn graph_relation_to_json_uses_expected_public_keys() { + let value = UnifiedMemory::graph_relation_to_json(GraphRelationRecord { + namespace: None, + subject: "Alice".into(), + predicate: "OWNS".into(), + object: "OpenHuman".into(), + attrs: json!({"extra": true}), + updated_at: 1.5, + evidence_count: 1, + order_index: Some(2), + document_ids: vec!["doc-1".into()], + chunk_ids: vec!["doc-1:chunk-1".into()], + }); + assert_eq!(value["subject"], "Alice"); + assert_eq!(value["predicate"], "OWNS"); + assert_eq!(value["evidenceCount"], 1); + assert_eq!(value["orderIndex"], 2); + assert_eq!(value["documentIds"], json!(["doc-1"])); + assert_eq!(value["chunkIds"], json!(["doc-1:chunk-1"])); + } + + fn test_memory() -> (TempDir, UnifiedMemory) { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + (tmp, memory) + } + + #[tokio::test] + async fn graph_upsert_namespace_merges_attrs_and_query_returns_json() { + let (_tmp, memory) = test_memory(); + memory + .graph_upsert_namespace( + "team alpha/#1", + "Alice", + "OWNS", + "Phoenix", + &json!({ + "document_id": "doc-1", + "chunk_id": "doc-1:chunk-1", + "evidence_count": 1 + }), + ) + .await + .unwrap(); + memory + .graph_upsert_namespace( + "team alpha/#1", + "Alice", + "OWNS", + "Phoenix", + &json!({ + "document_ids": ["doc-2"], + "chunk_ids": ["doc-2:chunk-9"], + "order_index": 2 + }), + ) + .await + .unwrap(); + + let rows = memory + .graph_query_namespace("team alpha/#1", Some("Alice"), Some("OWNS")) + .await + .unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0]["subject"], "ALICE"); + assert_eq!(rows[0]["predicate"], "OWNS"); + assert_eq!(rows[0]["object"], "PHOENIX"); + assert_eq!(rows[0]["evidenceCount"], 2); + assert_eq!(rows[0]["orderIndex"], 2); + assert_eq!(rows[0]["documentIds"], json!(["doc-1", "doc-2"])); + assert_eq!( + rows[0]["chunkIds"], + json!(["doc-1:chunk-1", "doc-2:chunk-9"]) + ); + + let scoped = memory + .graph_relations_for_scope("team alpha/#1") + .await + .unwrap(); + assert_eq!(scoped.len(), 1); + assert_eq!(scoped[0].namespace.as_deref(), Some("team_alpha/_1")); + } + + #[tokio::test] + async fn graph_global_and_all_queries_include_expected_rows() { + let (_tmp, memory) = test_memory(); + memory + .graph_upsert_global( + "Bob", + "MENTIONED", + "Launch", + &json!({"document_id": "doc-global"}), + ) + .await + .unwrap(); + memory + .graph_upsert_namespace( + "project", + "Alice", + "OWNS", + "Phoenix", + &json!({"document_id": "doc-local"}), + ) + .await + .unwrap(); + + let global = memory + .graph_query_global(Some("Bob"), Some("MENTIONED")) + .await + .unwrap(); + assert_eq!(global.len(), 1); + assert_eq!(global[0]["namespace"], Value::Null); + assert_eq!(global[0]["subject"], "BOB"); + + let all = memory.graph_query_all(None, None).await.unwrap(); + assert_eq!(all.len(), 2); + assert!(all.iter().any(|row| row["subject"] == "ALICE")); + assert!(all.iter().any(|row| row["subject"] == "BOB")); + } + + #[tokio::test] + async fn graph_relations_for_scope_includes_global_rows_and_sorts_newest_first() { + let (_tmp, memory) = test_memory(); + memory + .graph_upsert_namespace( + "scope-a", + "Alice", + "OWNS", + "Phoenix", + &json!({"document_id": "doc-local"}), + ) + .await + .unwrap(); + memory + .graph_upsert_global( + "Bob", + "MENTIONED", + "Launch", + &json!({"document_id": "doc-global"}), + ) + .await + .unwrap(); + + let scoped = memory.graph_relations_for_scope("scope-a").await.unwrap(); + assert_eq!(scoped.len(), 2); + assert!(scoped + .iter() + .any(|row| row.namespace.as_deref() == Some("scope-a"))); + assert!(scoped.iter().any(|row| row.namespace.is_none())); + assert!( + scoped[0].updated_at >= scoped[1].updated_at, + "scope queries should stay sorted newest-first across namespace+global rows" + ); + } + + #[tokio::test] + async fn graph_remove_document_namespace_prunes_or_deletes_relations() { + let (_tmp, memory) = test_memory(); + memory + .graph_upsert_namespace( + "cleanup", + "Alice", + "OWNS", + "Phoenix", + &json!({ + "document_ids": ["doc-1", "doc-2"], + "chunk_ids": ["doc-1:chunk-1", "doc-2:chunk-2"] + }), + ) + .await + .unwrap(); + memory + .graph_upsert_namespace( + "cleanup", + "Alice", + "BLOCKED", + "Atlas", + &json!({ + "document_id": "doc-1", + "chunk_id": "doc-1:chunk-9" + }), + ) + .await + .unwrap(); + + memory + .graph_remove_document_namespace("cleanup", "doc-1") + .await + .unwrap(); + + let rows = memory + .graph_query_namespace("cleanup", None, None) + .await + .unwrap(); + assert_eq!( + rows.len(), + 1, + "single-doc relation should be deleted entirely" + ); + assert_eq!(rows[0]["predicate"], "OWNS"); + assert_eq!(rows[0]["documentIds"], json!(["doc-2"])); + assert_eq!(rows[0]["chunkIds"], json!(["doc-2:chunk-2"])); + } + + #[tokio::test] + async fn graph_remove_document_namespace_is_noop_for_unrelated_document() { + let (_tmp, memory) = test_memory(); + memory + .graph_upsert_namespace( + "cleanup", + "Alice", + "OWNS", + "Phoenix", + &json!({ + "document_ids": ["doc-2"], + "chunk_ids": ["doc-2:chunk-2"] + }), + ) + .await + .unwrap(); + + memory + .graph_remove_document_namespace("cleanup", "doc-missing") + .await + .unwrap(); + + let rows = memory + .graph_query_namespace("cleanup", None, None) + .await + .unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0]["documentIds"], json!(["doc-2"])); + assert_eq!(rows[0]["chunkIds"], json!(["doc-2:chunk-2"])); + } +} diff --git a/src/openhuman/memory/store/unified/helpers.rs b/src/openhuman/memory_store/unified/helpers.rs similarity index 99% rename from src/openhuman/memory/store/unified/helpers.rs rename to src/openhuman/memory_store/unified/helpers.rs index 0f4cbd813..baeffe65f 100644 --- a/src/openhuman/memory/store/unified/helpers.rs +++ b/src/openhuman/memory_store/unified/helpers.rs @@ -2,7 +2,7 @@ //! cosine similarity, markdown chunking, text/predicate normalization, JSON //! attribute merging, and recency scoring. -use crate::openhuman::memory::chunker::chunk_markdown; +use crate::openhuman::memory_store::chunks::chunk_semantic as chunk_markdown; use super::UnifiedMemory; diff --git a/src/openhuman/memory/store/unified/init.rs b/src/openhuman/memory_store/unified/init.rs similarity index 87% rename from src/openhuman/memory/store/unified/init.rs rename to src/openhuman/memory_store/unified/init.rs index ba4c173ca..15736f807 100644 --- a/src/openhuman/memory/store/unified/init.rs +++ b/src/openhuman/memory_store/unified/init.rs @@ -13,7 +13,7 @@ use parking_lot::Mutex; use rusqlite::Connection; use crate::openhuman::embeddings::EmbeddingProvider; -use crate::openhuman::memory::store::types::GLOBAL_NAMESPACE; +use crate::openhuman::memory_store::types::GLOBAL_NAMESPACE; use super::UnifiedMemory; @@ -144,6 +144,19 @@ impl UnifiedMemory { // Conversation segmentation tables. conn.execute_batch(super::segments::SEGMENTS_INIT_SQL)?; + // Backfill the (start_seq, end_seq) columns on existing databases + // — fresh installs get them from SEGMENTS_INIT_SQL above; older DBs + // need the ALTER TABLEs. Idempotent: a duplicate-column error is + // expected and logged at trace level. + for sql in super::segments::SEGMENTS_MIGRATIONS_SQL { + match conn.execute(sql, []) { + Ok(_) => tracing::debug!("[segments:init] applied: {sql}"), + Err(e) => { + tracing::trace!("[segments:init] skipped (probably already exists): {e}") + } + } + } + // Event extraction tables. conn.execute_batch(super::events::EVENTS_INIT_SQL)?; @@ -295,3 +308,35 @@ impl UnifiedMemory { .join(Self::sanitize_namespace(namespace)) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::embeddings::NoopEmbedding; + use tempfile::TempDir; + + #[test] + fn sanitize_namespace_defaults_and_scrubs() { + assert_eq!(UnifiedMemory::sanitize_namespace(""), GLOBAL_NAMESPACE); + assert_eq!(UnifiedMemory::sanitize_namespace(" "), GLOBAL_NAMESPACE); + assert_eq!( + UnifiedMemory::sanitize_namespace("team alpha/#1"), + "team_alpha/_1" + ); + assert_eq!(UnifiedMemory::sanitize_namespace("a-b_c/ok"), "a-b_c/ok"); + } + + #[test] + fn namespace_dir_uses_sanitized_namespace() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + let dir = memory.namespace_dir("team alpha/#1"); + assert_eq!( + dir, + tmp.path() + .join("memory") + .join("namespaces") + .join("team_alpha/_1") + ); + } +} diff --git a/src/openhuman/memory/store/unified/mod.rs b/src/openhuman/memory_store/unified/mod.rs similarity index 99% rename from src/openhuman/memory/store/unified/mod.rs rename to src/openhuman/memory_store/unified/mod.rs index 5d292c03f..aa64afe66 100644 --- a/src/openhuman/memory/store/unified/mod.rs +++ b/src/openhuman/memory_store/unified/mod.rs @@ -26,7 +26,6 @@ pub mod fts5; mod graph; mod helpers; mod init; -mod kv; pub mod profile; mod query; pub mod segments; diff --git a/src/openhuman/memory/store/unified/profile.rs b/src/openhuman/memory_store/unified/profile.rs similarity index 100% rename from src/openhuman/memory/store/unified/profile.rs rename to src/openhuman/memory_store/unified/profile.rs diff --git a/src/openhuman/memory/store/unified/profile_tests.rs b/src/openhuman/memory_store/unified/profile_tests.rs similarity index 100% rename from src/openhuman/memory/store/unified/profile_tests.rs rename to src/openhuman/memory_store/unified/profile_tests.rs diff --git a/src/openhuman/memory/store/unified/query.rs b/src/openhuman/memory_store/unified/query.rs similarity index 99% rename from src/openhuman/memory/store/unified/query.rs rename to src/openhuman/memory_store/unified/query.rs index 9f9af8cf4..5164e249b 100644 --- a/src/openhuman/memory/store/unified/query.rs +++ b/src/openhuman/memory_store/unified/query.rs @@ -9,7 +9,7 @@ use rusqlite::params; use std::collections::{HashMap, HashSet}; -use crate::openhuman::memory::store::types::{ +use crate::openhuman::memory_store::types::{ GraphRelationRecord, MemoryItemKind, NamespaceMemoryHit, NamespaceQueryResult, NamespaceRetrievalContext, RetrievalScoreBreakdown, }; @@ -583,7 +583,7 @@ impl UnifiedMemory { fn build_retrieval_plan( &self, query: &str, - docs: &[crate::openhuman::memory::store::types::StoredMemoryDocument], + docs: &[crate::openhuman::memory_store::types::StoredMemoryDocument], graph_relations: &[GraphRelationRecord], ) -> RetrievalPlan { let query_terms = Self::tokenize_search_terms(query); @@ -615,7 +615,7 @@ impl UnifiedMemory { fn match_query_entities( &self, query: &str, - docs: &[crate::openhuman::memory::store::types::StoredMemoryDocument], + docs: &[crate::openhuman::memory_store::types::StoredMemoryDocument], graph_relations: &[GraphRelationRecord], ) -> Vec { let normalized_query = Self::normalize_search_text(query); @@ -907,7 +907,7 @@ impl UnifiedMemory { fn compute_graph_document_scores( &self, - docs: &[crate::openhuman::memory::store::types::StoredMemoryDocument], + docs: &[crate::openhuman::memory_store::types::StoredMemoryDocument], chunks: &[StoredChunk], relations: &[RelationMatch], ) -> HashMap { diff --git a/src/openhuman/memory/store/unified/query_tests.rs b/src/openhuman/memory_store/unified/query_tests.rs similarity index 98% rename from src/openhuman/memory/store/unified/query_tests.rs rename to src/openhuman/memory_store/unified/query_tests.rs index 21ac37a1f..2832911ca 100644 --- a/src/openhuman/memory/store/unified/query_tests.rs +++ b/src/openhuman/memory_store/unified/query_tests.rs @@ -147,7 +147,7 @@ async fn recall_namespace_memories_includes_namespace_kv() { #[tokio::test] async fn query_returns_episodic_hits_when_available() { - use crate::openhuman::memory::store::fts5::{self, EpisodicEntry}; + use crate::openhuman::memory_store::fts5::{self, EpisodicEntry}; let tmp = TempDir::new().unwrap(); let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); @@ -185,7 +185,7 @@ async fn query_returns_episodic_hits_when_available() { #[tokio::test] async fn query_returns_event_hits_when_available() { - use crate::openhuman::memory::store::events::{self, EventRecord, EventType}; + use crate::openhuman::memory_store::events::{self, EventRecord, EventType}; let tmp = TempDir::new().unwrap(); let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); @@ -227,7 +227,7 @@ async fn query_returns_event_hits_when_available() { #[tokio::test] async fn query_episodic_hits_have_correct_kind() { - use crate::openhuman::memory::store::fts5::{self, EpisodicEntry}; + use crate::openhuman::memory_store::fts5::{self, EpisodicEntry}; let tmp = TempDir::new().unwrap(); let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); diff --git a/src/openhuman/memory/store/unified/segments.rs b/src/openhuman/memory_store/unified/segments.rs similarity index 89% rename from src/openhuman/memory/store/unified/segments.rs rename to src/openhuman/memory_store/unified/segments.rs index 46e63ce10..8ad43d2b8 100644 --- a/src/openhuman/memory/store/unified/segments.rs +++ b/src/openhuman/memory_store/unified/segments.rs @@ -26,7 +26,13 @@ CREATE TABLE IF NOT EXISTS conversation_segments ( topic_keywords TEXT, status TEXT NOT NULL DEFAULT 'open', created_at REAL NOT NULL, - updated_at REAL NOT NULL + updated_at REAL NOT NULL, + -- Per-session sequence numbers from memory_archivist::store, populated + -- alongside start_episodic_id / end_episodic_id during the FTS5 -> md + -- migration. Once STM recall switches its segment-span dedup to use + -- (session_id, seq) the legacy episodic_id columns can be dropped. + start_seq INTEGER, + end_seq INTEGER ); CREATE INDEX IF NOT EXISTS idx_segments_session @@ -102,8 +108,26 @@ pub struct ConversationSegment { pub status: SegmentStatus, pub created_at: f64, pub updated_at: f64, + /// Per-session seq number assigned by `memory_archivist::store::record_turn` + /// for the user turn that opened this segment. `None` on legacy rows + /// written before the FTS5 -> md migration began. + #[serde(default)] + pub start_seq: Option, + /// Per-session seq for the latest turn appended to this segment. + /// `None` while the segment has no appended turns OR on legacy rows. + #[serde(default)] + pub end_seq: Option, } +/// Idempotent migrations applied alongside [`SEGMENTS_INIT_SQL`] for +/// databases created before the `(start_seq, end_seq)` columns existed. +/// Each statement either applies cleanly or fails with "duplicate column", +/// both of which are safe to swallow. +pub const SEGMENTS_MIGRATIONS_SQL: &[&str] = &[ + "ALTER TABLE conversation_segments ADD COLUMN start_seq INTEGER", + "ALTER TABLE conversation_segments ADD COLUMN end_seq INTEGER", +]; + /// Boundary detection configuration. #[derive(Debug, Clone)] pub struct BoundaryConfig { @@ -181,20 +205,22 @@ pub fn segment_create( session_id: &str, namespace: &str, start_episodic_id: i64, + start_seq: Option, start_timestamp: f64, now: f64, ) -> anyhow::Result<()> { let conn = conn.lock(); conn.execute( "INSERT INTO conversation_segments - (segment_id, session_id, namespace, start_episodic_id, start_timestamp, - turn_count, status, created_at, updated_at) - VALUES (?1, ?2, ?3, ?4, ?5, 1, 'open', ?6, ?6)", + (segment_id, session_id, namespace, start_episodic_id, start_seq, + start_timestamp, turn_count, status, created_at, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, 1, 'open', ?7, ?7)", params![ segment_id, session_id, namespace, start_episodic_id, + start_seq, start_timestamp, now ], @@ -203,11 +229,12 @@ pub fn segment_create( Ok(()) } -/// Increment turn count and update the latest episodic ID / timestamp. +/// Increment turn count and update the latest episodic ID + seq + timestamp. pub fn segment_append_turn( conn: &Arc>, segment_id: &str, episodic_id: i64, + end_seq: Option, timestamp: f64, now: f64, ) -> anyhow::Result<()> { @@ -216,10 +243,11 @@ pub fn segment_append_turn( "UPDATE conversation_segments SET turn_count = turn_count + 1, end_episodic_id = ?2, - end_timestamp = ?3, - updated_at = ?4 + end_seq = ?3, + end_timestamp = ?4, + updated_at = ?5 WHERE segment_id = ?1", - params![segment_id, episodic_id, timestamp, now], + params![segment_id, episodic_id, end_seq, timestamp, now], )?; Ok(()) } @@ -348,7 +376,8 @@ pub fn open_segment_for_session( .query_row( "SELECT segment_id, session_id, namespace, start_episodic_id, end_episodic_id, start_timestamp, end_timestamp, turn_count, summary, embedding, - topic_keywords, status, created_at, updated_at + topic_keywords, status, created_at, updated_at, + start_seq, end_seq FROM conversation_segments WHERE session_id = ?1 AND status = 'open' ORDER BY created_at DESC @@ -370,7 +399,8 @@ pub fn segments_by_namespace( let mut stmt = conn.prepare( "SELECT segment_id, session_id, namespace, start_episodic_id, end_episodic_id, start_timestamp, end_timestamp, turn_count, summary, embedding, - topic_keywords, status, created_at, updated_at + topic_keywords, status, created_at, updated_at, + start_seq, end_seq FROM conversation_segments WHERE namespace = ?1 ORDER BY updated_at DESC @@ -392,7 +422,8 @@ pub fn segment_get( .query_row( "SELECT segment_id, session_id, namespace, start_episodic_id, end_episodic_id, start_timestamp, end_timestamp, turn_count, summary, embedding, - topic_keywords, status, created_at, updated_at + topic_keywords, status, created_at, updated_at, + start_seq, end_seq FROM conversation_segments WHERE segment_id = ?1", params![segment_id], @@ -411,7 +442,8 @@ pub fn segments_pending_summary( let mut stmt = conn.prepare( "SELECT segment_id, session_id, namespace, start_episodic_id, end_episodic_id, start_timestamp, end_timestamp, turn_count, summary, embedding, - topic_keywords, status, created_at, updated_at + topic_keywords, status, created_at, updated_at, + start_seq, end_seq FROM conversation_segments WHERE status = 'closed' ORDER BY created_at ASC @@ -536,6 +568,8 @@ fn row_to_segment(row: &rusqlite::Row<'_>) -> rusqlite::Result>(14)?.map(|v| v.max(0) as u32), + end_seq: row.get::<_, Option>(15)?.map(|v| v.max(0) as u32), }) } diff --git a/src/openhuman/memory/store/unified/segments_tests.rs b/src/openhuman/memory_store/unified/segments_tests.rs similarity index 86% rename from src/openhuman/memory/store/unified/segments_tests.rs rename to src/openhuman/memory_store/unified/segments_tests.rs index 468439fc4..ae7a640ad 100644 --- a/src/openhuman/memory/store/unified/segments_tests.rs +++ b/src/openhuman/memory_store/unified/segments_tests.rs @@ -14,7 +14,7 @@ fn setup_db() -> Arc> { #[test] fn create_and_get_segment() { let conn = setup_db(); - segment_create(&conn, "seg-1", "s1", "global", 1, 1000.0, 1000.0).unwrap(); + segment_create(&conn, "seg-1", "s1", "global", 1, None, 1000.0, 1000.0).unwrap(); let seg = segment_get(&conn, "seg-1").unwrap().unwrap(); assert_eq!(seg.session_id, "s1"); assert_eq!(seg.turn_count, 1); @@ -24,7 +24,7 @@ fn create_and_get_segment() { #[test] fn segment_embeddings_are_scoped_by_model_signature() { let conn = setup_db(); - segment_create(&conn, "seg-embed", "s1", "global", 1, 1000.0, 1000.0).unwrap(); + segment_create(&conn, "seg-embed", "s1", "global", 1, None, 1000.0, 1000.0).unwrap(); segment_embedding_upsert( &conn, @@ -62,9 +62,9 @@ fn segment_embeddings_are_scoped_by_model_signature() { #[test] fn append_and_close_segment() { let conn = setup_db(); - segment_create(&conn, "seg-2", "s1", "global", 1, 1000.0, 1000.0).unwrap(); - segment_append_turn(&conn, "seg-2", 2, 1005.0, 1005.0).unwrap(); - segment_append_turn(&conn, "seg-2", 3, 1010.0, 1010.0).unwrap(); + segment_create(&conn, "seg-2", "s1", "global", 1, None, 1000.0, 1000.0).unwrap(); + segment_append_turn(&conn, "seg-2", 2, None, 1005.0, 1005.0).unwrap(); + segment_append_turn(&conn, "seg-2", 3, None, 1010.0, 1010.0).unwrap(); let seg = segment_get(&conn, "seg-2").unwrap().unwrap(); assert_eq!(seg.turn_count, 3); @@ -78,9 +78,9 @@ fn append_and_close_segment() { #[test] fn open_segment_for_session_returns_latest() { let conn = setup_db(); - segment_create(&conn, "seg-a", "s1", "global", 1, 1000.0, 1000.0).unwrap(); + segment_create(&conn, "seg-a", "s1", "global", 1, None, 1000.0, 1000.0).unwrap(); segment_close(&conn, "seg-a", 1001.0).unwrap(); - segment_create(&conn, "seg-b", "s1", "global", 5, 1010.0, 1010.0).unwrap(); + segment_create(&conn, "seg-b", "s1", "global", 5, None, 1010.0, 1010.0).unwrap(); let open = open_segment_for_session(&conn, "s1").unwrap(); assert!(open.is_some()); @@ -109,6 +109,8 @@ fn boundary_detection_time_gap() { status: SegmentStatus::Open, created_at: 1000.0, updated_at: 1050.0, + start_seq: None, + end_seq: None, }; // Within time gap — continue. @@ -141,6 +143,8 @@ fn boundary_detection_explicit_marker() { status: SegmentStatus::Open, created_at: 1000.0, updated_at: 1000.0, + start_seq: None, + end_seq: None, }; let decision = detect_boundary( @@ -177,6 +181,8 @@ fn boundary_detection_turn_count() { status: SegmentStatus::Open, created_at: 1000.0, updated_at: 1010.0, + start_seq: None, + end_seq: None, }; let decision = detect_boundary(&config, &seg, 1011.0, "next", None); @@ -204,6 +210,8 @@ fn boundary_detection_embedding_drift() { status: SegmentStatus::Open, created_at: 1000.0, updated_at: 1000.0, + start_seq: None, + end_seq: None, }; // Similar direction — continue. @@ -231,7 +239,7 @@ fn incremental_mean_embedding_works() { #[test] fn summary_set_and_read() { let conn = setup_db(); - segment_create(&conn, "seg-s", "s1", "global", 1, 1000.0, 1000.0).unwrap(); + segment_create(&conn, "seg-s", "s1", "global", 1, None, 1000.0, 1000.0).unwrap(); segment_close(&conn, "seg-s", 1001.0).unwrap(); segment_set_summary(&conn, "seg-s", "Discussed deployment strategy", 1002.0).unwrap(); let seg = segment_get(&conn, "seg-s").unwrap().unwrap(); @@ -246,9 +254,9 @@ fn summary_set_and_read() { fn segments_by_namespace_returns_most_recent_first() { let conn = setup_db(); // Create three segments with different updated_at timestamps. - segment_create(&conn, "seg-ns-1", "s1", "myns", 1, 1000.0, 1000.0).unwrap(); - segment_create(&conn, "seg-ns-2", "s1", "myns", 5, 2000.0, 2000.0).unwrap(); - segment_create(&conn, "seg-ns-3", "s1", "myns", 10, 3000.0, 3000.0).unwrap(); + segment_create(&conn, "seg-ns-1", "s1", "myns", 1, None, 1000.0, 1000.0).unwrap(); + segment_create(&conn, "seg-ns-2", "s1", "myns", 5, None, 2000.0, 2000.0).unwrap(); + segment_create(&conn, "seg-ns-3", "s1", "myns", 10, None, 3000.0, 3000.0).unwrap(); // Append a turn to seg-ns-1 with a later timestamp to bump its updated_at. // Leave seg-ns-3 as the most recently created (highest updated_at). @@ -261,7 +269,7 @@ fn segments_by_namespace_returns_most_recent_first() { assert_eq!(segs[2].segment_id, "seg-ns-1"); // Bump seg-ns-1's updated_at by appending a turn. - segment_append_turn(&conn, "seg-ns-1", 2, 9000.0, 9000.0).unwrap(); + segment_append_turn(&conn, "seg-ns-1", 2, None, 9000.0, 9000.0).unwrap(); let segs = segments_by_namespace(&conn, "myns", 10).unwrap(); assert_eq!(segs[0].segment_id, "seg-ns-1"); } @@ -270,14 +278,14 @@ fn segments_by_namespace_returns_most_recent_first() { fn segments_pending_summary_only_returns_closed() { let conn = setup_db(); // Open segment — should NOT appear. - segment_create(&conn, "seg-open", "s1", "global", 1, 1000.0, 1000.0).unwrap(); + segment_create(&conn, "seg-open", "s1", "global", 1, None, 1000.0, 1000.0).unwrap(); // Closed segment — SHOULD appear. - segment_create(&conn, "seg-closed", "s2", "global", 5, 2000.0, 2000.0).unwrap(); + segment_create(&conn, "seg-closed", "s2", "global", 5, None, 2000.0, 2000.0).unwrap(); segment_close(&conn, "seg-closed", 2001.0).unwrap(); // Summarised segment — should NOT appear (only status='closed' is pending). - segment_create(&conn, "seg-summ", "s3", "global", 10, 3000.0, 3000.0).unwrap(); + segment_create(&conn, "seg-summ", "s3", "global", 10, None, 3000.0, 3000.0).unwrap(); segment_close(&conn, "seg-summ", 3001.0).unwrap(); segment_set_summary(&conn, "seg-summ", "A summary", 3002.0).unwrap(); @@ -294,7 +302,7 @@ fn segments_pending_summary_only_returns_closed() { #[test] fn segment_set_embedding_roundtrip() { let conn = setup_db(); - segment_create(&conn, "seg-emb", "s1", "global", 1, 1000.0, 1000.0).unwrap(); + segment_create(&conn, "seg-emb", "s1", "global", 1, None, 1000.0, 1000.0).unwrap(); let embedding = vec![0.1_f32, 0.2, 0.3, 0.4, 0.5]; segment_set_embedding(&conn, "seg-emb", &embedding, 1001.0).unwrap(); @@ -313,7 +321,7 @@ fn segment_set_embedding_roundtrip() { #[test] fn segment_set_keywords_stores_and_reads() { let conn = setup_db(); - segment_create(&conn, "seg-kw", "s1", "global", 1, 1000.0, 1000.0).unwrap(); + segment_create(&conn, "seg-kw", "s1", "global", 1, None, 1000.0, 1000.0).unwrap(); let keywords = "rust,memory,performance"; segment_set_keywords(&conn, "seg-kw", keywords, 1001.0).unwrap(); @@ -344,6 +352,8 @@ fn boundary_no_false_positive_on_short_messages() { status: SegmentStatus::Open, created_at: 1000.0, updated_at: 1010.0, + start_seq: None, + end_seq: None, }; // Short single-word messages must not trigger explicit marker detection. diff --git a/src/openhuman/memory_store/vectors/mod.rs b/src/openhuman/memory_store/vectors/mod.rs new file mode 100644 index 000000000..cb65c42a8 --- /dev/null +++ b/src/openhuman/memory_store/vectors/mod.rs @@ -0,0 +1,8 @@ +//! Local vector store (VectorStore) — moved from embeddings::store. +//! +//! Previously at `embeddings::store`. Moved here as part of the +//! memory_store consolidation to co-locate all persistence with memory_store. + +pub mod store; + +pub use store::{bytes_to_vec, cosine_similarity, vec_to_bytes, SearchResult, VectorStore}; diff --git a/src/openhuman/embeddings/store.rs b/src/openhuman/memory_store/vectors/store.rs similarity index 99% rename from src/openhuman/embeddings/store.rs rename to src/openhuman/memory_store/vectors/store.rs index 873ba997f..90fd8594d 100644 --- a/src/openhuman/embeddings/store.rs +++ b/src/openhuman/memory_store/vectors/store.rs @@ -21,7 +21,7 @@ use std::sync::Arc; use parking_lot::Mutex; use rusqlite::Connection; -use super::EmbeddingProvider; +use crate::openhuman::embeddings::EmbeddingProvider; /// SQL to create the vector store schema. const INIT_SQL: &str = " diff --git a/src/openhuman/embeddings/store_tests.rs b/src/openhuman/memory_store/vectors/store_tests.rs similarity index 99% rename from src/openhuman/embeddings/store_tests.rs rename to src/openhuman/memory_store/vectors/store_tests.rs index f22ba3d93..c5b9e9af6 100644 --- a/src/openhuman/embeddings/store_tests.rs +++ b/src/openhuman/memory_store/vectors/store_tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::openhuman::embeddings::EmbeddingProvider; use serde_json::json; /// A test embedding provider that returns deterministic vectors. diff --git a/src/openhuman/memory_sync/README.md b/src/openhuman/memory_sync/README.md new file mode 100644 index 000000000..0e1930bdb --- /dev/null +++ b/src/openhuman/memory_sync/README.md @@ -0,0 +1,59 @@ +# memory_sync + +Every "pull data from upstream → land it in memory_store" pipeline in +one place, organised by the kind of upstream they talk to. + +## Three pipeline kinds + +| Kind | Submodule | Owns | +| --- | --- | --- | +| **Composio** | [`composio/`](composio/) | Per-provider sync via the Composio Edge API: gmail, slack, github, notion, linear, clickup, … | +| **Workspace** | [`workspace/`](workspace/) | Vault file watch, harness turn capture, dictation transcripts — anything local. | +| **MCP** | [`mcp/`](mcp/) | Third-party MCP servers via `mcp_clients/` transport. | + +## Trait + +Every pipeline implements [`SyncPipeline`]: + +```rust +async fn init(&self, &Config) -> anyhow::Result<()>; +async fn tick(&self, &Config) -> anyhow::Result; +fn id(&self) -> &str; +fn kind(&self) -> SyncPipelineKind; +``` + +`SyncOutcome { records_ingested, more_pending, note }` is the +orchestrator-facing result; pipelines own their own pagination cursors +and retry policy behind that. + +## Layout + +| Path | Role | +| --- | --- | +| [`mod.rs`](mod.rs) | Module root + re-exports. | +| [`traits.rs`](traits.rs) | `SyncPipeline`, `SyncOutcome`, `SyncPipelineKind`. | +| [`composio/`](composio/) | Per-provider pipelines (gmail, slack, github, notion, linear, clickup). | +| [`workspace/`](workspace/) | Vault, harness, dictation pipelines. | +| [`mcp/`](mcp/) | MCP-server pipelines (one per connected server). | + +## Status + +**Scaffold only.** Today's sync code still lives in: + +- `composio/providers//ingest.rs` + `bin/{slack_backfill,gmail_backfill_3d}.rs` +- `vault/sync.rs`, `agent_experience/`, `dictation_hotkeys/` +- `mcp_clients/` (transport only; no drain loop yet) + +Each migrates here as its own per-pipeline PR. The job-queue orchestration +in `memory::jobs` stays put — it just gains the ability to iterate over a +registered `Vec>`. + +## Layer rules + +- Sync writes go through `memory::ingest_pipeline` so every record + lands as raw md → chunks → tree leaves like any other ingest. +- No direct writes into trees or unified. No upstream-specific data + models leak past the pipeline boundary. +- One pipeline per upstream service. Composio's GitHub and MCP's GitHub + are distinct pipelines because they hit different surfaces with + different cadence and auth. diff --git a/src/openhuman/memory_tree/canonicalize/README.md b/src/openhuman/memory_sync/canonicalize/README.md similarity index 100% rename from src/openhuman/memory_tree/canonicalize/README.md rename to src/openhuman/memory_sync/canonicalize/README.md diff --git a/src/openhuman/memory_tree/canonicalize/chat.rs b/src/openhuman/memory_sync/canonicalize/chat.rs similarity index 98% rename from src/openhuman/memory_tree/canonicalize/chat.rs rename to src/openhuman/memory_sync/canonicalize/chat.rs index 42f8037db..c8ce37d19 100644 --- a/src/openhuman/memory_tree/canonicalize/chat.rs +++ b/src/openhuman/memory_sync/canonicalize/chat.rs @@ -18,7 +18,7 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use super::{normalize_source_ref, CanonicalisedSource}; -use crate::openhuman::memory_tree::types::{Metadata, SourceKind}; +use crate::openhuman::memory_store::chunks::types::{Metadata, SourceKind}; /// One chat message in a channel/group. #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/src/openhuman/memory_tree/canonicalize/document.rs b/src/openhuman/memory_sync/canonicalize/document.rs similarity index 99% rename from src/openhuman/memory_tree/canonicalize/document.rs rename to src/openhuman/memory_sync/canonicalize/document.rs index 3ea7fe642..a3ee1e926 100644 --- a/src/openhuman/memory_tree/canonicalize/document.rs +++ b/src/openhuman/memory_sync/canonicalize/document.rs @@ -9,7 +9,7 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Deserializer, Serialize}; use super::{normalize_source_ref, CanonicalisedSource}; -use crate::openhuman::memory_tree::types::{Metadata, SourceKind}; +use crate::openhuman::memory_store::chunks::types::{Metadata, SourceKind}; // ── Serde helpers ───────────────────────────────────────────────────────────── diff --git a/src/openhuman/memory_tree/canonicalize/email.rs b/src/openhuman/memory_sync/canonicalize/email.rs similarity index 99% rename from src/openhuman/memory_tree/canonicalize/email.rs rename to src/openhuman/memory_sync/canonicalize/email.rs index 7dc123e88..1fec42635 100644 --- a/src/openhuman/memory_tree/canonicalize/email.rs +++ b/src/openhuman/memory_sync/canonicalize/email.rs @@ -11,7 +11,7 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use super::{email_clean, normalize_source_ref, CanonicalisedSource}; -use crate::openhuman::memory_tree::types::{Metadata, SourceKind}; +use crate::openhuman::memory_store::chunks::types::{Metadata, SourceKind}; /// One email in a thread. #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/src/openhuman/memory_tree/canonicalize/email_clean.rs b/src/openhuman/memory_sync/canonicalize/email_clean.rs similarity index 100% rename from src/openhuman/memory_tree/canonicalize/email_clean.rs rename to src/openhuman/memory_sync/canonicalize/email_clean.rs diff --git a/src/openhuman/memory_tree/canonicalize/mod.rs b/src/openhuman/memory_sync/canonicalize/mod.rs similarity index 96% rename from src/openhuman/memory_tree/canonicalize/mod.rs rename to src/openhuman/memory_sync/canonicalize/mod.rs index c515678d1..7ef754939 100644 --- a/src/openhuman/memory_tree/canonicalize/mod.rs +++ b/src/openhuman/memory_sync/canonicalize/mod.rs @@ -16,7 +16,7 @@ pub mod email_clean; use serde::{Deserialize, Serialize}; -use crate::openhuman::memory_tree::types::{Metadata, SourceRef}; +use crate::openhuman::memory_store::chunks::types::{Metadata, SourceRef}; /// Output of a canonicaliser — one per logical source record /// (a chat batch, an email, a document). diff --git a/src/openhuman/memory_sync/composio/mod.rs b/src/openhuman/memory_sync/composio/mod.rs new file mode 100644 index 000000000..af040a48d --- /dev/null +++ b/src/openhuman/memory_sync/composio/mod.rs @@ -0,0 +1,29 @@ +//! Composio-backed sync pipelines. +//! +//! New home for the per-provider sync code that lives across +//! `composio/providers/{gmail,slack,github,notion,linear,clickup,...}/`. +//! Each provider gets a submodule here whose job is to: +//! +//! 1. Resolve the user's Composio connection for the provider. +//! 2. Paginate through the provider's upstream surface (messages, +//! issues, docs, …). +//! 3. Hand each record to `memory::ingest_pipeline` so it lands as raw +//! md → chunks → tree leaves like any other ingest. +//! +//! ## Status +//! +//! Scaffold only. The actual per-provider sync code still lives under +//! `composio/providers/*/ingest.rs` and is invoked from +//! `bin/slack_backfill.rs` / `bin/gmail_backfill_3d.rs`. Migration plan +//! is a per-provider PR per submodule below. +//! +//! ## Provider submodules (planned) +//! +//! | Submodule | Source | Notes | +//! | --- | --- | --- | +//! | `gmail` | `composio/providers/gmail/ingest.rs` | Backfill + incremental | +//! | `slack` | `composio/providers/slack/ingest.rs` | Channel + DM | +//! | `github` | `composio/providers/github/` | Issues + PRs + comments | +//! | `notion` | `composio/providers/notion/` | Pages + databases | +//! | `linear` | `composio/providers/linear/` | Issues + comments | +//! | `clickup` | `composio/providers/clickup/` | Tasks + comments | diff --git a/src/openhuman/memory_sync/mcp/mod.rs b/src/openhuman/memory_sync/mcp/mod.rs new file mode 100644 index 000000000..d75e39bdc --- /dev/null +++ b/src/openhuman/memory_sync/mcp/mod.rs @@ -0,0 +1,18 @@ +//! Third-party MCP-server sync pipelines. +//! +//! Pipelines that pull from MCP (Model Context Protocol) servers the user +//! has connected. One pipeline per server. +//! +//! ## Layer rules +//! +//! - Transport (stdio / SSE / websocket) is owned by `mcp_clients/`; sync +//! here calls into that surface, never re-implements it. +//! - Data shapes are MCP-generic — the pipeline normalises into raw md +//! per record so the rest of memory_store doesn't have to know about +//! MCP at all. +//! +//! ## Status +//! +//! Scaffold only. The existing `mcp_clients/` module already knows how +//! to talk to a server; what's missing is the "drain new records since +//! last cursor and ingest" loop on top. diff --git a/src/openhuman/memory_sync/mod.rs b/src/openhuman/memory_sync/mod.rs new file mode 100644 index 000000000..e08d7dca0 --- /dev/null +++ b/src/openhuman/memory_sync/mod.rs @@ -0,0 +1,48 @@ +//! Memory sync pipelines. +//! +//! One top-level module hosting every "pull data from upstream → land it +//! in memory_store" pipeline, organised by the kind of upstream it talks +//! to. Three kinds today: +//! +//! - [`composio`] — Composio managed connectors (Gmail, Slack, GitHub, +//! Notion, Linear, ClickUp, …). Pulls via the Composio Edge API. +//! - [`workspace`] — Local workspace connectors (filesystem vault sync, +//! local-only ingest, agent-experience capture from the harness). +//! - [`mcp`] — Third-party MCP servers. Pulls via the MCP protocol over +//! stdio/SSE. +//! +//! All three implement the [`SyncPipeline`] trait so the orchestrator +//! (`memory::jobs`) can drive them uniformly: `init` → `tick` → repeat. +//! +//! ## Layer rules +//! +//! - Sync writes into `memory_store` only — never directly into trees, +//! never directly into unified. The ingest pipeline in +//! `memory::ingest_pipeline` is the seam. +//! - One pipeline per upstream service. Composio's GitHub and MCP's +//! GitHub are distinct pipelines because they hit different surfaces +//! with different cadence and auth. +//! - Pipeline modules own their own types, their own state, and their +//! own retry/backoff policy. The trait gives the orchestrator a +//! single shape to call; everything else stays local. + +pub mod canonicalize; +pub mod composio; +pub mod mcp; +pub mod sync_status; +pub mod traits; +pub mod workspace; + +pub use traits::{SyncOutcome, SyncPipeline, SyncPipelineKind}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reexports_sync_pipeline_kind_labels() { + assert_eq!(SyncPipelineKind::Composio.as_str(), "composio"); + assert_eq!(SyncPipelineKind::Workspace.as_str(), "workspace"); + assert_eq!(SyncPipelineKind::Mcp.as_str(), "mcp"); + } +} diff --git a/src/openhuman/memory/sync_status/mod.rs b/src/openhuman/memory_sync/sync_status/mod.rs similarity index 100% rename from src/openhuman/memory/sync_status/mod.rs rename to src/openhuman/memory_sync/sync_status/mod.rs diff --git a/src/openhuman/memory/sync_status/rpc.rs b/src/openhuman/memory_sync/sync_status/rpc.rs similarity index 98% rename from src/openhuman/memory/sync_status/rpc.rs rename to src/openhuman/memory_sync/sync_status/rpc.rs index 52ed58355..945118d7a 100644 --- a/src/openhuman/memory/sync_status/rpc.rs +++ b/src/openhuman/memory_sync/sync_status/rpc.rs @@ -44,7 +44,7 @@ //! progress signal. use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::store::with_connection; +use crate::openhuman::memory_store::chunks::store::with_connection; use crate::rpc::RpcOutcome; use rusqlite::Connection; @@ -247,7 +247,7 @@ mod tests { /// column is NULL. #[test] fn pending_and_processed_key_off_sidecar_not_inline_column() { - use crate::openhuman::memory_tree::store::with_connection; + use crate::openhuman::memory_store::chunks::store::with_connection; use rusqlite::params; use tempfile::TempDir; @@ -311,7 +311,7 @@ mod tests { /// hides the progress bar): `batch_total = batch_processed = 0`. #[test] fn fully_embedded_provider_reports_no_active_wave() { - use crate::openhuman::memory_tree::store::with_connection; + use crate::openhuman::memory_store::chunks::store::with_connection; use rusqlite::params; use tempfile::TempDir; @@ -357,7 +357,7 @@ mod tests { /// provider whose only leftovers are terminal drains to 0 pending / no wave. #[test] fn dropped_and_skipped_chunks_count_as_resolved_not_pending() { - use crate::openhuman::memory_tree::store::with_connection; + use crate::openhuman::memory_store::chunks::store::with_connection; use rusqlite::params; use tempfile::TempDir; @@ -422,7 +422,7 @@ mod tests { /// still reflects the old straggler. #[test] fn stale_out_of_window_pending_does_not_open_a_wave() { - use crate::openhuman::memory_tree::store::with_connection; + use crate::openhuman::memory_store::chunks::store::with_connection; use rusqlite::params; use tempfile::TempDir; diff --git a/src/openhuman/memory/sync_status/schemas.rs b/src/openhuman/memory_sync/sync_status/schemas.rs similarity index 100% rename from src/openhuman/memory/sync_status/schemas.rs rename to src/openhuman/memory_sync/sync_status/schemas.rs diff --git a/src/openhuman/memory/sync_status/types.rs b/src/openhuman/memory_sync/sync_status/types.rs similarity index 100% rename from src/openhuman/memory/sync_status/types.rs rename to src/openhuman/memory_sync/sync_status/types.rs diff --git a/src/openhuman/memory_sync/traits.rs b/src/openhuman/memory_sync/traits.rs new file mode 100644 index 000000000..eb3c5c9c1 --- /dev/null +++ b/src/openhuman/memory_sync/traits.rs @@ -0,0 +1,93 @@ +//! Shared sync-pipeline trait. + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; + +use crate::openhuman::config::Config; + +/// The three flavors of sync pipeline. Knowing the kind at the orchestrator +/// is useful for surfaces like status dashboards and rate-limit budgeting. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SyncPipelineKind { + Composio, + Workspace, + Mcp, +} + +impl SyncPipelineKind { + pub fn as_str(self) -> &'static str { + match self { + SyncPipelineKind::Composio => "composio", + SyncPipelineKind::Workspace => "workspace", + SyncPipelineKind::Mcp => "mcp", + } + } +} + +/// Result of one sync tick — minimal enough that every pipeline can fill +/// it in. Detailed per-pipeline progress lives behind the pipeline's own +/// status surface. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct SyncOutcome { + /// How many upstream records were ingested into memory_store during + /// this tick. May be 0 when nothing new arrived. + pub records_ingested: u32, + /// `true` when the pipeline thinks there is more to fetch and the + /// orchestrator should tick again soon. + pub more_pending: bool, + /// Free-form note for logs / status UIs. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub note: Option, +} + +/// Contract every sync pipeline implements. Lifecycle: `init` exactly +/// once when the pipeline first comes up, then `tick` on a cadence the +/// orchestrator picks. +#[async_trait] +pub trait SyncPipeline: Send + Sync { + /// Stable identifier for the pipeline — e.g. `"composio:gmail"`, + /// `"workspace:vault"`, `"mcp:filesystem"`. Used as the key in + /// status surfaces and the job-queue. + fn id(&self) -> &str; + + /// Which kind of pipeline this is. + fn kind(&self) -> SyncPipelineKind; + + /// Cold-start work. Idempotent — the orchestrator may call it on + /// every process boot. + async fn init(&self, config: &Config) -> anyhow::Result<()>; + + /// Pull one batch from upstream and land it in memory_store. Pipeline + /// owns its own pagination / cursor state. + async fn tick(&self, config: &Config) -> anyhow::Result; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sync_pipeline_kind_as_str_matches_serde_names() { + let cases = [ + (SyncPipelineKind::Composio, "composio"), + (SyncPipelineKind::Workspace, "workspace"), + (SyncPipelineKind::Mcp, "mcp"), + ]; + for (kind, label) in cases { + assert_eq!(kind.as_str(), label); + let json = serde_json::to_string(&kind).unwrap(); + assert_eq!(json, format!("\"{label}\"")); + let decoded: SyncPipelineKind = serde_json::from_str(&json).unwrap(); + assert_eq!(decoded, kind); + } + } + + #[test] + fn sync_outcome_default_is_empty_and_not_pending() { + let outcome = SyncOutcome::default(); + assert_eq!(outcome.records_ingested, 0); + assert!(!outcome.more_pending); + assert!(outcome.note.is_none()); + } +} diff --git a/src/openhuman/memory_sync/workspace/mod.rs b/src/openhuman/memory_sync/workspace/mod.rs new file mode 100644 index 000000000..e3e3359c5 --- /dev/null +++ b/src/openhuman/memory_sync/workspace/mod.rs @@ -0,0 +1,17 @@ +//! Workspace-scoped sync pipelines. +//! +//! Pipelines that pull from sources local to the user's workspace rather +//! than third-party services. Three flavors expected: +//! +//! | Submodule | Source | Notes | +//! | --- | --- | --- | +//! | `vault` | Files dropped into the Obsidian vault by the user | Watch + diff | +//! | `harness` | Agent harness turns (memory_archivist's caller side) | Push-based | +//! | `dictation` | Local audio capture transcripts | Push-based | +//! +//! ## Status +//! +//! Scaffold only. Today the vault watch lives in `vault/sync.rs`, +//! harness capture in `agent_experience/`, and dictation in +//! `dictation_hotkeys/`. Each will land here as a [`SyncPipeline`] impl +//! in a follow-up. diff --git a/src/openhuman/memory_tools/README.md b/src/openhuman/memory_tools/README.md new file mode 100644 index 000000000..3a6e61dbf --- /dev/null +++ b/src/openhuman/memory_tools/README.md @@ -0,0 +1,41 @@ +# memory_tools + +Tool-scoped memory: durable rules / learnings keyed per tool name. Distinct +from generic namespace memory and from `learning::tool_tracker` statistics. + +## Namespace convention + +Each tool gets its own namespace `tool-{tool_name}`. Build the string via +[`types::tool_memory_namespace`] — never hard-code it. + +## Layout + +| Path | Role | +| --- | --- | +| [`mod.rs`](mod.rs) | Module root + public re-exports. | +| [`types.rs`](types.rs) | `ToolMemoryRule` (id, tool_name, rule text, priority, source, tags, created_at, updated_at) + `ToolMemoryPriority` (Normal / High / Critical) + `ToolMemorySource` (UserExplicit / PostTurn / Programmatic) + `tool_memory_namespace(tool_name)`. | +| [`store.rs`](store.rs) | `ToolMemoryStore` over `Arc`: `put_rule`, `get_rule`, `list_rules`, `delete_rule`, `rules_for_prompt`, `list_tool_names`, `record`, `list_rules_json`. | +| [`store_tests.rs`](store_tests.rs) | Store coverage against the `MockMemory` from `test_helpers`. | +| [`capture.rs`](capture.rs) | `ToolMemoryCaptureHook` — `PostTurnHook` impl that captures user edicts and repeated tool failures into the store. | +| [`prompt.rs`](prompt.rs) | `ToolMemoryRulesSection` + `render_tool_memory_rules` — prompt section that pins Critical / High rules into the system prompt so they survive compression. `TOOL_MEMORY_HEADING` + `TOOL_MEMORY_PROMPT_CAP` constants. | +| [`tools/`](tools/) | Agent-facing read/write tools: `MemoryToolsListTool` (list rules for a tool), `MemoryToolsPutTool` (upsert a rule). | +| [`test_helpers.rs`](test_helpers.rs) | `#[cfg(test)]` `MockMemory` used by `store_tests` + `capture::tests`. | + +## How it fits + +The agent harness: +1. **Reads** at session build — `ToolMemoryRulesSection::render` walks every + `tool-*` namespace and pins Critical/High rules into the system prompt. +2. **Writes** at turn end — `ToolMemoryCaptureHook` parses the user message + for edicts (`"never do X"`, `"always Y"`, …) and inserts rules. +3. **Direct read/write** — `tools::MemoryTools{List,Put}Tool` let the agent + itself inspect / record rules mid-session. + +## Layer rules + +- No upward dependencies — only `memory::Memory` trait (via `Arc`) + and project-wide primitives (`tools::traits::Tool`, `serde_json`). +- `MockMemory` is `#[cfg(test)]`-only — never available outside test builds. +- Re-exports in `mod.rs` are the public surface; the underlying submodules + are `pub` so test code can reach in but consumers should go through the + re-exports. diff --git a/src/openhuman/memory/tool_memory/capture.rs b/src/openhuman/memory_tools/capture.rs similarity index 100% rename from src/openhuman/memory/tool_memory/capture.rs rename to src/openhuman/memory_tools/capture.rs diff --git a/src/openhuman/memory/tool_memory/mod.rs b/src/openhuman/memory_tools/mod.rs similarity index 82% rename from src/openhuman/memory/tool_memory/mod.rs rename to src/openhuman/memory_tools/mod.rs index 02845c3ed..2750e963d 100644 --- a/src/openhuman/memory/tool_memory/mod.rs +++ b/src/openhuman/memory_tools/mod.rs @@ -18,16 +18,18 @@ //! //! ## Components //! -//! - [`types`] — [`ToolMemoryRule`], [`ToolMemoryPriority`], +//! - [`types`] — [`ToolMemoryRule`], [`ToolMemoryPriority`], //! [`ToolMemorySource`]. -//! - [`store`] — [`ToolMemoryStore`], the put/list/delete/prompt API +//! - [`store`] — [`ToolMemoryStore`], the put/list/delete/prompt API //! built on top of an `Arc`. //! - [`capture`] — [`ToolMemoryCaptureHook`], the post-turn //! [`PostTurnHook`] that records user edicts and repeated tool //! failures. -//! - [`prompt`] — [`ToolMemoryRulesSection`], the prompt section that +//! - [`prompt`] — [`ToolMemoryRulesSection`], the prompt section that //! pins Critical / High rules into the system prompt so they survive //! mid-session compression. +//! - [`tools`] — agent-facing read/write tools: +//! [`tools::MemoryToolsListTool`], [`tools::MemoryToolsPutTool`]. //! //! [`PostTurnHook`]: crate::openhuman::agent::hooks::PostTurnHook @@ -36,6 +38,7 @@ pub mod prompt; pub mod store; #[cfg(test)] pub mod test_helpers; +pub mod tools; pub mod types; pub use capture::ToolMemoryCaptureHook; diff --git a/src/openhuman/memory/tool_memory/prompt.rs b/src/openhuman/memory_tools/prompt.rs similarity index 100% rename from src/openhuman/memory/tool_memory/prompt.rs rename to src/openhuman/memory_tools/prompt.rs diff --git a/src/openhuman/memory/tool_memory/store.rs b/src/openhuman/memory_tools/store.rs similarity index 80% rename from src/openhuman/memory/tool_memory/store.rs rename to src/openhuman/memory_tools/store.rs index db4fa8c0a..4ef685d46 100644 --- a/src/openhuman/memory/tool_memory/store.rs +++ b/src/openhuman/memory_tools/store.rs @@ -16,6 +16,7 @@ use std::collections::HashMap; use std::sync::Arc; +use rusqlite::params; use serde_json::Value; use super::types::{tool_memory_namespace, ToolMemoryPriority, ToolMemoryRule, ToolMemorySource}; @@ -107,28 +108,65 @@ impl ToolMemoryStore { /// first) and then `updated_at` descending. pub async fn list_rules(&self, tool_name: &str) -> Result, String> { let namespace = tool_memory_namespace(tool_name); - let entries = self - .memory - .list(Some(&namespace), None, None) - .await - .map_err(|e| format!("list tool rules: {e:#}"))?; + let mut rules: Vec = if let Some(conn) = self.memory.sqlite_conn() { + let conn = conn.lock(); + let mut stmt = conn + .prepare( + "SELECT key, content + FROM memory_docs + WHERE namespace = ?1 AND key LIKE 'rule/%' + ORDER BY updated_at DESC", + ) + .map_err(|e| format!("prepare tool rule list: {e}"))?; + let rows = stmt + .query_map(params![namespace], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + }) + .map_err(|e| format!("query tool rule list: {e}"))?; - let mut rules: Vec = entries - .into_iter() - .filter(|entry| entry.key.starts_with("rule/")) - .filter_map( - |entry| match serde_json::from_str::(&entry.content) { + rows.filter_map(|row| match row { + Ok((key, content)) => match serde_json::from_str::(&content) { Ok(rule) => Some(rule), Err(err) => { log::warn!( - "[tool-memory] skipping malformed rule key={} tool={tool_name}: {err}", - entry.key + "[tool-memory] skipping malformed sqlite rule key={} tool={tool_name}: {err}", + key ); None } }, - ) - .collect(); + Err(err) => { + log::warn!( + "[tool-memory] skipping unreadable sqlite rule row tool={tool_name}: {err}" + ); + None + } + }) + .collect() + } else { + let entries = self + .memory + .list(Some(&namespace), None, None) + .await + .map_err(|e| format!("list tool rules: {e:#}"))?; + + entries + .into_iter() + .filter(|entry| entry.key.starts_with("rule/")) + .filter_map( + |entry| match serde_json::from_str::(&entry.content) { + Ok(rule) => Some(rule), + Err(err) => { + log::warn!( + "[tool-memory] skipping malformed rule key={} tool={tool_name}: {err}", + entry.key + ); + None + } + }, + ) + .collect() + }; rules.sort_by(|a, b| { b.priority diff --git a/src/openhuman/memory/tool_memory/store_tests.rs b/src/openhuman/memory_tools/store_tests.rs similarity index 100% rename from src/openhuman/memory/tool_memory/store_tests.rs rename to src/openhuman/memory_tools/store_tests.rs diff --git a/src/openhuman/memory_tools/test_helpers.rs b/src/openhuman/memory_tools/test_helpers.rs new file mode 100644 index 000000000..b74690465 --- /dev/null +++ b/src/openhuman/memory_tools/test_helpers.rs @@ -0,0 +1,227 @@ +//! Shared test infrastructure for the tool-scoped memory layer. +//! +//! Only compiled under `#[cfg(test)]`. + +use std::collections::HashMap; + +use async_trait::async_trait; +use parking_lot::Mutex; + +use crate::openhuman::memory::{Memory, MemoryCategory, MemoryEntry, NamespaceSummary, RecallOpts}; + +/// Minimal in-memory [`Memory`] backend for unit tests. +/// +/// Stores entries in a `HashMap` keyed by `(namespace, key)`. All methods +/// that are not needed by the store/capture tests are no-ops. +#[derive(Default)] +pub struct MockMemory { + pub entries: Mutex>, +} + +#[async_trait] +impl Memory for MockMemory { + fn name(&self) -> &str { + "mock" + } + async fn store( + &self, + namespace: &str, + key: &str, + content: &str, + category: MemoryCategory, + session_id: Option<&str>, + ) -> anyhow::Result<()> { + self.entries.lock().insert( + (namespace.to_string(), key.to_string()), + MemoryEntry { + id: format!("{namespace}/{key}"), + key: key.to_string(), + content: content.to_string(), + namespace: Some(namespace.to_string()), + category, + timestamp: "now".into(), + session_id: session_id.map(str::to_string), + score: None, + }, + ); + Ok(()) + } + async fn recall( + &self, + _query: &str, + _limit: usize, + _opts: RecallOpts<'_>, + ) -> anyhow::Result> { + Ok(Vec::new()) + } + async fn get(&self, namespace: &str, key: &str) -> anyhow::Result> { + Ok(self + .entries + .lock() + .get(&(namespace.to_string(), key.to_string())) + .cloned()) + } + async fn list( + &self, + namespace: Option<&str>, + _category: Option<&MemoryCategory>, + _session_id: Option<&str>, + ) -> anyhow::Result> { + let lock = self.entries.lock(); + Ok(match namespace { + Some(ns) => lock + .iter() + .filter(|((n, _), _)| n == ns) + .map(|(_, v)| v.clone()) + .collect(), + None => lock.iter().map(|(_, v)| v.clone()).collect(), + }) + } + async fn forget(&self, namespace: &str, key: &str) -> anyhow::Result { + Ok(self + .entries + .lock() + .remove(&(namespace.to_string(), key.to_string())) + .is_some()) + } + async fn namespace_summaries(&self) -> anyhow::Result> { + let mut counts: HashMap = HashMap::new(); + for ((ns, _), _) in self.entries.lock().iter() { + *counts.entry(ns.clone()).or_default() += 1; + } + Ok(counts + .into_iter() + .map(|(namespace, count)| NamespaceSummary { + namespace, + count, + last_updated: None, + }) + .collect()) + } + async fn count(&self) -> anyhow::Result { + Ok(self.entries.lock().len()) + } + async fn health_check(&self) -> bool { + true + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn mock_memory_store_get_list_and_count_roundtrip() { + let memory = MockMemory::default(); + memory + .store( + "tool-bash", + "rule/1", + "always dry run first", + MemoryCategory::Custom("tool_memory".into()), + Some("session-1"), + ) + .await + .unwrap(); + memory + .store( + "tool-web", + "rule/2", + "cite sources", + MemoryCategory::Conversation, + None, + ) + .await + .unwrap(); + + let got = memory.get("tool-bash", "rule/1").await.unwrap().unwrap(); + assert_eq!(got.id, "tool-bash/rule/1"); + assert_eq!(got.content, "always dry run first"); + assert_eq!(got.namespace.as_deref(), Some("tool-bash")); + assert_eq!(got.session_id.as_deref(), Some("session-1")); + + let scoped = memory.list(Some("tool-bash"), None, None).await.unwrap(); + assert_eq!(scoped.len(), 1); + assert_eq!(scoped[0].key, "rule/1"); + + let all = memory.list(None, None, None).await.unwrap(); + assert_eq!(all.len(), 2); + assert_eq!(memory.count().await.unwrap(), 2); + assert!(memory.health_check().await); + assert_eq!(memory.name(), "mock"); + + // The mock intentionally ignores category/session filters so tool + // tests can focus on caller behavior instead of backend indexing. + let filtered = memory + .list( + Some("tool-bash"), + Some(&MemoryCategory::Core), + Some("different-session"), + ) + .await + .unwrap(); + assert_eq!(filtered.len(), 1); + assert_eq!(filtered[0].key, "rule/1"); + } + + #[tokio::test] + async fn mock_memory_forget_and_namespace_summaries_track_entries() { + let memory = MockMemory::default(); + memory + .store("tool-bash", "rule/1", "first", MemoryCategory::Core, None) + .await + .unwrap(); + memory + .store("tool-bash", "rule/2", "second", MemoryCategory::Daily, None) + .await + .unwrap(); + memory + .store( + "tool-web", + "rule/3", + "third", + MemoryCategory::Conversation, + None, + ) + .await + .unwrap(); + + let mut summaries = memory.namespace_summaries().await.unwrap(); + summaries.sort_by(|a, b| a.namespace.cmp(&b.namespace)); + assert_eq!(summaries.len(), 2); + assert_eq!(summaries[0].namespace, "tool-bash"); + assert_eq!(summaries[0].count, 2); + assert_eq!(summaries[1].namespace, "tool-web"); + assert_eq!(summaries[1].count, 1); + + assert!(memory.forget("tool-bash", "rule/1").await.unwrap()); + assert!(!memory.forget("tool-bash", "missing").await.unwrap()); + + let remaining = memory.list(Some("tool-bash"), None, None).await.unwrap(); + assert_eq!(remaining.len(), 1); + assert_eq!(remaining[0].key, "rule/2"); + } + + #[tokio::test] + async fn mock_memory_recall_is_empty_noop() { + let memory = MockMemory::default(); + let recalled = memory + .recall("anything", 5, RecallOpts::default()) + .await + .unwrap(); + assert!(recalled.is_empty()); + } + + #[tokio::test] + async fn mock_memory_empty_state_helpers_return_empty_values() { + let memory = MockMemory::default(); + assert!(memory.get("missing", "rule").await.unwrap().is_none()); + assert!(memory + .list(Some("missing"), None, None) + .await + .unwrap() + .is_empty()); + assert!(memory.namespace_summaries().await.unwrap().is_empty()); + assert_eq!(memory.count().await.unwrap(), 0); + } +} diff --git a/src/openhuman/memory_tools/tools/list.rs b/src/openhuman/memory_tools/tools/list.rs new file mode 100644 index 000000000..c4feb2a26 --- /dev/null +++ b/src/openhuman/memory_tools/tools/list.rs @@ -0,0 +1,162 @@ +//! `memory_tools_list` — list every stored rule for a given tool. + +use async_trait::async_trait; +use serde::Deserialize; +use serde_json::json; + +use crate::openhuman::memory::ops::helpers::active_memory_client; +use crate::openhuman::memory_tools::ToolMemoryStore; +use crate::openhuman::tools::traits::{Tool, ToolResult}; + +pub struct MemoryToolsListTool; + +#[derive(Debug, Deserialize)] +struct Args { + tool_name: String, +} + +#[async_trait] +impl Tool for MemoryToolsListTool { + fn name(&self) -> &str { + "memory_tools_list" + } + + fn description(&self) -> &str { + "List every stored memory rule for the given tool. Rules are durable \ + learnings about how to use the tool — priorities, gotchas, user \ + edicts. Returns the rules ordered by priority (Critical → Low) and \ + updated_at DESC within each priority." + } + + fn parameters_schema(&self) -> serde_json::Value { + json!({ + "type": "object", + "required": ["tool_name"], + "properties": { + "tool_name": { + "type": "string", + "description": "Exact tool name (e.g. `bash`, `web_search`)." + } + } + }) + } + + async fn execute(&self, args: serde_json::Value) -> anyhow::Result { + let parsed: Args = serde_json::from_value(args) + .map_err(|e| anyhow::anyhow!("invalid arguments for memory_tools_list: {e}"))?; + log::debug!("[tool][memory_tools] list tool_name={}", parsed.tool_name); + let client = active_memory_client() + .await + .map_err(|e| anyhow::anyhow!("memory_tools_list: {e}"))?; + let store = ToolMemoryStore::new(client.memory_handle()); + let rules = store + .list_rules(&parsed.tool_name) + .await + .map_err(|e| anyhow::anyhow!("memory_tools_list: {e}"))?; + let json = serde_json::to_string(&rules)?; + Ok(ToolResult::success(json)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::OsString; + + use tempfile::TempDir; + + use crate::openhuman::config::{Config, TEST_ENV_LOCK}; + use crate::openhuman::tools::traits::Tool; + use serde_json::json; + + struct WorkspaceEnvGuard { + _lock: std::sync::MutexGuard<'static, ()>, + previous: Option, + } + + impl WorkspaceEnvGuard { + fn set(path: &std::path::Path) -> Self { + let lock = TEST_ENV_LOCK.lock().unwrap_or_else(|err| err.into_inner()); + let previous = std::env::var_os("OPENHUMAN_WORKSPACE"); + std::env::set_var("OPENHUMAN_WORKSPACE", path); + Self { + _lock: lock, + previous, + } + } + } + + impl Drop for WorkspaceEnvGuard { + fn drop(&mut self) { + if let Some(previous) = self.previous.as_ref() { + std::env::set_var("OPENHUMAN_WORKSPACE", previous); + } else { + std::env::remove_var("OPENHUMAN_WORKSPACE"); + } + } + } + + async fn isolated_config(tmp: &TempDir) -> (WorkspaceEnvGuard, Config) { + let guard = WorkspaceEnvGuard::set(tmp.path()); + let config = Config::load_or_init().await.expect("load config"); + (guard, config) + } + + #[test] + fn args_require_tool_name() { + let args: Args = serde_json::from_value(json!({ "tool_name": "bash" })).unwrap(); + assert_eq!(args.tool_name, "bash"); + } + + #[test] + fn parameters_schema_requires_tool_name() { + let tool = MemoryToolsListTool; + let schema = tool.parameters_schema(); + assert_eq!(schema["type"], "object"); + assert_eq!(schema["required"], json!(["tool_name"])); + assert_eq!(schema["properties"]["tool_name"]["type"], "string"); + } + + #[tokio::test] + async fn execute_rejects_missing_tool_name() { + let tool = MemoryToolsListTool; + let err = tool + .execute(json!({})) + .await + .expect_err("missing tool_name should fail"); + assert!(err + .to_string() + .contains("invalid arguments for memory_tools_list")); + } + + #[tokio::test] + async fn execute_success_path_returns_json_array_for_isolated_workspace() { + let tmp = TempDir::new().expect("tempdir"); + let (_workspace, _cfg) = isolated_config(&tmp).await; + let tool = MemoryToolsListTool; + let result = tool + .execute(json!({ "tool_name": "bash" })) + .await + .expect("valid tool list request should succeed in isolated workspace"); + assert!(!result.is_error); + let payload = result.text(); + let parsed: serde_json::Value = + serde_json::from_str(&payload).expect("result should be valid json"); + assert!( + parsed.is_array(), + "list tool rules should serialize a JSON array" + ); + } + + #[tokio::test] + async fn execute_accepts_other_tool_names_without_rules() { + let tmp = TempDir::new().expect("tempdir"); + let (_workspace, _cfg) = isolated_config(&tmp).await; + let tool = MemoryToolsListTool; + let result = tool + .execute(json!({ "tool_name": "web_search" })) + .await + .expect("arbitrary tool names should succeed even when empty"); + assert!(!result.is_error); + } +} diff --git a/src/openhuman/memory_tools/tools/mod.rs b/src/openhuman/memory_tools/tools/mod.rs new file mode 100644 index 000000000..a5830024a --- /dev/null +++ b/src/openhuman/memory_tools/tools/mod.rs @@ -0,0 +1,23 @@ +//! Agent tools for reading and writing tool-scoped memory. +//! +//! The agent uses these to introspect what rules / learnings exist for a +//! specific tool and to record new ones discovered mid-session. They are +//! the user-facing read/write surface on top of [`ToolMemoryStore`]. + +mod list; +mod put; + +pub use list::MemoryToolsListTool; +pub use put::MemoryToolsPutTool; + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::tools::traits::Tool; + + #[test] + fn exports_memory_tool_wrappers_with_stable_names() { + assert_eq!(MemoryToolsListTool.name(), "memory_tools_list"); + assert_eq!(MemoryToolsPutTool.name(), "memory_tools_put"); + } +} diff --git a/src/openhuman/memory_tools/tools/put.rs b/src/openhuman/memory_tools/tools/put.rs new file mode 100644 index 000000000..1355afe3b --- /dev/null +++ b/src/openhuman/memory_tools/tools/put.rs @@ -0,0 +1,270 @@ +//! `memory_tools_put` — upsert a tool-scoped memory rule. + +use async_trait::async_trait; +use serde::Deserialize; +use serde_json::json; + +use crate::openhuman::memory::ops::helpers::active_memory_client; +use crate::openhuman::memory_tools::{ + ToolMemoryPriority, ToolMemoryRule, ToolMemorySource, ToolMemoryStore, +}; +use crate::openhuman::tools::traits::{Tool, ToolResult}; + +pub struct MemoryToolsPutTool; + +#[derive(Debug, Deserialize)] +struct Args { + tool_name: String, + rule: String, + #[serde(default)] + priority: Option, + #[serde(default)] + tags: Vec, +} + +fn parse_priority(s: Option<&str>) -> ToolMemoryPriority { + match s.map(|x| x.to_ascii_lowercase()) { + Some(ref v) if v == "critical" => ToolMemoryPriority::Critical, + Some(ref v) if v == "high" => ToolMemoryPriority::High, + _ => ToolMemoryPriority::Normal, + } +} + +#[async_trait] +impl Tool for MemoryToolsPutTool { + fn name(&self) -> &str { + "memory_tools_put" + } + + fn description(&self) -> &str { + "Record a durable rule / learning for the given tool. Use when the \ + user gives a directive that should survive future sessions, or \ + when a tool failure pattern is worth pinning. Returns the stored \ + rule with its assigned id and timestamps." + } + + fn parameters_schema(&self) -> serde_json::Value { + json!({ + "type": "object", + "required": ["tool_name", "rule"], + "properties": { + "tool_name": { + "type": "string", + "description": "Exact tool name the rule applies to." + }, + "rule": { + "type": "string", + "description": "Free-text rule, edict, or learning to pin." + }, + "priority": { + "type": "string", + "enum": ["critical", "high", "normal"], + "description": "How aggressively to surface the rule. Default: normal." + }, + "tags": { + "type": "array", + "items": { "type": "string" }, + "description": "Optional free-form tags (e.g. `safety`, `permission`)." + } + } + }) + } + + async fn execute(&self, args: serde_json::Value) -> anyhow::Result { + let parsed: Args = serde_json::from_value(args) + .map_err(|e| anyhow::anyhow!("invalid arguments for memory_tools_put: {e}"))?; + log::debug!( + "[tool][memory_tools] put tool_name={} priority={:?} tags={}", + parsed.tool_name, + parsed.priority, + parsed.tags.len() + ); + let client = active_memory_client() + .await + .map_err(|e| anyhow::anyhow!("memory_tools_put: {e}"))?; + let store = ToolMemoryStore::new(client.memory_handle()); + let mut rule = ToolMemoryRule::new( + &parsed.tool_name, + &parsed.rule, + parse_priority(parsed.priority.as_deref()), + ToolMemorySource::UserExplicit, + ); + rule.tags = parsed.tags; + let stored = store + .put_rule(rule) + .await + .map_err(|e| anyhow::anyhow!("memory_tools_put: {e}"))?; + let json = serde_json::to_string(&stored)?; + Ok(ToolResult::success(json)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::OsString; + + use tempfile::TempDir; + + use crate::openhuman::config::{Config, TEST_ENV_LOCK}; + use crate::openhuman::memory_tools::ToolMemoryStore; + use crate::openhuman::tools::traits::Tool; + use serde_json::json; + + struct WorkspaceEnvGuard { + _lock: std::sync::MutexGuard<'static, ()>, + previous: Option, + } + + impl WorkspaceEnvGuard { + fn set(path: &std::path::Path) -> Self { + let lock = TEST_ENV_LOCK.lock().unwrap_or_else(|err| err.into_inner()); + let previous = std::env::var_os("OPENHUMAN_WORKSPACE"); + std::env::set_var("OPENHUMAN_WORKSPACE", path); + Self { + _lock: lock, + previous, + } + } + } + + impl Drop for WorkspaceEnvGuard { + fn drop(&mut self) { + if let Some(previous) = self.previous.as_ref() { + std::env::set_var("OPENHUMAN_WORKSPACE", previous); + } else { + std::env::remove_var("OPENHUMAN_WORKSPACE"); + } + } + } + + async fn isolated_config(tmp: &TempDir) -> (WorkspaceEnvGuard, Config) { + let guard = WorkspaceEnvGuard::set(tmp.path()); + let config = Config::load_or_init().await.expect("load config"); + (guard, config) + } + + #[test] + fn parse_priority_defaults_to_normal() { + assert_eq!(parse_priority(None), ToolMemoryPriority::Normal); + assert_eq!(parse_priority(Some("normal")), ToolMemoryPriority::Normal); + assert_eq!(parse_priority(Some("unknown")), ToolMemoryPriority::Normal); + } + + #[test] + fn parse_priority_accepts_critical_and_high_case_insensitively() { + assert_eq!( + parse_priority(Some("critical")), + ToolMemoryPriority::Critical + ); + assert_eq!( + parse_priority(Some("CRITICAL")), + ToolMemoryPriority::Critical + ); + assert_eq!(parse_priority(Some("high")), ToolMemoryPriority::High); + assert_eq!(parse_priority(Some("HiGh")), ToolMemoryPriority::High); + } + + #[test] + fn args_default_tags_to_empty() { + let args: Args = serde_json::from_value(json!({ + "tool_name": "bash", + "rule": "Never run rm -rf" + })) + .unwrap(); + assert_eq!(args.tool_name, "bash"); + assert_eq!(args.rule, "Never run rm -rf"); + assert!(args.priority.is_none()); + assert!(args.tags.is_empty()); + } + + #[test] + fn parameters_schema_describes_priority_enum() { + let tool = MemoryToolsPutTool; + let schema = tool.parameters_schema(); + assert_eq!(schema["required"], json!(["tool_name", "rule"])); + assert_eq!( + schema["properties"]["priority"]["enum"], + json!(["critical", "high", "normal"]) + ); + } + + #[tokio::test] + async fn execute_rejects_missing_required_fields() { + let tool = MemoryToolsPutTool; + let err = tool + .execute(json!({ "tool_name": "bash" })) + .await + .expect_err("missing rule should fail"); + assert!(err + .to_string() + .contains("invalid arguments for memory_tools_put")); + + let err = tool + .execute(json!({ "rule": "Never run rm -rf" })) + .await + .expect_err("missing tool_name should fail"); + assert!(err + .to_string() + .contains("invalid arguments for memory_tools_put")); + } + + #[tokio::test] + async fn execute_success_path_persists_rule_in_isolated_workspace() { + let tmp = TempDir::new().expect("tempdir"); + let (_workspace, _cfg) = isolated_config(&tmp).await; + let tool = MemoryToolsPutTool; + let result = tool + .execute(json!({ + "tool_name": "bash", + "rule": "Always dry-run dangerous commands first", + "priority": "high", + "tags": ["safety", "shell"] + })) + .await + .expect("valid memory_tools_put request should succeed in isolated workspace"); + assert!(!result.is_error); + + let parsed: serde_json::Value = + serde_json::from_str(&result.text()).expect("tool result should be json"); + assert_eq!(parsed["tool_name"], "bash"); + assert_eq!(parsed["rule"], "Always dry-run dangerous commands first"); + assert_eq!(parsed["priority"], "high"); + assert_eq!(parsed["source"], "user_explicit"); + assert_eq!(parsed["tags"], json!(["safety", "shell"])); + assert!(parsed["id"].as_str().is_some()); + + let client = crate::openhuman::memory::ops::helpers::active_memory_client() + .await + .expect("active memory client"); + let store = ToolMemoryStore::new(client.memory_handle()); + let rules = store.list_rules("bash").await.expect("list stored rules"); + let stored = rules + .iter() + .find(|rule| rule.rule == "Always dry-run dangerous commands first") + .expect("stored bash rule should be present"); + assert_eq!(stored.priority, ToolMemoryPriority::High); + assert_eq!(stored.source, ToolMemorySource::UserExplicit); + assert_eq!(stored.tags, vec!["safety".to_string(), "shell".to_string()]); + } + + #[tokio::test] + async fn execute_defaults_unknown_priority_to_normal() { + let tmp = TempDir::new().expect("tempdir"); + let (_workspace, _cfg) = isolated_config(&tmp).await; + let tool = MemoryToolsPutTool; + let result = tool + .execute(json!({ + "tool_name": "bash", + "rule": "Prefer printf over echo for escapes", + "priority": "unexpected" + })) + .await + .expect("unknown priority should still succeed"); + assert!(!result.is_error); + + let parsed: serde_json::Value = + serde_json::from_str(&result.text()).expect("tool result should be json"); + assert_eq!(parsed["priority"], "normal"); + } +} diff --git a/src/openhuman/memory/tool_memory/types.rs b/src/openhuman/memory_tools/types.rs similarity index 100% rename from src/openhuman/memory/tool_memory/types.rs rename to src/openhuman/memory_tools/types.rs diff --git a/src/openhuman/memory_tree/README.md b/src/openhuman/memory_tree/README.md index e619c9903..44f7c62b8 100644 --- a/src/openhuman/memory_tree/README.md +++ b/src/openhuman/memory_tree/README.md @@ -1,57 +1,42 @@ -# Memory tree +# memory_tree -Bucket-seal-ready local memory architecture (Phase 1 of issue #707; the LLD design doc `docs/MEMORY_ARCHITECTURE_LLD.md` is referenced by the in-tree module headers but is not checked into this repo). Coexists with the legacy `store/` backend until full replacement. - -## Pipeline +Generic tree mechanics on top of `memory_store::trees`. Kind-agnostic: a +`Source`, `Global`, or `Topic` tree all flow through the same code here. +Kind-specific policy (when to spawn a topic tree, what scope a global tree +covers, how digests are written) lives in `memory::tree_global` and +`memory::tree_topic`; this module is unaware of it. ```text -source adapters (chat / email / document) - │ - ▼ -canonicalize/ ── normalised Markdown + provenance Metadata - │ - ▼ -chunker.rs ── deterministic IDs, ≤3k-token bounded segments - │ - ▼ -content_store/── atomic .md files on disk (body + tags) - │ - ▼ -store.rs ── SQLite persistence (chunks, scores, summaries, jobs, hotness) - │ - ▼ -score/ ── signals + embeddings + entity extraction - │ - ▼ -tree_source/ tree_topic/ tree_global/ ── per-scope summary trees - │ - ▼ -retrieval/ ── search / drill_down / topic / global / fetch - │ - ▼ -jobs/ ── background workers + scheduler (extract, admit, seal, digest) +memory (orchestrator) ──┐ + │ writes leaves via TreeWriteRequest + ▼ +memory_tree (this module — generic mechanics) + ├── tree/ append + cascade seal + flush + ├── summarise.rs L_n -> L_{n+1} text via the chat model + ├── sources/ per-source tree registry + .md mirror + ├── tools/ agent-facing read tools (walk, drill, fetch) + └── io.rs canonical Tree{Write,Read}{Request,Outcome,Result} + │ + ▼ +memory_store::trees (persistence: one Tree table, one schema) ``` -## Files at this level +## Layout -- [`mod.rs`](mod.rs) — Phase 1 module banner; re-exports controller registries (`all_memory_tree_*`, `all_retrieval_*`). -- [`chunker.rs`](chunker.rs) — slice canonical Markdown into ≤`DEFAULT_CHUNK_MAX_TOKENS` chunks; chat/email split at message boundaries, document at paragraphs. -- [`ingest.rs`](ingest.rs) — orchestrator: `canonicalize -> chunk -> stage_chunks -> fast score -> persist -> enqueue extract jobs`. Hot path; heavy work runs out of `jobs/`. -- [`rpc.rs`](rpc.rs) — JSON-RPC handlers for `memory_tree_ingest`, `list_chunks`, `get_chunk`, `trigger_digest`. Delegates to `ingest`/`store`/`jobs`. -- [`schemas.rs`](schemas.rs) — `ControllerSchema` definitions + `RegisteredController` wiring for the four `memory_tree_*` RPC methods. -- [`store.rs`](store.rs) — SQLite schema (chunks, score, entity index, trees, summaries, buffers, hotness, jobs) and accessors. Lazily initialised at `/memory_tree/chunks.db`. -- [`store_tests.rs`](store_tests.rs) — store-layer unit tests. -- [`types.rs`](types.rs) — `Chunk`, `Metadata`, `SourceKind`, `DataSource`, `SourceRef`; deterministic `chunk_id` hash; `approx_token_count` heuristic. +| Path | Role | +| --- | --- | +| [`mod.rs`](mod.rs) | Re-exports `io::*` and the controller-schema registries hosted in `memory`. Re-exports `memory::tree_global` + `memory::tree_topic` under the legacy `memory_tree::tree_{global,topic}` paths. | +| [`io.rs`](io.rs) | Canonical contract types: `TreeWriteRequest`/`TreeWriteOutcome`, `TreeReadRequest`/`TreeReadHit`/`TreeReadResult`, `TreeLeafPayload`, `TreeLabelStrategy`. Pure types, no IO. | +| [`tree/`](tree/) | `bucket_seal` (append leaf + cascade seal), `flush` (time-based partial seal), `registry` (kind-parameterized `get_or_create_tree` with UNIQUE-race recovery), `mod.rs` (re-exports + `memory_store::trees` shims for legacy paths). | +| [`summarise.rs`](summarise.rs) | One function: produce the next-level summary text for a bucket. Wraps the chat model with a fixed prompt and token budget. | +| [`sources/`](sources/) | `registry::get_or_create_source_tree` wrapper that adds the `_source.md` on-disk mirror to the generic registry. `file.rs` writes the mirror. | +| [`tools/`](tools/) | Agent-facing tools. Read: `walk` (agentic), `drill_down`, `fetch_leaves`, `query_{source,global,topic}`, `search_entities`. Write: `ingest_document` (orchestrator-facing). | -## Subdirectories +## Layer rules -- [`canonicalize/`](canonicalize/README.md) — chat / email / document → canonical Markdown + email body cleaner. -- [`chunker.rs`](chunker.rs) — see above. -- [`content_store/`](content_store/README.md) — on-disk `.md` files (atomic writes, paths, YAML compose, read+verify, tag rewrites). -- [`jobs/`](jobs/) — async job queue (extract / admit / seal / topic / digest workers). -- [`retrieval/`](retrieval/) — search and drill-down RPC surface. -- [`score/`](score/) — fast scorer, embeddings, entity extraction, score persistence. -- [`tree_source/`](tree_source/) — per-source summary trees (L0 buffer → L1 seal → cascade). -- [`tree_topic/`](tree_topic/) — per-entity topic trees, materialised lazily by hotness. -- [`tree_global/`](tree_global/) — daily global digest tree. -- [`util/`](util/README.md) — shared helpers (`redact` for log PII). +- **No tree-kind branching here.** `bucket_seal`, `flush`, `registry`, + `summarise` all take `TreeKind` as a parameter or treat it as opaque. +- **No persistence here.** Reads and writes go through + `memory_store::trees::{store, registry, hotness}`. +- **No policy here.** Curator gates (hotness thresholds), digest cadence, + global scope sentinels — all live in `memory::tree_{global,topic}`. diff --git a/src/openhuman/memory_tree/chat/cloud.rs b/src/openhuman/memory_tree/chat/cloud.rs deleted file mode 100644 index a95a979e0..000000000 --- a/src/openhuman/memory_tree/chat/cloud.rs +++ /dev/null @@ -1,259 +0,0 @@ -//! Cloud chat provider — routes through the OpenHuman backend's -//! `/openai/v1/chat/completions` surface using the existing -//! [`crate::openhuman::inference::provider::openhuman_backend::OpenHumanBackendProvider`]. -//! -//! Used when `memory_tree.llm_backend = "cloud"` (the default). The -//! request shape is the standard OpenAI-compatible chat-completions -//! protocol, with `temperature: 0.0` and a `summarization-v1` (or -//! caller-configured) model. -//! -//! When the configured model is unavailable for the user's organization, -//! the provider automatically falls back through a list of known -//! summarization-capable models before giving up. - -use std::path::PathBuf; - -use anyhow::{Context, Result}; -use async_trait::async_trait; - -use crate::openhuman::inference::provider::openhuman_backend::OpenHumanBackendProvider; -use crate::openhuman::inference::provider::traits::{ChatMessage, Provider}; -use crate::openhuman::inference::provider::ProviderRuntimeOptions; - -use super::{ChatPrompt, ChatProvider}; - -/// Fallback models tried in order when the configured model is unavailable. -const FALLBACK_MODELS: &[&str] = &[ - "summarization-v1", - "deepseek-ai/DeepSeek-V3-0324", - "deepseek-ai/DeepSeek-V3", -]; - -/// Returns true if the error indicates the model is not provisioned for the org. -/// Only matches the explicit "not available for your organization" phrase from -/// the GMI API — generic 404s are NOT treated as model-unavailable to avoid -/// masking unrelated backend failures. -fn is_model_unavailable_error(err: &anyhow::Error) -> bool { - let msg = format!("{err:?}"); - msg.contains("not available for your organization") -} - -/// Cloud-routed chat provider. Holds an [`OpenHumanBackendProvider`] and -/// forwards each [`ChatProvider::chat_for_json`] call through its -/// `chat_with_history` method. -pub struct CloudChatProvider { - inner: OpenHumanBackendProvider, - model: String, - /// Cached display name `"cloud:"` for logs. - display: String, -} - -impl CloudChatProvider { - /// Build a new cloud provider against `api_url` (or the default - /// `effective_api_url` when `None`) for `model`. The provider does NOT - /// resolve the bearer token at construction — it does so per request, - /// matching the existing `OpenHumanBackendProvider` contract. That way - /// a session refresh between memory-tree calls is picked up - /// transparently. - /// - /// `openhuman_dir` is the directory containing `auth-profiles.json` (i.e. - /// the parent of `config.config_path`). Without it the inner provider - /// would fall back to `~/.openhuman` and fail with "No backend session" - /// on workspaces not located at the home default. - pub fn new( - api_url: Option, - model: String, - openhuman_dir: Option, - secrets_encrypt: bool, - ) -> Self { - let opts = ProviderRuntimeOptions { - openhuman_dir, - secrets_encrypt, - ..ProviderRuntimeOptions::default() - }; - let inner = OpenHumanBackendProvider::new(api_url.as_deref(), &opts); - let display = format!("cloud:{model}"); - Self { - inner, - model, - display, - } - } - - /// Try a single model, returning Ok(text) or the error. - async fn try_model( - &self, - messages: &[ChatMessage], - model: &str, - temperature: f64, - ) -> Result { - self.inner - .chat_with_history(messages, model, temperature) - .await - } -} - -#[async_trait] -impl ChatProvider for CloudChatProvider { - fn name(&self) -> &str { - &self.display - } - - async fn chat_for_json(&self, prompt: &ChatPrompt) -> Result { - log::debug!( - "[memory_tree::chat::cloud] kind={} model={} sys_chars={} user_chars={}", - prompt.kind, - self.model, - prompt.system.len(), - prompt.user.len() - ); - - let messages = vec![ - ChatMessage::system(prompt.system.clone()), - ChatMessage::user(prompt.user.clone()), - ]; - - // Try the configured model first. - match self - .try_model(&messages, &self.model, prompt.temperature) - .await - { - Ok(text) => { - log::debug!( - "[memory_tree::chat::cloud] response chars={} kind={}", - text.len(), - prompt.kind - ); - return Ok(text); - } - Err(e) if is_model_unavailable_error(&e) => { - log::warn!( - "[memory_tree::chat::cloud] model={} unavailable, trying fallbacks", - self.model - ); - } - Err(e) => { - log::warn!( - "[memory_tree::chat::cloud] model={} failed kind={} err={:#}", - self.model, - prompt.kind, - e - ); - return Err(e).with_context(|| { - format!( - "cloud chat request kind={} model={} failed", - prompt.kind, self.model - ) - }); - } - } - - // Fallback chain — skip the configured model if it appears in the list. - for &fallback in FALLBACK_MODELS { - if fallback == self.model { - continue; - } - log::debug!( - "[memory_tree::chat::cloud] trying fallback model={}", - fallback - ); - match self - .try_model(&messages, fallback, prompt.temperature) - .await - { - Ok(text) => { - log::info!( - "[memory_tree::chat::cloud] fallback model={} succeeded kind={}", - fallback, - prompt.kind - ); - return Ok(text); - } - Err(e) if is_model_unavailable_error(&e) => { - log::debug!( - "[memory_tree::chat::cloud] fallback model={} also unavailable", - fallback - ); - continue; - } - Err(e) => { - log::warn!( - "[memory_tree::chat::cloud] fallback model={} failed kind={} err={:#}", - fallback, - prompt.kind, - e - ); - return Err(e).with_context(|| { - format!( - "cloud chat request kind={} fallback model={} failed", - prompt.kind, fallback - ) - }); - } - } - } - - log::warn!( - "[memory_tree::chat::cloud] configured model={} and all fallbacks unavailable kind={}", - self.model, - prompt.kind - ); - anyhow::bail!( - "cloud chat kind={}: configured model '{}' and all fallback models are unavailable \ - for this organization", - prompt.kind, - self.model - ) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn name_includes_model() { - let p = CloudChatProvider::new(None, "summarization-v1".into(), None, true); - assert_eq!(p.name(), "cloud:summarization-v1"); - } - - #[test] - fn name_changes_with_model() { - let p = CloudChatProvider::new(None, "claude-haiku-4.5".into(), None, true); - assert!(p.name().contains("claude-haiku-4.5")); - } - - #[test] - fn detects_model_unavailable_error() { - let err = anyhow::anyhow!( - "OpenHuman API error (404 Not Found): {{\"success\":false,\"error\":\"GMI model \ - 'deepseek-ai/DeepSeek-V4-Flash' is not available for your organization.\"}}" - ); - assert!(is_model_unavailable_error(&err)); - } - - #[test] - fn non_model_error_not_detected_as_unavailable() { - let err = anyhow::anyhow!("connection timeout after 30s"); - assert!(!is_model_unavailable_error(&err)); - } - - #[test] - fn generic_404_with_model_not_treated_as_unavailable() { - // A generic 404 mentioning "model" should NOT trigger fallback — - // only the explicit "not available for your organization" phrase should. - let err = - anyhow::anyhow!("OpenHuman API error (404 Not Found): model endpoint returned 404"); - assert!(!is_model_unavailable_error(&err)); - } - - #[test] - fn fallback_list_contains_summarization_v1() { - assert!(FALLBACK_MODELS.contains(&"summarization-v1")); - } - - #[test] - fn fallback_list_not_empty() { - assert!(!FALLBACK_MODELS.is_empty()); - } -} diff --git a/src/openhuman/memory_tree/chat/local.rs b/src/openhuman/memory_tree/chat/local.rs deleted file mode 100644 index 18dbb7707..000000000 --- a/src/openhuman/memory_tree/chat/local.rs +++ /dev/null @@ -1,281 +0,0 @@ -//! Local Ollama chat provider — the legacy `llm_backend = "local"` path. -//! -//! Speaks Ollama's `/api/chat` with `format: "json"` and -//! `temperature: 0.0`. Mirrors what the per-extractor/summariser HTTP client -//! used to do, but behind the [`super::ChatProvider`] trait so the same -//! call site can be cloud-routed instead. - -use std::time::Duration; - -use anyhow::{anyhow, Context, Result}; -use async_trait::async_trait; -use reqwest::Client; -use serde::{Deserialize, Serialize}; - -use super::{ChatPrompt, ChatProvider}; - -/// Ollama-direct chat provider. -pub struct OllamaChatProvider { - endpoint: String, - model: String, - http: Client, - /// Cached display name `"local:ollama:"` for logs. - display: String, -} - -impl OllamaChatProvider { - /// Build the provider. `endpoint` and `model` may be `None` — when - /// either is unset, [`ChatProvider::chat_for_json`] returns a clear - /// error so the caller's soft-fallback path engages and the seal/admit - /// pipeline keeps running. - pub fn new(endpoint: Option, model: Option, timeout: Duration) -> Result { - // No body-read timeout. Ollama is a local process — slow responses - // mean the model is genuinely processing under CPU load (e.g. - // gemma3:1b on CPU-only inference can take minutes per call), not - // that the network broke. A body-read timeout here would cancel - // mid-flight generation and force pointless retries against the - // same slow model. `timeout` becomes the TCP connect timeout — - // short enough to fail fast when Ollama is actually unreachable. - let http = Client::builder() - .connect_timeout(timeout) - .build() - .context("build ollama http client")?; - let endpoint = endpoint.unwrap_or_default(); - let model = model.unwrap_or_default(); - let display = format!( - "local:ollama:{}", - if model.is_empty() { "" } else { &model } - ); - Ok(Self { - endpoint, - model, - http, - display, - }) - } -} - -#[async_trait] -impl ChatProvider for OllamaChatProvider { - fn name(&self) -> &str { - &self.display - } - - async fn chat_for_json(&self, prompt: &ChatPrompt) -> Result { - self.run_chat(prompt, Some("json")).await - } - - async fn chat_for_text(&self, prompt: &ChatPrompt) -> Result { - // Omit `format` entirely — Ollama's `/api/chat` only accepts - // `"json"`, a JSON-schema object, or absence-of-field for - // free-form text. Sending `format: ""` is undefined behaviour, - // so the field is dropped from the request body when None. - self.run_chat(prompt, None).await - } -} - -impl OllamaChatProvider { - async fn run_chat(&self, prompt: &ChatPrompt, format: Option<&str>) -> Result { - if self.endpoint.is_empty() || self.model.is_empty() { - return Err(anyhow!( - "[memory_tree::chat::local] Ollama endpoint or model not configured \ - (endpoint_set={}, model_set={}); set memory_tree.llm_*_endpoint / \ - _model in config or switch memory_tree.llm_backend to cloud", - !self.endpoint.is_empty(), - !self.model.is_empty() - )); - } - let url = format!("{}/api/chat", self.endpoint.trim_end_matches('/')); - let body = OllamaChatRequest { - model: self.model.clone(), - messages: vec![ - OllamaMessage { - role: "system".to_string(), - content: prompt.system.clone(), - }, - OllamaMessage { - role: "user".to_string(), - content: prompt.user.clone(), - }, - ], - format: format.map(str::to_string), - stream: false, - options: OllamaOptions { - temperature: prompt.temperature as f32, - }, - }; - - log::debug!( - "[memory_tree::chat::local] POST {url} kind={} model={} sys_chars={} user_chars={}", - prompt.kind, - self.model, - prompt.system.len(), - prompt.user.len() - ); - - let resp = self - .http - .post(&url) - .json(&body) - .send() - .await - .with_context(|| format!("ollama POST {url}"))?; - - if !resp.status().is_success() { - let status = resp.status(); - let snippet = resp.text().await.unwrap_or_default(); - return Err(anyhow!( - "ollama non-success status {status}: {}", - truncate_for_log(&snippet, 200) - )); - } - - let envelope: OllamaChatResponse = resp - .json() - .await - .context("decode ollama chat response envelope")?; - log::debug!( - "[memory_tree::chat::local] ollama response chars={} kind={}", - envelope.message.content.len(), - prompt.kind - ); - Ok(envelope.message.content) - } -} - -fn truncate_for_log(s: &str, max_chars: usize) -> String { - if s.chars().count() <= max_chars { - return s.to_string(); - } - let truncated: String = s.chars().take(max_chars).collect(); - format!("{truncated}…") -} - -#[derive(Debug, Serialize)] -struct OllamaChatRequest { - model: String, - messages: Vec, - /// Omitted from the wire body when `None` (`#[serde(skip_serializing_if)]`), - /// so the JSON-mode flag is only present for the `chat_for_json` path. - /// Ollama treats absence as "free-form text". - #[serde(skip_serializing_if = "Option::is_none")] - format: Option, - stream: bool, - options: OllamaOptions, -} - -#[derive(Debug, Serialize)] -struct OllamaMessage { - role: String, - content: String, -} - -#[derive(Debug, Serialize)] -struct OllamaOptions { - temperature: f32, -} - -#[derive(Debug, Deserialize)] -struct OllamaChatResponse { - message: OllamaResponseMessage, -} - -#[derive(Debug, Deserialize)] -struct OllamaResponseMessage { - content: String, -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn errors_clearly_when_endpoint_missing() { - let p = OllamaChatProvider::new(None, Some("m".into()), Duration::from_millis(50)).unwrap(); - let err = p - .chat_for_json(&ChatPrompt { - system: "s".into(), - user: "u".into(), - temperature: 0.0, - kind: "test", - }) - .await - .unwrap_err(); - let msg = format!("{err}"); - assert!( - msg.contains("not configured"), - "expected config error, got: {msg}" - ); - } - - #[tokio::test] - async fn errors_when_model_missing() { - let p = OllamaChatProvider::new( - Some("http://localhost:11434".into()), - None, - Duration::from_millis(50), - ) - .unwrap(); - let err = p - .chat_for_json(&ChatPrompt { - system: "s".into(), - user: "u".into(), - temperature: 0.0, - kind: "test", - }) - .await - .unwrap_err(); - assert!(format!("{err}").contains("not configured")); - } - - #[tokio::test] - async fn transport_failure_returns_err() { - // Endpoint pointing at an unreachable port. The provider returns - // Err — the consumer is responsible for soft-fallback. - let p = OllamaChatProvider::new( - Some("http://127.0.0.1:1".into()), - Some("m".into()), - Duration::from_millis(50), - ) - .unwrap(); - let err = p - .chat_for_json(&ChatPrompt { - system: "s".into(), - user: "u".into(), - temperature: 0.0, - kind: "test", - }) - .await - .unwrap_err(); - // Connection error chain — message contains "ollama POST" prefix. - assert!(format!("{err}").contains("ollama POST")); - } - - #[test] - fn name_includes_model() { - let p = - OllamaChatProvider::new(None, Some("qwen2.5:0.5b".into()), Duration::from_millis(50)) - .unwrap(); - assert!(p.name().contains("qwen2.5:0.5b")); - assert!(p.name().starts_with("local:ollama:")); - } - - #[test] - fn name_handles_unset_model() { - let p = OllamaChatProvider::new(None, None, Duration::from_millis(50)).unwrap(); - assert!(p.name().contains("")); - } - - #[test] - fn truncate_for_log_short_unchanged() { - assert_eq!(truncate_for_log("hi", 10), "hi"); - } - - #[test] - fn truncate_for_log_long_appends_ellipsis() { - let long = "x".repeat(500); - let out = truncate_for_log(&long, 10); - assert_eq!(out.chars().count(), 11); - assert!(out.ends_with('…')); - } -} diff --git a/src/openhuman/memory_tree/chat/mod.rs b/src/openhuman/memory_tree/chat/mod.rs deleted file mode 100644 index 744841c63..000000000 --- a/src/openhuman/memory_tree/chat/mod.rs +++ /dev/null @@ -1,288 +0,0 @@ -//! Memory-tree chat backend abstraction. -//! -//! The memory_tree's two LLM consumers (the entity extractor and the -//! summariser) both want a small, structured "give me JSON for this prompt" -//! call. Historically each built its own `reqwest::Client` and talked to a -//! local Ollama daemon directly. This module replaces that with a single -//! [`ChatProvider`] trait so the same call site can be served by either: -//! -//! - **Cloud** — `providers::router` against the OpenHuman backend with -//! the `summarization-v1` model. No local daemon required. Default for new -//! installs. -//! - **Local** — the legacy Ollama-direct path. Opt-in via -//! `memory_tree.llm_backend = "local"` in config or -//! `OPENHUMAN_MEMORY_TREE_LLM_BACKEND=local`. -//! -//! ## Why a memory-tree-local trait -//! -//! The existing top-level [`crate::openhuman::inference::provider::Provider`] trait -//! is rich (streaming, native tool calling, vision, …) and depends on the -//! agent's full conversation surface. The extractor and summariser only -//! need: -//! -//! 1. Send a (system, user) prompt pair. -//! 2. Get a JSON-shaped string back. -//! -//! Defining [`ChatProvider`] here keeps the memory_tree decoupled from -//! the agent's prompt/tool-calling stack, makes the extractor/summariser -//! trivial to mock in unit tests, and lets us route either the cloud or -//! the local backend through the same trait object. -//! -//! ## Soft-fallback contract -//! -//! Implementations of `chat_for_json` MUST NOT return `Err` for transient -//! upstream issues. Both memory_tree consumers fall back to a deterministic -//! no-op when the LLM is unavailable; bubbling the error up would abort -//! ingest cascades. Real bugs (e.g. malformed config) are still acceptable -//! `Err` cases — they should be rare and surfaced loudly. -//! -//! See [`local::OllamaChatProvider`] and [`cloud::CloudChatProvider`] for -//! the two production implementations. - -use std::sync::Arc; - -use anyhow::Result; -use async_trait::async_trait; - -use crate::openhuman::config::{Config, DEFAULT_CLOUD_LLM_MODEL}; - -pub mod cloud; -pub mod local; - -/// One pair of prompt messages handed to the chat backend. -/// -/// Keeps the surface deliberately tiny — the memory_tree's two consumers -/// both build a system prompt + a single user message. Multi-turn, -/// streaming, and tool calling are out of scope. -#[derive(Debug, Clone)] -pub struct ChatPrompt { - /// System prompt anchoring the model's role and expected output schema. - pub system: String, - /// User prompt carrying the dynamic input (the chunk text, the inputs - /// to summarise, etc.). - pub user: String, - /// Sampling temperature. Both consumers use 0.0 today (max determinism). - pub temperature: f64, - /// Diagnostic tag included in tracing logs so seal-time and admit-time - /// calls are easy to disambiguate. Stable, lowercase, no PII. - pub kind: &'static str, -} - -/// Pluggable chat surface used by the memory_tree's extractor + summariser. -/// -/// Returns the model's raw output as a string. Callers parse it themselves -/// (typically as JSON conforming to a schema embedded in the system prompt) -/// because the parsing logic is consumer-specific. -#[async_trait] -pub trait ChatProvider: Send + Sync { - /// Stable, grep-friendly name for logs. e.g. `"cloud:summarization-v1"`. - fn name(&self) -> &str; - - /// Run one chat completion and return the assistant's content, - /// constraining the model to JSON output where the wire format - /// supports it (Ollama's `format: "json"`). - /// - /// Implementations should log entry / exit at debug level under the - /// `[memory_tree::chat]` prefix. - async fn chat_for_json(&self, prompt: &ChatPrompt) -> Result; - - /// Run one chat completion and return the assistant's plain-text - /// content. Unlike [`chat_for_json`], implementations MUST NOT - /// enable any wire-level JSON-mode flag — used by the summariser - /// which emits prose, not a structured envelope. - /// - /// Default impl forwards to `chat_for_json`; providers that gate - /// JSON-mode at the wire (e.g. Ollama) override to skip it. - async fn chat_for_text(&self, prompt: &ChatPrompt) -> Result { - self.chat_for_json(prompt).await - } -} - -/// Build the [`ChatProvider`] dictated by the unified -/// `Config::workload_local_model("memory")`. -/// -/// - When that returns `None` (i.e. `memory_provider` is unset / `"cloud"`): -/// wires [`cloud::CloudChatProvider`] against the OpenHuman backend with -/// `cloud_llm_model` (defaulting to `summarization-v1`). -/// - When it returns `Some(model)` (i.e. `memory_provider = "ollama:"`): -/// wires [`local::OllamaChatProvider`] against the legacy -/// `llm_extractor_endpoint` / `llm_summariser_endpoint` (the daemon -/// endpoints stay in the `memory_tree` block — only the cloud/local -/// routing decision moves to the unified `memory_provider`). -/// -/// `consumer` is one of `"extract"` / `"summarise"` and selects the local -/// endpoint+model pair (extract uses `llm_extractor_*`, summarise uses -/// `llm_summariser_*`). For cloud both consumers share the same model. -pub fn build_chat_provider( - config: &Config, - consumer: ChatConsumer, -) -> Result> { - if let Some(routed_model) = config.workload_local_model("memory") { - let (endpoint, model, timeout_ms) = match consumer { - ChatConsumer::Extract => ( - config.memory_tree.llm_extractor_endpoint.clone(), - // Prefer the legacy per-path model for back-compat; fall back - // to the unified workload_local_model from memory_provider. - config - .memory_tree - .llm_extractor_model - .clone() - .or_else(|| Some(routed_model.clone())), - config - .memory_tree - .llm_extractor_timeout_ms - .unwrap_or(15_000), - ), - ChatConsumer::Summarise => ( - config.memory_tree.llm_summariser_endpoint.clone(), - // Same fallback for the summarise path. - config - .memory_tree - .llm_summariser_model - .clone() - .or_else(|| Some(routed_model)), - config - .memory_tree - .llm_summariser_timeout_ms - .unwrap_or(120_000), - ), - }; - log::debug!( - "[memory_tree::chat] building Local (Ollama) provider consumer={} \ - endpoint_set={} model_set={} timeout_ms={}", - consumer.as_str(), - endpoint.is_some(), - model.is_some(), - timeout_ms - ); - Ok(Arc::new(local::OllamaChatProvider::new( - endpoint, - model, - std::time::Duration::from_millis(timeout_ms), - )?)) - } else { - let model = config - .memory_tree - .cloud_llm_model - .clone() - .unwrap_or_else(|| DEFAULT_CLOUD_LLM_MODEL.to_string()); - // The `auth-profiles.json` lives next to `config.toml`, so the - // openhuman_dir is the parent of config_path. Without this the - // inner OpenHumanBackendProvider falls back to `~/.openhuman` - // and fails with "No backend session" on any workspace not - // located at the home default — the bug observed when running - // with `OPENHUMAN_WORKSPACE` pointed elsewhere. - let openhuman_dir = config.config_path.parent().map(std::path::PathBuf::from); - log::debug!( - "[memory_tree::chat] building Cloud provider consumer={} model={} \ - openhuman_dir={:?}", - consumer.as_str(), - model, - openhuman_dir - ); - Ok(Arc::new(cloud::CloudChatProvider::new( - config.api_url.clone(), - model, - openhuman_dir, - config.secrets.encrypt, - ))) - } -} - -/// Which memory-tree consumer is requesting a chat provider. Determines -/// which `llm_*_endpoint` / `llm_*_model` config fields are read in the -/// `Local` branch. Both consumers share the same cloud model. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum ChatConsumer { - /// `LlmEntityExtractor` (per-chunk NER + importance rating). - Extract, - /// `LlmSummariser` (bucket-seal summary of N children). - Summarise, -} - -impl ChatConsumer { - /// Stable wire string used in logs. - pub fn as_str(self) -> &'static str { - match self { - Self::Extract => "extract", - Self::Summarise => "summarise", - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// In-memory chat provider for unit tests. Returns a canned response - /// regardless of the prompt and counts invocations so tests can assert - /// they were exercised. - pub struct StaticChatProvider { - pub response: String, - pub calls: std::sync::atomic::AtomicUsize, - } - - impl StaticChatProvider { - pub fn new(response: impl Into) -> Self { - Self { - response: response.into(), - calls: std::sync::atomic::AtomicUsize::new(0), - } - } - } - - #[async_trait] - impl ChatProvider for StaticChatProvider { - fn name(&self) -> &str { - "test:static" - } - async fn chat_for_json(&self, _prompt: &ChatPrompt) -> Result { - self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - Ok(self.response.clone()) - } - } - - #[test] - fn build_provider_returns_cloud_when_default() { - let cfg = Config::default(); - // Default is LlmBackend::Cloud — provider construction must succeed - // without a configured local Ollama endpoint. - let provider = build_chat_provider(&cfg, ChatConsumer::Extract).unwrap(); - assert!(provider.name().contains("cloud")); - } - - #[test] - fn build_provider_returns_local_when_configured() { - // After #1710 the local-vs-cloud decision is driven by - // `memory_provider` (via `Config::workload_uses_local("memory")`), - // not the legacy `memory_tree.llm_backend` flag — so the test - // needs to set the new workload field. Endpoint + model on - // `memory_tree` are still consumed for endpoint/model resolution - // inside the local branch. - let mut cfg = Config::default(); - cfg.memory_provider = Some("ollama:qwen2.5:0.5b".into()); - cfg.memory_tree.llm_extractor_endpoint = Some("http://localhost:11434".into()); - cfg.memory_tree.llm_extractor_model = Some("qwen2.5:0.5b".into()); - let provider = build_chat_provider(&cfg, ChatConsumer::Extract).unwrap(); - assert!(provider.name().contains("ollama") || provider.name().contains("local")); - } - - #[test] - fn chat_consumer_str_round_trip() { - assert_eq!(ChatConsumer::Extract.as_str(), "extract"); - assert_eq!(ChatConsumer::Summarise.as_str(), "summarise"); - } - - #[tokio::test] - async fn static_chat_provider_returns_response_and_counts() { - let p = StaticChatProvider::new("hello"); - let prompt = ChatPrompt { - system: "sys".into(), - user: "u".into(), - temperature: 0.0, - kind: "test", - }; - assert_eq!(p.chat_for_json(&prompt).await.unwrap(), "hello"); - assert_eq!(p.calls.load(std::sync::atomic::Ordering::SeqCst), 1); - } -} diff --git a/src/openhuman/memory_tree/tree_global/README.md b/src/openhuman/memory_tree/global/README.md similarity index 100% rename from src/openhuman/memory_tree/tree_global/README.md rename to src/openhuman/memory_tree/global/README.md diff --git a/src/openhuman/memory_tree/tree_global/digest.rs b/src/openhuman/memory_tree/global/digest.rs similarity index 89% rename from src/openhuman/memory_tree/tree_global/digest.rs rename to src/openhuman/memory_tree/global/digest.rs index 02eda1579..788b6000e 100644 --- a/src/openhuman/memory_tree/tree_global/digest.rs +++ b/src/openhuman/memory_tree/global/digest.rs @@ -26,21 +26,19 @@ use chrono::{DateTime, Duration, NaiveDate, TimeZone, Utc}; use rusqlite::OptionalExtension; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::content_store::{ +use crate::openhuman::memory::score::embed::build_embedder_from_config; +use crate::openhuman::memory_store::chunks::store::with_connection; +use crate::openhuman::memory_store::content::{ atomic::stage_summary, paths::slugify_source_id, read as content_read, SummaryComposeInput, SummaryTreeKind, }; -use crate::openhuman::memory_tree::score::embed::build_embedder_from_config; -use crate::openhuman::memory_tree::store::with_connection; -use crate::openhuman::memory_tree::tree_global::registry::get_or_create_global_tree; -use crate::openhuman::memory_tree::tree_global::seal::append_daily_and_cascade; -use crate::openhuman::memory_tree::tree_global::GLOBAL_TOKEN_BUDGET; -use crate::openhuman::memory_tree::tree_source::registry::new_summary_id; -use crate::openhuman::memory_tree::tree_source::store; -use crate::openhuman::memory_tree::tree_source::summariser::{ - Summariser, SummaryContext, SummaryInput, -}; -use crate::openhuman::memory_tree::tree_source::types::{SummaryNode, Tree, TreeKind}; +use crate::openhuman::memory_store::trees::registry::get_or_create_global_tree; +use crate::openhuman::memory_store::trees::types::{SummaryNode, Tree, TreeKind}; +use crate::openhuman::memory_tree::global::seal::append_daily_and_cascade; +use crate::openhuman::memory_tree::global::GLOBAL_TOKEN_BUDGET; +use crate::openhuman::memory_tree::summarise::{summarise, SummaryContext, SummaryInput}; +use crate::openhuman::memory_tree::tree::registry::new_summary_id; +use crate::openhuman::memory_tree::tree::store; /// Outcome of a single `end_of_day_digest` call — lets the caller decide /// whether to log skip details or propagate seal counts to telemetry. @@ -65,16 +63,13 @@ pub enum DigestOutcome { /// Run an end-of-day digest for `day`, appending one L0 node to the global /// tree and cascade-sealing upward if thresholds are crossed. The -/// summariser is called once to fold the per-source material into a single -/// cross-source recap. +/// `summarise` function is called once to fold the per-source material into +/// a single cross-source recap; on failure it falls back to a deterministic +/// concat-and-truncate so the digest never aborts due to an LLM error. /// /// `day` is the calendar date in UTC the digest should cover. Callers that /// simply want "yesterday" can pass `Utc::now().date_naive() - Duration::days(1)`. -pub async fn end_of_day_digest( - config: &Config, - day: NaiveDate, - summariser: &dyn Summariser, -) -> Result { +pub async fn end_of_day_digest(config: &Config, day: NaiveDate) -> Result { let (day_start, day_end) = day_bounds_utc(day)?; log::info!( "[tree_global::digest] end_of_day_digest day={} window=[{}, {})", @@ -138,10 +133,15 @@ pub async fn end_of_day_digest( target_level: 0, // daily node lives at L0 on the global tree token_budget: GLOBAL_TOKEN_BUDGET, }; - let output = summariser - .summarise(&inputs, &ctx) - .await - .context("summariser failed during end-of-day digest")?; + let output = match summarise(config, &inputs, &ctx).await { + Ok(o) => o, + Err(e) => { + log::warn!( + "[tree_global::digest] summarise failed for day={day}: {e:#} — using fallback" + ); + crate::openhuman::memory_tree::summarise::fallback_summary(&inputs, ctx.token_budget) + } + }; // Envelope: time range is the day's bounds, score carries the max // contribution score so recall still has a ranking signal. @@ -165,7 +165,7 @@ pub async fn end_of_day_digest( // seal time, so emergent themes don't need another extractor pass // here — global is a sink; union preserves "days that mentioned X" // retrieval without an extra LLM call. See LabelStrategy in - // tree_source::bucket_seal for the full design. + // tree::bucket_seal for the full design. let mut entities_set: BTreeSet = BTreeSet::new(); let mut topics_set: BTreeSet = BTreeSet::new(); for inp in &inputs { @@ -251,10 +251,10 @@ pub async fn end_of_day_digest( &tx, &daily_clone, Some(&staged_daily), - &crate::openhuman::memory_tree::store::tree_active_signature(config), + &crate::openhuman::memory_store::chunks::store::tree_active_signature(config), )?; // Index any entities the summariser emitted (no-op under inert). - crate::openhuman::memory_tree::score::store::index_summary_entity_ids_tx( + crate::openhuman::memory::score::store::index_summary_entity_ids_tx( &tx, &daily_clone.entities, &daily_clone.id, @@ -274,7 +274,7 @@ pub async fn end_of_day_digest( ); // Append into L0 buffer + cascade-seal if thresholds crossed. - let sealed_ids = append_daily_and_cascade(config, &global, &daily, summariser).await?; + let sealed_ids = append_daily_and_cascade(config, &global, &daily).await?; Ok(DigestOutcome::Emitted { daily_id: daily.id, diff --git a/src/openhuman/memory_tree/tree_global/digest_tests.rs b/src/openhuman/memory_tree/global/digest_tests.rs similarity index 80% rename from src/openhuman/memory_tree/tree_global/digest_tests.rs rename to src/openhuman/memory_tree/global/digest_tests.rs index b990fb26d..a66cfbbf3 100644 --- a/src/openhuman/memory_tree/tree_global/digest_tests.rs +++ b/src/openhuman/memory_tree/global/digest_tests.rs @@ -3,15 +3,16 @@ //! cascade-seal trigger for weekly/monthly/yearly levels. use super::*; -use crate::openhuman::memory_tree::content_store; -use crate::openhuman::memory_tree::store::upsert_chunks; -use crate::openhuman::memory_tree::tree_source::bucket_seal::{ - append_leaf, LabelStrategy, LeafRef, +use crate::openhuman::memory::chat::{test_override, ChatProvider, StaticChatProvider}; +use crate::openhuman::memory_store::chunks::store::upsert_chunks; +use crate::openhuman::memory_store::chunks::types::{ + chunk_id, Chunk, Metadata, SourceKind, SourceRef, }; -use crate::openhuman::memory_tree::tree_source::registry::get_or_create_source_tree; -use crate::openhuman::memory_tree::tree_source::summariser::inert::InertSummariser; -use crate::openhuman::memory_tree::tree_source::types::TreeStatus; -use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; +use crate::openhuman::memory_store::content as content_store; +use crate::openhuman::memory_store::trees::types::TreeStatus; +use crate::openhuman::memory_tree::sources::registry::get_or_create_source_tree; +use crate::openhuman::memory_tree::tree::bucket_seal::{append_leaf, LabelStrategy, LeafRef}; +use std::sync::Arc; use tempfile::TempDir; /// Stage a batch of chunks to the content store so that `read_chunk_body` @@ -23,9 +24,9 @@ fn stage_test_chunks(cfg: &Config, chunks: &[Chunk]) { std::fs::create_dir_all(&content_root).expect("create content_root for test"); let staged = content_store::stage_chunks(&content_root, chunks).expect("stage_chunks for test chunks"); - crate::openhuman::memory_tree::store::with_connection(cfg, |conn| { + crate::openhuman::memory_store::chunks::store::with_connection(cfg, |conn| { let tx = conn.unchecked_transaction()?; - crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?; + crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(&tx, &staged)?; tx.commit()?; Ok(()) }) @@ -47,7 +48,7 @@ async fn seed_source_tree_with_sealed_l1(cfg: &Config, scope: &str, ts: DateTime // Use chunks with the source_tree bucket-seal mechanics so we get a // real L1 summary persisted that intersects the target day. let tree = get_or_create_source_tree(cfg, scope).unwrap(); - let summariser = InertSummariser::new(); + let provider: Arc = Arc::new(StaticChatProvider::new("test summary content")); let c1 = Chunk { id: chunk_id(SourceKind::Chat, scope, 0, "test-content"), @@ -104,12 +105,15 @@ async fn seed_source_tree_with_sealed_l1(cfg: &Config, scope: &str, ts: DateTime topics: vec![], score: 0.5, }; - append_leaf(cfg, &tree, &leaf1, &summariser, &LabelStrategy::Empty) - .await - .unwrap(); - append_leaf(cfg, &tree, &leaf2, &summariser, &LabelStrategy::Empty) - .await - .unwrap(); + test_override::with_provider(Arc::clone(&provider), async { + append_leaf(cfg, &tree, &leaf1, &LabelStrategy::Empty) + .await + .unwrap(); + append_leaf(cfg, &tree, &leaf2, &LabelStrategy::Empty) + .await + .unwrap(); + }) + .await; // 12k tokens > 10k budget → one L1 summary covering `ts`. } @@ -117,9 +121,12 @@ async fn seed_source_tree_with_sealed_l1(cfg: &Config, scope: &str, ts: DateTime async fn empty_day_returns_empty_day_outcome() { // No source trees exist yet — digest should no-op. let (_tmp, cfg) = test_config(); - let summariser = InertSummariser::new(); + let provider: Arc = Arc::new(StaticChatProvider::new("test summary content")); let day = NaiveDate::from_ymd_opt(2025, 1, 15).unwrap(); - let outcome = end_of_day_digest(&cfg, day, &summariser).await.unwrap(); + let outcome = test_override::with_provider(provider, async { + end_of_day_digest(&cfg, day).await.unwrap() + }) + .await; assert!(matches!(outcome, DigestOutcome::EmptyDay)); // No L0 nodes emitted on the global tree. @@ -130,7 +137,7 @@ async fn empty_day_returns_empty_day_outcome() { #[tokio::test] async fn populated_day_emits_one_daily_leaf() { let (_tmp, cfg) = test_config(); - let summariser = InertSummariser::new(); + let provider: Arc = Arc::new(StaticChatProvider::new("test summary content")); // Seed 3 source trees with sealed L1s on the target day. This // exercises the main cross-source path end to end. @@ -140,7 +147,10 @@ async fn populated_day_emits_one_daily_leaf() { seed_source_tree_with_sealed_l1(&cfg, "email:alice", ts).await; seed_source_tree_with_sealed_l1(&cfg, "notion:workspace", ts).await; - let outcome = end_of_day_digest(&cfg, day, &summariser).await.unwrap(); + let outcome = test_override::with_provider(Arc::clone(&provider), async { + end_of_day_digest(&cfg, day).await.unwrap() + }) + .await; let (daily_id, source_count) = match outcome { DigestOutcome::Emitted { daily_id, @@ -175,18 +185,24 @@ async fn populated_day_emits_one_daily_leaf() { #[tokio::test] async fn rerun_on_same_day_is_idempotent() { let (_tmp, cfg) = test_config(); - let summariser = InertSummariser::new(); + let provider: Arc = Arc::new(StaticChatProvider::new("test summary content")); let day = NaiveDate::from_ymd_opt(2025, 2, 3).unwrap(); let ts = day.and_hms_opt(9, 0, 0).unwrap().and_utc(); seed_source_tree_with_sealed_l1(&cfg, "slack:#eng", ts).await; - let first = end_of_day_digest(&cfg, day, &summariser).await.unwrap(); + let first = test_override::with_provider(Arc::clone(&provider), async { + end_of_day_digest(&cfg, day).await.unwrap() + }) + .await; let first_id = match first { DigestOutcome::Emitted { daily_id, .. } => daily_id, other => panic!("expected Emitted, got {other:?}"), }; - let second = end_of_day_digest(&cfg, day, &summariser).await.unwrap(); + let second = test_override::with_provider(Arc::clone(&provider), async { + end_of_day_digest(&cfg, day).await.unwrap() + }) + .await; match second { DigestOutcome::Skipped { existing_id } => assert_eq!(existing_id, first_id), other => panic!("expected Skipped on rerun, got {other:?}"), @@ -200,7 +216,7 @@ async fn rerun_on_same_day_is_idempotent() { #[tokio::test] async fn seven_days_cascade_to_weekly_seal() { let (_tmp, cfg) = test_config(); - let summariser = InertSummariser::new(); + let provider: Arc = Arc::new(StaticChatProvider::new("test summary content")); // Emit 7 daily nodes across 7 consecutive days. The 7th should // cascade to seal an L1 weekly node. @@ -212,7 +228,10 @@ async fn seven_days_cascade_to_weekly_seal() { // Fresh source scope per day keeps L1s day-specific. seed_source_tree_with_sealed_l1(&cfg, &format!("slack:#day{i}"), ts).await; - let outcome = end_of_day_digest(&cfg, day, &summariser).await.unwrap(); + let outcome = test_override::with_provider(Arc::clone(&provider), async { + end_of_day_digest(&cfg, day).await.unwrap() + }) + .await; match outcome { DigestOutcome::Emitted { sealed_ids, @@ -260,12 +279,12 @@ async fn seed_source_tree_with_labeled_l1( entities: Vec, topics: Vec, ) { - use crate::openhuman::memory_tree::score::extract::EntityKind; - use crate::openhuman::memory_tree::score::resolver::CanonicalEntity; - use crate::openhuman::memory_tree::score::store::index_entity; + use crate::openhuman::memory::score::extract::EntityKind; + use crate::openhuman::memory::score::resolver::CanonicalEntity; + use crate::openhuman::memory::score::store::index_entity; let tree = get_or_create_source_tree(cfg, scope).unwrap(); - let summariser = InertSummariser::new(); + let provider: Arc = Arc::new(StaticChatProvider::new("test summary content")); let mut chunks: Vec = Vec::new(); for seq in 0..2u32 { @@ -323,8 +342,9 @@ async fn seed_source_tree_with_labeled_l1( // Two 6k-token leaves total 12k → exceeds L0 budget → seal fires on // the second append, producing one L1 summary that unions all leaf // labels (every leaf has the same set, so dedup yields the input set). - for chunk in &chunks { - let leaf = LeafRef { + let leaves: Vec = chunks + .iter() + .map(|chunk| LeafRef { chunk_id: chunk.id.clone(), token_count: 30_000, timestamp: ts, @@ -332,23 +352,22 @@ async fn seed_source_tree_with_labeled_l1( entities: entities.clone(), topics: topics.clone(), score: 0.5, - }; - append_leaf( - cfg, - &tree, - &leaf, - &summariser, - &LabelStrategy::UnionFromChildren, - ) - .await - .unwrap(); - } + }) + .collect(); + test_override::with_provider(provider, async { + for leaf in &leaves { + append_leaf(cfg, &tree, leaf, &LabelStrategy::UnionFromChildren) + .await + .unwrap(); + } + }) + .await; } #[tokio::test] async fn daily_digest_unions_labels_from_source_summaries() { let (_tmp, cfg) = test_config(); - let summariser = InertSummariser::new(); + let provider: Arc = Arc::new(StaticChatProvider::new("test summary content")); let day = NaiveDate::from_ymd_opt(2025, 5, 1).unwrap(); let ts = day.and_hms_opt(10, 0, 0).unwrap().and_utc(); @@ -374,7 +393,10 @@ async fn daily_digest_unions_labels_from_source_summaries() { ) .await; - let outcome = end_of_day_digest(&cfg, day, &summariser).await.unwrap(); + let outcome = test_override::with_provider(Arc::clone(&provider), async { + end_of_day_digest(&cfg, day).await.unwrap() + }) + .await; let daily_id = match outcome { DigestOutcome::Emitted { daily_id, .. } => daily_id, other => panic!("expected Emitted, got {other:?}"), diff --git a/src/openhuman/memory_tree/tree_global/mod.rs b/src/openhuman/memory_tree/global/mod.rs similarity index 88% rename from src/openhuman/memory_tree/tree_global/mod.rs rename to src/openhuman/memory_tree/global/mod.rs index 30ce33df3..b8a535456 100644 --- a/src/openhuman/memory_tree/tree_global/mod.rs +++ b/src/openhuman/memory_tree/global/mod.rs @@ -24,15 +24,20 @@ //! - [`registry::get_or_create_global_tree`] — singleton (scope="global") //! - [`digest::end_of_day_digest`] — build one L0 daily node, cascade-seal //! - [`recap::recap`] — select the right level for a time window +//! +//! Persistence (registry) has moved to `memory_store::trees`. pub mod digest; pub mod recap; -pub mod registry; pub mod seal; +// Re-export persistence registry from memory_store so callers using +// tree_global::registry:: still work. +pub use crate::openhuman::memory_store::trees::registry; + +pub use crate::openhuman::memory_store::trees::get_or_create_global_tree; pub use digest::{end_of_day_digest, DigestOutcome}; pub use recap::{recap, RecapOutput}; -pub use registry::get_or_create_global_tree; /// Number of L0 (daily) nodes that seal into one L1 (weekly) node. pub const WEEKLY_SEAL_THRESHOLD: usize = 7; diff --git a/src/openhuman/memory_tree/tree_global/recap.rs b/src/openhuman/memory_tree/global/recap.rs similarity index 80% rename from src/openhuman/memory_tree/tree_global/recap.rs rename to src/openhuman/memory_tree/global/recap.rs index 000bb8a71..332edc52f 100644 --- a/src/openhuman/memory_tree/tree_global/recap.rs +++ b/src/openhuman/memory_tree/global/recap.rs @@ -22,9 +22,9 @@ use anyhow::Result; use chrono::{DateTime, Duration, Utc}; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::tree_global::registry::get_or_create_global_tree; -use crate::openhuman::memory_tree::tree_source::store; -use crate::openhuman::memory_tree::tree_source::types::SummaryNode; +use crate::openhuman::memory_store::trees::registry::get_or_create_global_tree; +use crate::openhuman::memory_store::trees::types::SummaryNode; +use crate::openhuman::memory_tree::tree::store; /// Aggregated recap returned to the caller. #[derive(Debug, Clone)] @@ -158,15 +158,16 @@ fn assemble_recap(covering: &[&SummaryNode], level: u32) -> RecapOutput { #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory_tree::content_store; - use crate::openhuman::memory_tree::store::upsert_chunks; - use crate::openhuman::memory_tree::tree_global::digest::{end_of_day_digest, DigestOutcome}; - use crate::openhuman::memory_tree::tree_source::bucket_seal::{ - append_leaf, LabelStrategy, LeafRef, + use crate::openhuman::memory::chat::{test_override, ChatProvider, StaticChatProvider}; + use crate::openhuman::memory_store::chunks::store::upsert_chunks; + use crate::openhuman::memory_store::chunks::types::{ + chunk_id, Chunk, Metadata, SourceKind, SourceRef, }; - use crate::openhuman::memory_tree::tree_source::registry::get_or_create_source_tree; - use crate::openhuman::memory_tree::tree_source::summariser::inert::InertSummariser; - use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; + use crate::openhuman::memory_store::content as content_store; + use crate::openhuman::memory_tree::global::digest::{end_of_day_digest, DigestOutcome}; + use crate::openhuman::memory_tree::sources::registry::get_or_create_source_tree; + use crate::openhuman::memory_tree::tree::bucket_seal::{append_leaf, LabelStrategy, LeafRef}; + use std::sync::Arc; use tempfile::TempDir; fn stage_test_chunks(cfg: &Config, chunks: &[Chunk]) { @@ -174,9 +175,9 @@ mod tests { std::fs::create_dir_all(&content_root).expect("create content_root for test"); let staged = content_store::stage_chunks(&content_root, chunks) .expect("stage_chunks for test chunks"); - crate::openhuman::memory_tree::store::with_connection(cfg, |conn| { + crate::openhuman::memory_store::chunks::store::with_connection(cfg, |conn| { let tx = conn.unchecked_transaction()?; - crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?; + crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(&tx, &staged)?; tx.commit()?; Ok(()) }) @@ -217,7 +218,8 @@ mod tests { async fn seed_source_l1(cfg: &Config, scope: &str, ts: DateTime) { let tree = get_or_create_source_tree(cfg, scope).unwrap(); - let summariser = InertSummariser::new(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); let c1 = Chunk { id: chunk_id(SourceKind::Chat, scope, 0, "test-content"), content: format!("c1-{scope}"), @@ -254,40 +256,33 @@ mod tests { }; upsert_chunks(cfg, &[c1.clone(), c2.clone()]).unwrap(); stage_test_chunks(cfg, &[c1.clone(), c2.clone()]); - append_leaf( - cfg, - &tree, - &LeafRef { - chunk_id: c1.id.clone(), - token_count: 30_000, - timestamp: ts, - content: c1.content.clone(), - entities: vec![], - topics: vec![], - score: 0.5, - }, - &summariser, - &LabelStrategy::Empty, - ) - .await - .unwrap(); - append_leaf( - cfg, - &tree, - &LeafRef { - chunk_id: c2.id.clone(), - token_count: 30_000, - timestamp: ts, - content: c2.content.clone(), - entities: vec![], - topics: vec![], - score: 0.5, - }, - &summariser, - &LabelStrategy::Empty, - ) - .await - .unwrap(); + let leaf1 = LeafRef { + chunk_id: c1.id.clone(), + token_count: 30_000, + timestamp: ts, + content: c1.content.clone(), + entities: vec![], + topics: vec![], + score: 0.5, + }; + let leaf2 = LeafRef { + chunk_id: c2.id.clone(), + token_count: 30_000, + timestamp: ts, + content: c2.content.clone(), + entities: vec![], + topics: vec![], + score: 0.5, + }; + test_override::with_provider(Arc::clone(&provider), async { + append_leaf(cfg, &tree, &leaf1, &LabelStrategy::Empty) + .await + .unwrap(); + append_leaf(cfg, &tree, &leaf2, &LabelStrategy::Empty) + .await + .unwrap(); + }) + .await; } #[tokio::test] @@ -295,12 +290,16 @@ mod tests { // One daily digest → recap(1 day) should return the L0 at the // correct level. let (_tmp, cfg) = test_config(); - let summariser = InertSummariser::new(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); // Use "today" so the digest's time range covers now. let day = Utc::now().date_naive(); let ts = day.and_hms_opt(12, 0, 0).unwrap().and_utc(); seed_source_l1(&cfg, "slack:#eng", ts).await; - let outcome = end_of_day_digest(&cfg, day, &summariser).await.unwrap(); + let outcome = test_override::with_provider(Arc::clone(&provider), async { + end_of_day_digest(&cfg, day).await.unwrap() + }) + .await; assert!(matches!(outcome, DigestOutcome::Emitted { .. })); let r = recap(&cfg, Duration::hours(24)) @@ -318,14 +317,18 @@ mod tests { // should fall back from level 1 to level 0 and return whatever // daily nodes exist. let (_tmp, cfg) = test_config(); - let summariser = InertSummariser::new(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); let today = Utc::now().date_naive(); let base = today - Duration::days(2); for i in 0..3 { let day = base + Duration::days(i); let ts = day.and_hms_opt(10, 0, 0).unwrap().and_utc(); seed_source_l1(&cfg, &format!("slack:#d{i}"), ts).await; - end_of_day_digest(&cfg, day, &summariser).await.unwrap(); + test_override::with_provider(Arc::clone(&provider), async { + end_of_day_digest(&cfg, day).await.unwrap() + }) + .await; } let r = recap(&cfg, Duration::days(7)) .await @@ -343,14 +346,18 @@ mod tests { // After 7 daily digests a weekly L1 exists. A 7-day recap should // return that L1 at level 1. let (_tmp, cfg) = test_config(); - let summariser = InertSummariser::new(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); let today = Utc::now().date_naive(); let base = today - Duration::days(6); for i in 0..7 { let day = base + Duration::days(i); let ts = day.and_hms_opt(10, 0, 0).unwrap().and_utc(); seed_source_l1(&cfg, &format!("slack:#w{i}"), ts).await; - end_of_day_digest(&cfg, day, &summariser).await.unwrap(); + test_override::with_provider(Arc::clone(&provider), async { + end_of_day_digest(&cfg, day).await.unwrap() + }) + .await; } let r = recap(&cfg, Duration::days(7)) .await diff --git a/src/openhuman/memory_tree/tree_global/seal.rs b/src/openhuman/memory_tree/global/seal.rs similarity index 86% rename from src/openhuman/memory_tree/tree_global/seal.rs rename to src/openhuman/memory_tree/global/seal.rs index c83e5aeb8..e10c84fba 100644 --- a/src/openhuman/memory_tree/tree_global/seal.rs +++ b/src/openhuman/memory_tree/global/seal.rs @@ -6,7 +6,7 @@ //! tree aligned to the time axis (day / week / month / year) so //! window-scoped recap queries can map a duration to a level deterministically. //! -//! Reuses Phase 3a storage primitives from `tree_source::store` without +//! Reuses Phase 3a storage primitives from `tree::store` without //! their token-budget cascade logic — all global seals route through //! `mem_tree_summaries` on both sides (children and output), since even L0 //! is a sealed summary node rather than a raw chunk. @@ -17,20 +17,20 @@ use anyhow::{Context, Result}; use chrono::{DateTime, Utc}; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::content_store::{ +use crate::openhuman::memory::score::embed::build_embedder_from_config; +use crate::openhuman::memory_store::chunks::store::with_connection; +use crate::openhuman::memory_store::content::{ atomic::stage_summary, SummaryComposeInput, SummaryTreeKind, }; -use crate::openhuman::memory_tree::score::embed::build_embedder_from_config; -use crate::openhuman::memory_tree::store::with_connection; -use crate::openhuman::memory_tree::tree_global::{ +use crate::openhuman::memory_store::trees::types::{Buffer, SummaryNode, Tree, TreeKind}; +use crate::openhuman::memory_tree::global::{ GLOBAL_TOKEN_BUDGET, MONTHLY_SEAL_THRESHOLD, WEEKLY_SEAL_THRESHOLD, YEARLY_SEAL_THRESHOLD, }; -use crate::openhuman::memory_tree::tree_source::registry::new_summary_id; -use crate::openhuman::memory_tree::tree_source::store; -use crate::openhuman::memory_tree::tree_source::summariser::{ - Summariser, SummaryContext, SummaryInput, +use crate::openhuman::memory_tree::summarise::{ + fallback_summary, summarise, SummaryContext, SummaryInput, }; -use crate::openhuman::memory_tree::tree_source::types::{Buffer, SummaryNode, Tree, TreeKind}; +use crate::openhuman::memory_tree::tree::registry::new_summary_id; +use crate::openhuman::memory_tree::tree::store; /// Hard cap on cascade depth — mirrors the source-tree constant. L0→L1→L2→L3 /// is only 3 hops so we have ample slack. @@ -46,7 +46,6 @@ pub async fn append_daily_and_cascade( config: &Config, tree: &Tree, daily_summary: &SummaryNode, - summariser: &dyn Summariser, ) -> Result> { log::debug!( "[tree_global::seal] append_daily tree_id={} daily_id={} tokens={}", @@ -64,7 +63,7 @@ pub async fn append_daily_and_cascade( daily_summary.time_range_start, )?; - cascade_seals(config, tree, summariser).await + cascade_seals(config, tree).await } /// Transactionally append a single summary id to the buffer at @@ -100,11 +99,7 @@ fn append_to_buffer( }) } -async fn cascade_seals( - config: &Config, - tree: &Tree, - summariser: &dyn Summariser, -) -> Result> { +async fn cascade_seals(config: &Config, tree: &Tree) -> Result> { let mut sealed_ids: Vec = Vec::new(); // `level` is independent of the iteration counter — it only bumps when a // seal fires, and the loop can break early if `should_seal` returns @@ -124,7 +119,7 @@ async fn cascade_seals( break; } - let summary_id = seal_one_level(config, tree, &buf, summariser).await?; + let summary_id = seal_one_level(config, tree, &buf).await?; sealed_ids.push(summary_id); level += 1; } @@ -146,12 +141,7 @@ fn should_seal(buf: &Buffer, level: u32) -> bool { !buf.is_empty() && buf.item_ids.len() >= threshold } -async fn seal_one_level( - config: &Config, - tree: &Tree, - buf: &Buffer, - summariser: &dyn Summariser, -) -> Result { +async fn seal_one_level(config: &Config, tree: &Tree, buf: &Buffer) -> Result { let level = buf.level; let target_level = level + 1; @@ -186,10 +176,17 @@ async fn seal_one_level( target_level, token_budget: GLOBAL_TOKEN_BUDGET, }; - let output = summariser - .summarise(&inputs, &ctx) - .await - .context("summariser failed during global seal")?; + let output = match summarise(config, &inputs, &ctx).await { + Ok(o) => o, + Err(e) => { + log::warn!( + "[tree_global::seal] summarise failed tree_id={} level={}: {e:#} — using fallback", + tree.id, + level + ); + fallback_summary(&inputs, ctx.token_budget) + } + }; // Global-tree summaries inherit their entity/topic labels via union // from their already-labeled inputs (source-tree summaries carry @@ -268,7 +265,7 @@ async fn seal_one_level( // Global tree scope is typically the literal "global" string. // Use it as-is for the path (slugify passes through short ascii strings unchanged). let global_scope_slug = - crate::openhuman::memory_tree::content_store::paths::slugify_source_id(&tree.scope); + crate::openhuman::memory_store::content::paths::slugify_source_id(&tree.scope); let staged_global = stage_summary( &content_root_global, &compose_input_global, @@ -311,13 +308,13 @@ async fn seal_one_level( &tx, &node, Some(&staged_global), - &crate::openhuman::memory_tree::store::tree_active_signature(config), + &crate::openhuman::memory_store::chunks::store::tree_active_signature(config), )?; // Index any entities the summariser emitted. No-op under // InertSummariser (entities stays empty by design — see // summariser/inert.rs). Becomes active when the Ollama summariser // lands and emits curated canonical ids. - crate::openhuman::memory_tree::score::store::index_summary_entity_ids_tx( + crate::openhuman::memory::score::store::index_summary_entity_ids_tx( &tx, &node.entities, &node.id, @@ -415,9 +412,10 @@ fn hydrate_summary_inputs(config: &Config, summary_ids: &[String]) -> Result (TempDir, Config) { @@ -461,7 +459,7 @@ mod tests { &tx, node, None, - &crate::openhuman::memory_tree::store::tree_active_signature(cfg), + &crate::openhuman::memory_store::chunks::store::tree_active_signature(cfg), )?; tx.commit()?; Ok(()) @@ -473,7 +471,8 @@ mod tests { async fn below_threshold_does_not_seal() { let (_tmp, cfg) = test_config(); let tree = get_or_create_global_tree(&cfg).unwrap(); - let summariser = InertSummariser::new(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); // Append 3 daily nodes — well below the 7-day weekly threshold. for i in 0..3 { @@ -483,9 +482,10 @@ mod tests { 1_700_000_000_000 + i, ); insert_daily(&cfg, &node); - let sealed = append_daily_and_cascade(&cfg, &tree, &node, &summariser) - .await - .unwrap(); + let sealed = test_override::with_provider(Arc::clone(&provider), async { + append_daily_and_cascade(&cfg, &tree, &node).await.unwrap() + }) + .await; assert!(sealed.is_empty(), "no cascade expected below threshold"); } @@ -497,7 +497,8 @@ mod tests { async fn crossing_weekly_threshold_seals_l1() { let (_tmp, cfg) = test_config(); let tree = get_or_create_global_tree(&cfg).unwrap(); - let summariser = InertSummariser::new(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); // Append exactly 7 daily nodes — should trigger one L0→L1 seal. for i in 0..WEEKLY_SEAL_THRESHOLD { @@ -507,9 +508,10 @@ mod tests { 1_700_000_000_000 + i as i64, ); insert_daily(&cfg, &node); - let sealed = append_daily_and_cascade(&cfg, &tree, &node, &summariser) - .await - .unwrap(); + let sealed = test_override::with_provider(Arc::clone(&provider), async { + append_daily_and_cascade(&cfg, &tree, &node).await.unwrap() + }) + .await; if i + 1 < WEEKLY_SEAL_THRESHOLD { assert!(sealed.is_empty(), "no seal before threshold (i={i})"); } else { @@ -540,16 +542,19 @@ mod tests { async fn append_is_idempotent_on_retry() { let (_tmp, cfg) = test_config(); let tree = get_or_create_global_tree(&cfg).unwrap(); - let summariser = InertSummariser::new(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); let node = mk_daily("summary:L0:dayA", &tree.id, 1_700_000_000_000); insert_daily(&cfg, &node); - append_daily_and_cascade(&cfg, &tree, &node, &summariser) - .await - .unwrap(); - append_daily_and_cascade(&cfg, &tree, &node, &summariser) - .await - .unwrap(); + test_override::with_provider(Arc::clone(&provider), async { + append_daily_and_cascade(&cfg, &tree, &node).await.unwrap() + }) + .await; + test_override::with_provider(Arc::clone(&provider), async { + append_daily_and_cascade(&cfg, &tree, &node).await.unwrap() + }) + .await; let buf = store::get_buffer(&cfg, &tree.id, 0).unwrap(); assert_eq!( diff --git a/src/openhuman/memory_tree/io.rs b/src/openhuman/memory_tree/io.rs new file mode 100644 index 000000000..a029cd5bf --- /dev/null +++ b/src/openhuman/memory_tree/io.rs @@ -0,0 +1,231 @@ +//! Canonical input/output types for the memory_tree module. +//! +//! memory_tree exposes two fundamental operations against any tree: +//! +//! - **Write** — append a chunk (leaf) into a tree; cascading bucket-seals +//! may produce new summary nodes at higher levels. +//! - **Read** — navigate from a node into its descendants, optionally +//! reranked by a query embedding. +//! +//! Internal mechanics (bucket_seal, flush, walk, summarise) take a mix of +//! `&Tree`, `&LeafRef`, `WalkOptions`, etc. This module defines a single +//! pair of contract types per direction so callers above memory_tree +//! (orchestrator, jobs, RPC) can talk to the module in one consistent +//! shape regardless of which tree kind they're targeting. +//! +//! These are pure contract types — no logic, no IO, no storage. They +//! compose the existing primitives from [`crate::openhuman::memory_store::trees`] +//! and the mechanics submodules; convert at the call boundary. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +use crate::openhuman::memory_store::trees::{Tree, TreeKind}; +use crate::openhuman::memory_tree::tree::bucket_seal::LeafRef; + +// ───────────────────────── Write ───────────────────────── + +/// A leaf payload ready to be appended to a tree. Mirror of [`LeafRef`] +/// with serde derives so RPC callers and job payloads can share one shape. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TreeLeafPayload { + pub chunk_id: String, + pub token_count: u32, + pub timestamp: DateTime, + pub content: String, + #[serde(default)] + pub entities: Vec, + #[serde(default)] + pub topics: Vec, + #[serde(default)] + pub score: f32, +} + +impl From<&TreeLeafPayload> for LeafRef { + fn from(p: &TreeLeafPayload) -> Self { + LeafRef { + chunk_id: p.chunk_id.clone(), + token_count: p.token_count, + timestamp: p.timestamp, + content: p.content.clone(), + entities: p.entities.clone(), + topics: p.topics.clone(), + score: p.score, + } + } +} + +impl From for TreeLeafPayload { + fn from(l: LeafRef) -> Self { + Self { + chunk_id: l.chunk_id, + token_count: l.token_count, + timestamp: l.timestamp, + content: l.content, + entities: l.entities, + topics: l.topics, + score: l.score, + } + } +} + +/// How sealed summaries should be labelled with entities/topics. Mirrors +/// [`crate::openhuman::memory_tree::tree::bucket_seal::LabelStrategy`] in a +/// serde-friendly shape. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TreeLabelStrategy { + /// Inherit entities/topics from child leaves (default). + #[default] + Inherit, + /// Re-extract from the summary text via the resolver. + Extract, + /// Leave entities/topics empty. + Empty, +} + +/// Canonical write request: "append this leaf to this tree". +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TreeWriteRequest { + /// Target tree id. Must already exist (callers go through the + /// kind-specific registry to get one). + pub tree_id: String, + /// Tree kind — informational, carried through to log lines and + /// downstream policy. Storage doesn't branch on it. + pub tree_kind: TreeKind, + /// The leaf to append. + pub leaf: TreeLeafPayload, + /// Labelling strategy applied to any summaries that seal during this + /// call. Defaults to [`TreeLabelStrategy::Inherit`]. + #[serde(default)] + pub label_strategy: TreeLabelStrategy, + /// When `true`, only stage the leaf in the L0 buffer; do NOT cascade + /// seals synchronously. Use this from job-driven pipelines where seal + /// work is enqueued separately. Default `false`. + #[serde(default)] + pub deferred: bool, +} + +/// Canonical write outcome: which buffers sealed (if any) and the summary +/// ids produced. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct TreeWriteOutcome { + /// Ids of summary nodes that sealed during this append. Empty when the + /// L0 buffer was below budget or the call was deferred. + pub new_summary_ids: Vec, + /// Set to `true` when the caller used `deferred = true` and should + /// enqueue a follow-up seal job for level 0. + pub seal_pending: bool, +} + +// ───────────────────────── Read ───────────────────────── + +/// What the caller wants out of the read. Bounds the BFS and controls +/// query-driven reranking. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TreeReadRequest { + /// Tree id. Required so the read scope is explicit even when starting + /// from a known node. + pub tree_id: String, + /// Starting node. `None` → start from the tree root. + #[serde(default)] + pub start_node_id: Option, + /// Maximum levels to descend from `start_node_id`. `0` returns an + /// empty result. + pub max_depth: u32, + /// Optional natural-language query. When `Some`, hits are reranked by + /// cosine similarity to the query embedding; hits with no stored + /// embedding sort to the bottom. When `None`, BFS order is preserved. + #[serde(default)] + pub query: Option, + /// Max hits to return. `None` → backend default. + #[serde(default)] + pub limit: Option, +} + +/// One hit returned by a tree read. Compact projection — for the full +/// SummaryNode/Chunk row use the memory_store retrieval surface directly. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TreeReadHit { + pub node_id: String, + /// `"summary"` for sealed nodes, `"chunk"` for leaves. + pub node_kind: String, + /// Level in the tree (0 = leaf, 1+ = summary). + pub level: u32, + /// Summary text or chunk content, truncated by the backend if oversize. + pub content: String, + /// Cosine similarity score when `query` was set; `0.0` otherwise. + #[serde(default)] + pub score: f32, +} + +/// Result of a tree read. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct TreeReadResult { + pub hits: Vec, + /// Total matches BEFORE `limit` truncation. + pub total: usize, + /// Echoes back the tree id the read targeted — useful for callers that + /// fan out and need to attribute hits. + pub tree_id: String, +} + +impl TreeReadResult { + pub fn empty(tree: &Tree) -> Self { + Self { + hits: Vec::new(), + total: 0, + tree_id: tree.id.clone(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::memory_store::trees::TreeStatus; + + fn sample_tree() -> Tree { + Tree { + id: "tree-1".into(), + kind: TreeKind::Source, + scope: "chat:slack:#eng".into(), + root_id: Some("root-1".into()), + max_level: 2, + status: TreeStatus::Active, + created_at: Utc::now(), + last_sealed_at: None, + } + } + + #[test] + fn tree_leaf_payload_converts_to_and_from_leaf_ref() { + let payload = TreeLeafPayload { + chunk_id: "chunk-1".into(), + token_count: 12, + timestamp: Utc::now(), + content: "hello".into(), + entities: vec!["person:alice".into()], + topics: vec!["deploy".into()], + score: 0.75, + }; + let leaf: LeafRef = (&payload).into(); + let roundtrip = TreeLeafPayload::from(leaf); + + assert_eq!(roundtrip.chunk_id, payload.chunk_id); + assert_eq!(roundtrip.token_count, payload.token_count); + assert_eq!(roundtrip.content, payload.content); + assert_eq!(roundtrip.entities, payload.entities); + assert_eq!(roundtrip.topics, payload.topics); + assert_eq!(roundtrip.score, payload.score); + } + + #[test] + fn tree_read_result_empty_copies_tree_id() { + let tree = sample_tree(); + let result = TreeReadResult::empty(&tree); + assert_eq!(result.tree_id, tree.id); + assert_eq!(result.total, 0); + assert!(result.hits.is_empty()); + } +} diff --git a/src/openhuman/memory_tree/mod.rs b/src/openhuman/memory_tree/mod.rs index d9dcaeac8..3ea7349ba 100644 --- a/src/openhuman/memory_tree/mod.rs +++ b/src/openhuman/memory_tree/mod.rs @@ -22,31 +22,28 @@ //! Phases 2-4 (#708 scoring, #709 summary trees, #710 retrieval) build on //! top of these chunks without modifying the Phase 1 surface. -pub mod canonicalize; -pub mod chat; -pub mod chunker; -pub mod content_store; -pub mod ingest; -pub mod jobs; -pub mod read_rpc; -pub mod retrieval; -pub mod rpc; -pub mod schemas; -pub mod score; -pub mod store; -pub mod summarizer; +pub mod global; +pub mod io; +pub mod sources; +pub mod summarise; pub mod tools; -pub mod tree_global; -pub mod tree_source; -pub mod tree_topic; -pub mod types; -pub mod util; +pub mod topic; +pub mod tree; +pub mod tree_runtime; -pub use retrieval::{all_retrieval_controller_schemas, all_retrieval_registered_controllers}; -pub use schemas::{ +pub use io::{ + TreeLabelStrategy, TreeLeafPayload, TreeReadHit, TreeReadRequest, TreeReadResult, + TreeWriteOutcome, TreeWriteRequest, +}; + +// Re-export controller registries — moved to memory but keep export names stable. +pub use crate::openhuman::memory::retrieval::{ + all_retrieval_controller_schemas, all_retrieval_registered_controllers, +}; +pub use crate::openhuman::memory::schema::{ all_controller_schemas as all_memory_tree_controller_schemas, all_registered_controllers as all_memory_tree_registered_controllers, }; -pub use summarizer::{ +pub use crate::openhuman::memory_tree::tree_runtime::{ all_tree_summarizer_controller_schemas, all_tree_summarizer_registered_controllers, }; diff --git a/src/openhuman/memory_tree/score/extract/mod.rs b/src/openhuman/memory_tree/score/extract/mod.rs deleted file mode 100644 index fd3f588c5..000000000 --- a/src/openhuman/memory_tree/score/extract/mod.rs +++ /dev/null @@ -1,113 +0,0 @@ -//! Entity extraction (Phase 2 / #708). -//! -//! Exposes [`EntityExtractor`] as a pluggable interface and a default -//! [`CompositeExtractor`] that runs a chain of extractors and merges their -//! output. Phase 2 ships with the mechanical regex extractor only; semantic -//! NER (GLiNER / LLM) plugs in later without changing any call sites. - -mod extractor; -pub mod llm; -pub mod regex; -pub mod types; - -use std::sync::Arc; - -use crate::openhuman::config::{Config, DEFAULT_CLOUD_LLM_MODEL}; -use crate::openhuman::memory_tree::chat::{build_chat_provider, ChatConsumer}; - -pub use extractor::{CompositeExtractor, EntityExtractor, RegexEntityExtractor}; -pub use llm::{LlmEntityExtractor, LlmExtractorConfig}; -pub use types::{EntityKind, ExtractedEntities, ExtractedEntity, ExtractedTopic}; - -/// Build the extractor used by seal handlers to label new summary nodes. -/// -/// Composition: -/// - regex extractor — always on, mechanical, near-zero cost -/// - LLM extractor with `emit_topics: true` — added when the LLM backend -/// is reachable. For `llm_backend = "cloud"` (default) that's always. For -/// `llm_backend = "local"` we still require `llm_extractor_endpoint` + -/// `_model` to be set (otherwise the legacy regex-only path stays). -/// -/// Differs from [`super::ScoringConfig::from_config`] (the chunk-admission -/// builder) in two ways: returns *just* an extractor (no thresholds / -/// weights / drop logic — none of which apply at seal time), and flips -/// `emit_topics` on so summaries surface thematic labels alongside -/// entities. Leaf-side scoring is unchanged. -pub fn build_summary_extractor(config: &Config) -> Arc { - let model = resolve_extractor_model(config); - let Some(model) = model else { - log::debug!( - "[memory_tree::extract] summary extractor: LLM model not resolvable for \ - memory_provider={:?} — using regex-only", - config.memory_provider.as_deref().unwrap_or("cloud") - ); - return Arc::new(CompositeExtractor::regex_only()); - }; - - let cfg = LlmExtractorConfig { - model: model.clone(), - emit_topics: true, - output_language: config.output_language.clone(), - ..LlmExtractorConfig::default() - }; - - let provider = match build_chat_provider(config, ChatConsumer::Extract) { - Ok(p) => p, - Err(err) => { - log::warn!( - "[memory_tree::extract] summary extractor: build_chat_provider failed: \ - {err:#} — falling back to regex-only" - ); - return Arc::new(CompositeExtractor::regex_only()); - } - }; - - log::debug!( - "[memory_tree::extract] summary extractor: regex + LLM provider={} model={} \ - emit_topics=true", - provider.name(), - model - ); - Arc::new(CompositeExtractor::new(vec![ - Box::new(RegexEntityExtractor), - Box::new(LlmEntityExtractor::new(cfg, provider)), - ])) -} - -/// Resolve the model identifier the extractor's [`ChatProvider`] should -/// target, returning `None` when the configured backend can't be served: -/// -/// - Cloud (i.e. `Config::workload_uses_local("memory")` is false): always -/// returns the configured `cloud_llm_model` or its `summarization-v1` -/// default. -/// - Local (i.e. `memory_provider = "ollama:"`): returns `Some(model)` -/// only when both `llm_extractor_endpoint` AND `llm_extractor_model` are -/// set — otherwise the legacy regex-only path engages. -pub(super) fn resolve_extractor_model(config: &Config) -> Option { - if config.workload_uses_local("memory") { - let endpoint = config - .memory_tree - .llm_extractor_endpoint - .as_deref() - .map(str::trim) - .filter(|s| !s.is_empty()); - let model = config - .memory_tree - .llm_extractor_model - .as_deref() - .map(str::trim) - .filter(|s| !s.is_empty()); - match (endpoint, model) { - (Some(_), Some(m)) => Some(m.to_string()), - _ => None, - } - } else { - Some( - config - .memory_tree - .cloud_llm_model - .clone() - .unwrap_or_else(|| DEFAULT_CLOUD_LLM_MODEL.to_string()), - ) - } -} diff --git a/src/openhuman/memory_tree/tree_source/source_file.rs b/src/openhuman/memory_tree/sources/file.rs similarity index 97% rename from src/openhuman/memory_tree/tree_source/source_file.rs rename to src/openhuman/memory_tree/sources/file.rs index a4acfc82c..2ef587135 100644 --- a/src/openhuman/memory_tree/tree_source/source_file.rs +++ b/src/openhuman/memory_tree/sources/file.rs @@ -13,7 +13,7 @@ //! one-way: nothing reads back from this file at runtime. //! //! Future direction: as more per-source state moves out of SQLite (the -//! sibling `tree_source/store.rs` rows that are naturally one-row-per +//! sibling `tree/store.rs` rows that are naturally one-row-per //! source), this file becomes the load-into-memory authority and the //! SQLite columns get retired. We keep that migration small and explicit //! by gating it behind callers; this module just owns the on-disk shape. @@ -31,8 +31,8 @@ use anyhow::{Context, Result}; use chrono::{DateTime, Utc}; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::content_store::raw::raw_source_dir; -use crate::openhuman::memory_tree::tree_source::types::{Tree, TreeKind, TreeStatus}; +use crate::openhuman::memory_store::content::raw::raw_source_dir; +use crate::openhuman::memory_store::trees::types::{Tree, TreeKind, TreeStatus}; /// Filename of the per-source registry mirror inside `raw//`. pub const SOURCE_FILE_NAME: &str = "_source.md"; diff --git a/src/openhuman/memory_tree/sources/mod.rs b/src/openhuman/memory_tree/sources/mod.rs new file mode 100644 index 000000000..216d34b34 --- /dev/null +++ b/src/openhuman/memory_tree/sources/mod.rs @@ -0,0 +1,15 @@ +//! Source-specific policy layer for the memory tree. +//! +//! This module owns the parts of the source-tree path that are not generic: +//! - [`file`] — the `_source.md` on-disk mirror (one file per ingest source) +//! - [`registry`] — `get_or_create_source_tree`: wraps the generic +//! [`crate::openhuman::memory_tree::tree::registry::get_or_create_tree`] +//! and triggers the `_source.md` write as a source-specific side-effect. +//! +//! Generic tree mechanics (storage, buffer management, bucket-seal, +//! flush, id generation) live in [`crate::openhuman::memory_tree::tree`]. + +pub mod file; +pub mod registry; + +pub use registry::get_or_create_source_tree; diff --git a/src/openhuman/memory_tree/sources/registry.rs b/src/openhuman/memory_tree/sources/registry.rs new file mode 100644 index 000000000..0394b8c98 --- /dev/null +++ b/src/openhuman/memory_tree/sources/registry.rs @@ -0,0 +1,69 @@ +//! Source-tree registry — thin wrapper around the generic +//! [`crate::openhuman::memory_tree::tree::registry::get_or_create_tree`] +//! that adds the source-specific `_source.md` on-disk mirror write after +//! every get-or-create call. + +use anyhow::Result; + +use crate::openhuman::config::Config; +use crate::openhuman::memory::util::redact::redact; +use crate::openhuman::memory_store::trees::types::{Tree, TreeKind}; +use crate::openhuman::memory_tree::sources::file; +use crate::openhuman::memory_tree::tree::registry::get_or_create_tree; + +/// Look up the source tree for `scope`, or create a new one. +/// +/// Scope format convention (Phase 3a): use the ingested chunk's +/// `metadata.source_id` verbatim, so re-ingesting the same Slack channel +/// or Gmail account keeps appending to the same tree. +/// +/// After every successful get-or-create the `_source.md` on-disk mirror +/// for this source is (re)written. The write is best-effort — a failure +/// is logged but does not abort the call. +pub fn get_or_create_source_tree(config: &Config, scope: &str) -> Result { + let scope_redacted = redact(scope); + log::debug!("[sources::registry] get_or_create_source_tree scope={scope_redacted}"); + let tree = get_or_create_tree(config, TreeKind::Source, scope)?; + if let Err(e) = file::write_source_file(config, &tree) { + log::warn!("[sources::registry] write_source_file failed scope={scope_redacted} err={e:#}"); + } + Ok(tree) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn test_config() -> (TempDir, Config) { + let tmp = TempDir::new().unwrap(); + let mut cfg = Config::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + (tmp, cfg) + } + + #[test] + fn get_or_create_is_idempotent_on_scope() { + let (_tmp, cfg) = test_config(); + let first = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); + let second = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); + assert_eq!(first.id, second.id); + assert_eq!(first.kind, TreeKind::Source); + } + + #[test] + fn different_scopes_yield_different_trees() { + let (_tmp, cfg) = test_config(); + let a = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); + let b = get_or_create_source_tree(&cfg, "gmail:user@example.com").unwrap(); + assert_ne!(a.id, b.id); + } + + #[test] + fn writes_source_file_on_create() { + let (_tmp, cfg) = test_config(); + let tree = get_or_create_source_tree(&cfg, "gmail:user@example.com").unwrap(); + let path = file::source_file_path(&cfg, &tree.scope); + assert!(path.exists(), "expected _source.md at {}", path.display()); + } +} diff --git a/src/openhuman/memory_tree/summarise.rs b/src/openhuman/memory_tree/summarise.rs new file mode 100644 index 000000000..11a0657c9 --- /dev/null +++ b/src/openhuman/memory_tree/summarise.rs @@ -0,0 +1,254 @@ +//! Memory-tree summariser: fold N items into one parent summary via a +//! single LLM call. +//! +//! This module replaces the previous trait-based summariser ladder +//! (`Summariser` + `InertSummariser` + `LlmSummariser`) with one plain +//! `async fn`. Callers pass inputs + context + config and get back +//! either a [`SummaryOutput`] or an error. Resilience (retry, graceful +//! degradation) is the caller's responsibility — see +//! [`fallback_summary`] for the deterministic concat-and-truncate +//! helper used by seal cascades that must never abort. +//! +//! The structured-facet-extraction side-channel that the old summariser +//! carried has been removed from this layer; facet extraction is the +//! `learning` domain's job and runs independently. + +use anyhow::{Context, Result}; +use chrono::{DateTime, Utc}; + +use crate::openhuman::config::Config; +use crate::openhuman::memory::chat::{build_chat_provider, ChatPrompt}; +use crate::openhuman::memory_store::chunks::types::approx_token_count; +use crate::openhuman::memory_store::trees::types::TreeKind; + +/// Hard cap on summariser output length (in approximate tokens). Sized +/// to fit the downstream embedder (`nomic-embed-text-v1.5`, 8192-token +/// input ceiling) with headroom for tokenizer drift. +const MAX_SUMMARY_OUTPUT_TOKENS: u32 = 5_000; + +/// Context window assumed for the model. Sized for the cloud +/// summariser's 120k-token window with headroom; used as the divisor +/// in the per-input clamp so the joined prompt body stays under it +/// at upper-level seals where many children fold together. +const NUM_CTX_TOKENS: u32 = 60_000; + +/// Tokens reserved for system prompt + envelope overhead + tokenizer +/// drift between our 4-chars/token heuristic and the model's tokenizer. +const OVERHEAD_RESERVE_TOKENS: u32 = 2_048; + +/// One contribution being folded — a raw leaf at L0→L1, or a +/// lower-level summary at L_n→L_{n+1}. +#[derive(Clone, Debug)] +pub struct SummaryInput { + pub id: String, + pub content: String, + pub token_count: u32, + pub entities: Vec, + pub topics: Vec, + pub time_range_start: DateTime, + pub time_range_end: DateTime, + pub score: f32, +} + +/// Per-seal context — lets logs identify which tree is being sealed +/// without threading config globally. +#[derive(Clone, Debug)] +pub struct SummaryContext<'a> { + pub tree_id: &'a str, + pub tree_kind: TreeKind, + pub target_level: u32, + pub token_budget: u32, +} + +/// Output of a summarise call. +#[derive(Clone, Debug)] +pub struct SummaryOutput { + pub content: String, + pub token_count: u32, + /// Always emitted empty by [`summarise`]. Canonical entity ids are + /// populated separately by the entity extractor; rolling up children's + /// labels mechanically is anti-pattern (see prior `InertSummariser` + /// design note). + pub entities: Vec, + pub topics: Vec, +} + +/// Fold `inputs` into a single summary by making one chat-provider call. +/// +/// Returns `Err` on provider build failure, network failure, or empty +/// upstream response. Callers that must not abort (e.g. seal cascades) +/// should match on the error and fall back to [`fallback_summary`]. +pub async fn summarise( + config: &Config, + inputs: &[SummaryInput], + ctx: &SummaryContext<'_>, +) -> Result { + let effective_budget = ctx.token_budget.min(MAX_SUMMARY_OUTPUT_TOKENS); + let per_input_cap = if inputs.is_empty() { + 0 + } else { + NUM_CTX_TOKENS + .saturating_sub(effective_budget) + .saturating_sub(OVERHEAD_RESERVE_TOKENS) + / inputs.len() as u32 + }; + + let body = build_user_prompt(inputs, per_input_cap); + if body.trim().is_empty() { + return Ok(SummaryOutput { + content: String::new(), + token_count: 0, + entities: Vec::new(), + topics: Vec::new(), + }); + } + + let provider = + build_chat_provider(config).context("memory_tree::summarise: build chat provider")?; + + let prompt = ChatPrompt { + system: system_prompt(effective_budget, config.output_language.as_deref()), + user: body, + temperature: 0.0, + kind: "memory_tree::summarise", + }; + + log::debug!( + "[memory_tree::summarise] provider={} tree_id={} level={} inputs={} budget={}", + provider.name(), + ctx.tree_id, + ctx.target_level, + inputs.len(), + ctx.token_budget, + ); + + let raw = provider + .chat_for_text(&prompt) + .await + .with_context(|| format!("memory_tree::summarise: provider={}", provider.name()))?; + + let (content, token_count) = clamp_to_budget(raw.trim(), effective_budget); + log::debug!( + "[memory_tree::summarise] sealed tree_id={} level={} inputs={} tokens={}", + ctx.tree_id, + ctx.target_level, + inputs.len(), + token_count, + ); + + Ok(SummaryOutput { + content, + token_count, + entities: Vec::new(), + topics: Vec::new(), + }) +} + +/// Deterministic, dependency-free summary — concatenate inputs with a +/// provenance prefix and truncate to budget. Used by seal cascades when +/// [`summarise`] returns an error and the cascade must still produce a +/// parent row (replaces the old `InertSummariser` soft-fallback role). +pub fn fallback_summary(inputs: &[SummaryInput], budget: u32) -> SummaryOutput { + const PROVENANCE_PREFIX: &str = "— "; + let mut parts: Vec = Vec::with_capacity(inputs.len()); + for inp in inputs { + let trimmed = inp.content.trim(); + if trimmed.is_empty() { + continue; + } + parts.push(format!("{PROVENANCE_PREFIX}{trimmed}")); + } + let joined = parts.join("\n\n"); + let (content, token_count) = clamp_to_budget(&joined, budget); + SummaryOutput { + content, + token_count, + entities: Vec::new(), + topics: Vec::new(), + } +} + +fn build_user_prompt(inputs: &[SummaryInput], per_input_cap_tokens: u32) -> String { + let mut out = String::new(); + for inp in inputs { + let trimmed = inp.content.trim(); + if trimmed.is_empty() { + continue; + } + let (clamped, _) = clamp_to_budget(trimmed, per_input_cap_tokens); + if !out.is_empty() { + out.push_str("\n\n"); + } + out.push_str(&format!("[{}]\n{clamped}", inp.id)); + } + out +} + +fn system_prompt(budget: u32, output_language: Option<&str>) -> String { + let lang_line = match output_language { + Some(lang) if !lang.trim().is_empty() => { + format!("\nWrite the summary in {lang}.") + } + _ => String::new(), + }; + format!( + "You are folding multiple notes into one compact summary.\n\ + Aim for ~{budget} tokens or fewer. Capture key facts, decisions, and entities.\n\ + Output only the summary prose — no preamble, no JSON, no markdown headings.{lang_line}" + ) +} + +fn clamp_to_budget(text: &str, budget: u32) -> (String, u32) { + let initial = approx_token_count(text); + if initial <= budget { + return (text.to_string(), initial); + } + let char_ceiling = (budget as usize).saturating_mul(4); + let truncated: String = text.chars().take(char_ceiling).collect(); + let tokens = approx_token_count(&truncated); + (truncated, tokens) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample_input(id: &str, content: &str) -> SummaryInput { + let ts = Utc::now(); + SummaryInput { + id: id.to_string(), + content: content.to_string(), + token_count: approx_token_count(content), + entities: Vec::new(), + topics: Vec::new(), + time_range_start: ts, + time_range_end: ts, + score: 0.5, + } + } + + #[test] + fn fallback_concatenates_with_provenance_prefix() { + let inputs = vec![sample_input("a", "hello"), sample_input("b", "world")]; + let out = fallback_summary(&inputs, 10_000); + assert!(out.content.contains("hello")); + assert!(out.content.contains("world")); + assert!(out.content.contains("— ")); + assert!(out.entities.is_empty()); + } + + #[test] + fn fallback_truncates_at_budget() { + let inputs = vec![sample_input("a", &"x".repeat(1000))]; + let out = fallback_summary(&inputs, 5); + assert!(out.token_count <= 6); + } + + #[test] + fn fallback_skips_blank_inputs() { + let inputs = vec![sample_input("a", " "), sample_input("b", "kept")]; + let out = fallback_summary(&inputs, 10_000); + assert!(out.content.contains("kept")); + assert_eq!(out.content.matches("— ").count(), 1); + } +} diff --git a/src/openhuman/memory_tree/summarizer/ops.rs b/src/openhuman/memory_tree/summarizer/ops.rs deleted file mode 100644 index 14bd393bf..000000000 --- a/src/openhuman/memory_tree/summarizer/ops.rs +++ /dev/null @@ -1,192 +0,0 @@ -//! RPC operation wrappers for the tree summarizer. - -use chrono::{DateTime, Utc}; -use serde_json::{json, Value}; - -use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::summarizer::{engine, store, types::*}; -use crate::rpc::RpcOutcome; - -/// Append raw content to the ingestion buffer. -pub async fn tree_summarizer_ingest( - config: &Config, - namespace: &str, - content: &str, - timestamp: Option>, - metadata: Option<&Value>, -) -> Result, String> { - store::validate_namespace(namespace)?; - if content.trim().is_empty() { - return Err("content must not be empty".to_string()); - } - - let ts = timestamp.unwrap_or_else(Utc::now); - let path = store::buffer_write(config, namespace.trim(), content, &ts, metadata) - .map_err(|e| format!("buffer write failed: {e}"))?; - - Ok(RpcOutcome::single_log( - json!({ - "buffered": true, - "namespace": namespace.trim(), - "timestamp": ts.to_rfc3339(), - "tokens": estimate_tokens(content), - "path": path.display().to_string(), - "has_metadata": metadata.is_some(), - }), - format!("content buffered for namespace '{}'", namespace.trim()), - )) -} - -/// Trigger the summarization job for a namespace (drain buffer + summarize + propagate). -pub async fn tree_summarizer_run( - config: &Config, - namespace: &str, -) -> Result, String> { - store::validate_namespace(namespace)?; - - let provider = create_provider(config)?; - let ts = Utc::now(); - - match engine::run_summarization(config, provider.as_ref(), namespace.trim(), ts).await { - Ok(Some(node)) => Ok(RpcOutcome::single_log( - serde_json::to_value(&node).map_err(|e| e.to_string())?, - format!( - "summarization completed for '{}': node {} ({} tokens)", - namespace.trim(), - node.node_id, - node.token_count - ), - )), - Ok(None) => Ok(RpcOutcome::single_log( - json!({ "skipped": true, "reason": "no buffered data" }), - format!( - "summarization skipped for '{}': no buffered data", - namespace.trim() - ), - )), - Err(e) => Err(format!("summarization failed: {e:#}")), - } -} - -/// Query the tree at a specific node or level. -pub async fn tree_summarizer_query( - config: &Config, - namespace: &str, - node_id: Option<&str>, -) -> Result, String> { - store::validate_namespace(namespace)?; - - let target_id = node_id.unwrap_or("root"); - store::validate_node_id(target_id)?; - - let node = store::read_node(config, namespace.trim(), target_id) - .map_err(|e| format!("read node: {e}"))? - .ok_or_else(|| { - format!( - "node '{}' not found in namespace '{}'", - target_id, - namespace.trim() - ) - })?; - - let children = store::read_children(config, namespace.trim(), target_id) - .map_err(|e| format!("read children: {e}"))?; - - let result = QueryResult { node, children }; - Ok(RpcOutcome::single_log( - serde_json::to_value(&result).map_err(|e| e.to_string())?, - format!( - "queried node '{}' in namespace '{}'", - target_id, - namespace.trim() - ), - )) -} - -/// Get tree status/metadata for a namespace. -pub async fn tree_summarizer_status( - config: &Config, - namespace: &str, -) -> Result, String> { - store::validate_namespace(namespace)?; - - let status = - store::get_tree_status(config, namespace.trim()).map_err(|e| format!("get status: {e}"))?; - - Ok(RpcOutcome::single_log( - serde_json::to_value(&status).map_err(|e| e.to_string())?, - format!("tree status for namespace '{}'", namespace.trim()), - )) -} - -/// Rebuild the entire tree from hour leaves (background task). -pub async fn tree_summarizer_rebuild( - config: &Config, - namespace: &str, -) -> Result, String> { - store::validate_namespace(namespace)?; - - let provider = create_provider(config)?; - - let status = engine::rebuild_tree(config, provider.as_ref(), namespace.trim()) - .await - .map_err(|e| format!("rebuild failed: {e:#}"))?; - - Ok(RpcOutcome::single_log( - serde_json::to_value(&status).map_err(|e| e.to_string())?, - format!( - "tree rebuilt for '{}': {} nodes", - namespace.trim(), - status.total_nodes - ), - )) -} - -// ── Helper ───────────────────────────────────────────────────────────── - -fn create_provider( - config: &Config, -) -> Result, String> { - // Tree summarization runs exclusively on local AI to keep memory - // processing private and offline — no backend calls. - if !config.local_ai.runtime_enabled { - return Err("tree summarizer requires local_ai to be enabled in config".to_string()); - } - create_local_ai_provider(config) -} - -/// Create a provider backed by the local Ollama instance for summarization, -/// wrapped in `ReliableProvider` for retry/backoff on transient failures. -fn create_local_ai_provider( - config: &Config, -) -> Result, String> { - use crate::openhuman::inference::local::OLLAMA_BASE_URL; - use crate::openhuman::inference::provider::compatible::{AuthStyle, OpenAiCompatibleProvider}; - use crate::openhuman::inference::provider::reliable::ReliableProvider; - - let base_url = format!("{}/v1", OLLAMA_BASE_URL); - let inner = OpenAiCompatibleProvider::new_no_responses_fallback( - "ollama-local", - &base_url, - None, - AuthStyle::None, - ); - - let providers: Vec<( - String, - Box, - )> = vec![("ollama-local".to_string(), Box::new(inner))]; - let reliable = ReliableProvider::new( - providers, - config.reliability.provider_retries, - config.reliability.provider_backoff_ms, - ); - - tracing::debug!( - "[tree_summarizer] using local Ollama provider at {} with model '{}'", - base_url, - config.local_ai.chat_model_id - ); - - Ok(Box::new(reliable)) -} diff --git a/src/openhuman/memory_tree/tools/drill_down.rs b/src/openhuman/memory_tree/tools/drill_down.rs index 04626d2e8..5ea301fe6 100644 --- a/src/openhuman/memory_tree/tools/drill_down.rs +++ b/src/openhuman/memory_tree/tools/drill_down.rs @@ -1,6 +1,6 @@ use crate::openhuman::config::rpc as config_rpc; -use crate::openhuman::memory_tree::retrieval; -use crate::openhuman::memory_tree::retrieval::rpc::DrillDownRequest; +use crate::openhuman::memory::retrieval; +use crate::openhuman::memory::retrieval::rpc::DrillDownRequest; use crate::openhuman::tools::traits::{Tool, ToolResult}; use async_trait::async_trait; use serde_json::json; @@ -76,3 +76,141 @@ impl Tool for MemoryTreeDrillDownTool { Ok(ToolResult::success(json)) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::OsString; + + use tempfile::TempDir; + + use crate::openhuman::config::{Config, TEST_ENV_LOCK}; + use crate::openhuman::tools::traits::Tool; + use serde_json::json; + + struct WorkspaceEnvGuard { + _lock: std::sync::MutexGuard<'static, ()>, + previous: Option, + } + + impl WorkspaceEnvGuard { + fn set(path: &std::path::Path) -> Self { + let lock = TEST_ENV_LOCK.lock().unwrap_or_else(|err| err.into_inner()); + let previous = std::env::var_os("OPENHUMAN_WORKSPACE"); + std::env::set_var("OPENHUMAN_WORKSPACE", path); + Self { + _lock: lock, + previous, + } + } + } + + impl Drop for WorkspaceEnvGuard { + fn drop(&mut self) { + if let Some(previous) = self.previous.as_ref() { + std::env::set_var("OPENHUMAN_WORKSPACE", previous); + } else { + std::env::remove_var("OPENHUMAN_WORKSPACE"); + } + } + } + + async fn isolated_config(tmp: &TempDir) -> (WorkspaceEnvGuard, Config) { + let guard = WorkspaceEnvGuard::set(tmp.path()); + let config = Config::load_or_init().await.expect("load config"); + (guard, config) + } + + #[test] + fn parameters_schema_requires_node_id() { + let tool = MemoryTreeDrillDownTool; + let schema = tool.parameters_schema(); + assert_eq!(schema["required"], json!(["node_id"])); + assert_eq!(schema["properties"]["max_depth"]["minimum"], 1); + } + + #[test] + fn drill_down_request_deserializes_optional_fields() { + let req: DrillDownRequest = serde_json::from_value(json!({ + "node_id": "summary-1", + "max_depth": 2, + "query": "deployment blockers", + "limit": 7 + })) + .unwrap(); + assert_eq!(req.node_id, "summary-1"); + assert_eq!(req.max_depth, Some(2)); + assert_eq!(req.query.as_deref(), Some("deployment blockers")); + assert_eq!(req.limit, Some(7)); + } + + #[tokio::test] + async fn execute_rejects_missing_node_id() { + let tool = MemoryTreeDrillDownTool; + let err = tool + .execute(json!({})) + .await + .expect_err("missing node_id should fail"); + assert!(err + .to_string() + .contains("invalid arguments for memory_tree_drill_down")); + } + + #[tokio::test] + async fn execute_rejects_zero_max_depth() { + let tool = MemoryTreeDrillDownTool; + let err = tool + .execute(json!({ + "node_id": "summary-1", + "max_depth": 0 + })) + .await + .expect_err("max_depth=0 should fail at tool boundary"); + assert!(err.to_string().contains("max_depth must be >= 1")); + } + + #[tokio::test] + async fn execute_success_path_returns_empty_json_array_for_isolated_workspace() { + let tmp = TempDir::new().expect("tempdir"); + let (_workspace, cfg) = isolated_config(&tmp).await; + let tool = MemoryTreeDrillDownTool; + let result = tool + .execute(json!({ + "node_id": "summary-does-not-exist", + "max_depth": 1 + })) + .await + .expect("valid drill_down request should succeed in isolated workspace"); + assert!(!result.is_error); + let payload = result.text(); + let parsed: serde_json::Value = + serde_json::from_str(&payload).expect("result should be valid json"); + assert!( + parsed.is_array(), + "drill_down should serialize a JSON array" + ); + assert_eq!(parsed, json!([])); + + let direct = retrieval::drill_down(&cfg, "summary-does-not-exist", 1, None, None) + .await + .expect("direct drill_down on empty workspace"); + assert!(direct.is_empty()); + } + + #[tokio::test] + async fn execute_accepts_query_and_limit_together() { + let tmp = TempDir::new().expect("tempdir"); + let (_workspace, _cfg) = isolated_config(&tmp).await; + let tool = MemoryTreeDrillDownTool; + let result = tool + .execute(json!({ + "node_id": "summary-does-not-exist", + "max_depth": 2, + "query": "deployment blockers", + "limit": 5 + })) + .await + .expect("query+limit drill_down should succeed"); + assert!(!result.is_error); + } +} diff --git a/src/openhuman/memory_tree/tools/fetch_leaves.rs b/src/openhuman/memory_tree/tools/fetch_leaves.rs index e87bc355f..193469fe1 100644 --- a/src/openhuman/memory_tree/tools/fetch_leaves.rs +++ b/src/openhuman/memory_tree/tools/fetch_leaves.rs @@ -1,6 +1,6 @@ use crate::openhuman::config::rpc as config_rpc; -use crate::openhuman::memory_tree::retrieval; -use crate::openhuman::memory_tree::retrieval::rpc::FetchLeavesRequest; +use crate::openhuman::memory::retrieval; +use crate::openhuman::memory::retrieval::rpc::FetchLeavesRequest; use crate::openhuman::tools::traits::{Tool, ToolResult}; use async_trait::async_trait; use serde_json::json; @@ -66,3 +66,144 @@ impl Tool for MemoryTreeFetchLeavesTool { Ok(ToolResult::success(json)) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::OsString; + + use tempfile::TempDir; + + use crate::openhuman::config::{Config, TEST_ENV_LOCK}; + use crate::openhuman::tools::traits::Tool; + use serde_json::json; + + struct WorkspaceEnvGuard { + _lock: std::sync::MutexGuard<'static, ()>, + previous: Option, + } + + impl WorkspaceEnvGuard { + fn set(path: &std::path::Path) -> Self { + let lock = TEST_ENV_LOCK.lock().unwrap_or_else(|err| err.into_inner()); + let previous = std::env::var_os("OPENHUMAN_WORKSPACE"); + std::env::set_var("OPENHUMAN_WORKSPACE", path); + Self { + _lock: lock, + previous, + } + } + } + + impl Drop for WorkspaceEnvGuard { + fn drop(&mut self) { + if let Some(previous) = self.previous.as_ref() { + std::env::set_var("OPENHUMAN_WORKSPACE", previous); + } else { + std::env::remove_var("OPENHUMAN_WORKSPACE"); + } + } + } + + async fn isolated_config(tmp: &TempDir) -> (WorkspaceEnvGuard, Config) { + let guard = WorkspaceEnvGuard::set(tmp.path()); + let config = Config::load_or_init().await.expect("load config"); + (guard, config) + } + + #[test] + fn parameters_schema_requires_chunk_ids() { + let tool = MemoryTreeFetchLeavesTool; + let schema = tool.parameters_schema(); + assert_eq!(schema["required"], json!(["chunk_ids"])); + assert_eq!(schema["properties"]["chunk_ids"]["type"], "array"); + } + + #[test] + fn max_chunk_ids_per_call_matches_description() { + assert_eq!(MAX_CHUNK_IDS_PER_CALL, 20); + } + + #[test] + fn request_slice_is_truncated_to_cap() { + let ids: Vec = (0..25).map(|i| format!("chunk-{i}")).collect(); + let take = ids.len().min(MAX_CHUNK_IDS_PER_CALL); + assert_eq!(take, 20); + assert_eq!(ids[..take].len(), 20); + assert_eq!(ids[..take].first().map(String::as_str), Some("chunk-0")); + assert_eq!(ids[..take].last().map(String::as_str), Some("chunk-19")); + } + + #[tokio::test] + async fn execute_rejects_missing_chunk_ids() { + let tool = MemoryTreeFetchLeavesTool; + let err = tool + .execute(json!({})) + .await + .expect_err("missing chunk_ids should fail"); + assert!(err + .to_string() + .contains("invalid arguments for memory_tree_fetch_leaves")); + } + + #[tokio::test] + async fn execute_rejects_wrong_type_for_chunk_ids() { + let tool = MemoryTreeFetchLeavesTool; + let err = tool + .execute(json!({"chunk_ids": "not-an-array"})) + .await + .expect_err("wrong chunk_ids type should fail"); + assert!(err + .to_string() + .contains("invalid arguments for memory_tree_fetch_leaves")); + } + + #[tokio::test] + async fn execute_success_path_returns_empty_json_array_for_isolated_workspace() { + let tmp = TempDir::new().expect("tempdir"); + let (_workspace, cfg) = isolated_config(&tmp).await; + let tool = MemoryTreeFetchLeavesTool; + let result = tool + .execute(json!({ + "chunk_ids": ["chunk-does-not-exist-1", "chunk-does-not-exist-2"] + })) + .await + .expect("valid fetch_leaves request should succeed in isolated workspace"); + assert!(!result.is_error); + let payload = result.text(); + let parsed: serde_json::Value = + serde_json::from_str(&payload).expect("result should be valid json"); + assert!( + parsed.is_array(), + "fetch_leaves should serialize a JSON array" + ); + assert_eq!(parsed, json!([])); + + let direct = retrieval::fetch_leaves( + &cfg, + &[ + "chunk-does-not-exist-1".to_string(), + "chunk-does-not-exist-2".to_string(), + ], + ) + .await + .expect("direct fetch_leaves on empty workspace"); + assert!(direct.is_empty()); + } + + #[tokio::test] + async fn execute_truncates_requests_to_twenty_ids() { + let tmp = TempDir::new().expect("tempdir"); + let (_workspace, _cfg) = isolated_config(&tmp).await; + let tool = MemoryTreeFetchLeavesTool; + let ids: Vec = (0..25).map(|i| format!("chunk-{i}")).collect(); + let result = tool + .execute(json!({ "chunk_ids": ids })) + .await + .expect("over-cap request should still succeed"); + assert!(!result.is_error); + let parsed: serde_json::Value = + serde_json::from_str(&result.text()).expect("result should be valid json"); + assert_eq!(parsed, json!([])); + } +} diff --git a/src/openhuman/memory_tree/tools/ingest_document.rs b/src/openhuman/memory_tree/tools/ingest_document.rs index da41d6415..ad69ae56a 100644 --- a/src/openhuman/memory_tree/tools/ingest_document.rs +++ b/src/openhuman/memory_tree/tools/ingest_document.rs @@ -1,7 +1,7 @@ use crate::openhuman::config::rpc as config_rpc; -use crate::openhuman::memory_tree::canonicalize::document::DocumentInput; -use crate::openhuman::memory_tree::rpc; -use crate::openhuman::memory_tree::types::SourceKind; +use crate::openhuman::memory::tree_rpc as rpc; +use crate::openhuman::memory_store::chunks::types::SourceKind; +use crate::openhuman::memory_sync::canonicalize::document::DocumentInput; use crate::openhuman::tools::traits::{Tool, ToolResult}; use async_trait::async_trait; use chrono::Utc; @@ -139,3 +139,244 @@ impl Tool for MemoryTreeIngestDocumentTool { ))) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::OsString; + + use tempfile::TempDir; + + use crate::openhuman::config::{Config, TEST_ENV_LOCK}; + use crate::openhuman::memory_store::chunks::types::SourceRef; + use crate::openhuman::tools::traits::Tool; + use serde_json::json; + + struct WorkspaceEnvGuard { + _lock: std::sync::MutexGuard<'static, ()>, + previous: Option, + } + + impl WorkspaceEnvGuard { + fn set(path: &std::path::Path) -> Self { + let lock = TEST_ENV_LOCK.lock().unwrap_or_else(|err| err.into_inner()); + let previous = std::env::var_os("OPENHUMAN_WORKSPACE"); + std::env::set_var("OPENHUMAN_WORKSPACE", path); + Self { + _lock: lock, + previous, + } + } + } + + impl Drop for WorkspaceEnvGuard { + fn drop(&mut self) { + if let Some(previous) = self.previous.as_ref() { + std::env::set_var("OPENHUMAN_WORKSPACE", previous); + } else { + std::env::remove_var("OPENHUMAN_WORKSPACE"); + } + } + } + + async fn isolated_config(tmp: &TempDir) -> (WorkspaceEnvGuard, Config) { + let guard = WorkspaceEnvGuard::set(tmp.path()); + let config = Config::load_or_init().await.expect("load config"); + (guard, config) + } + + #[test] + fn parameters_schema_requires_title_body_and_source_id() { + let tool = MemoryTreeIngestDocumentTool; + let schema = tool.parameters_schema(); + assert_eq!(schema["required"], json!(["title", "body", "source_id"])); + assert_eq!(schema["properties"]["provider"]["type"], "string"); + } + + #[test] + fn missing_required_fields_produce_none_via_json_accessors() { + let value = json!({ + "title": "Doc title", + "body": "Body" + }); + assert_eq!(value.get("source_id").and_then(|v| v.as_str()), None); + } + + #[test] + fn source_kind_document_string_is_expected() { + assert_eq!(SourceKind::Document.as_str(), "document"); + } + + #[tokio::test] + async fn execute_rejects_missing_title_before_config_load() { + let tool = MemoryTreeIngestDocumentTool; + let err = tool + .execute(json!({ + "body": "Body text", + "source_id": "doc-1" + })) + .await + .expect_err("missing title should fail"); + assert!(err + .to_string() + .contains("ingest_document: missing required field `title`")); + } + + #[tokio::test] + async fn execute_rejects_missing_body_before_config_load() { + let tool = MemoryTreeIngestDocumentTool; + let err = tool + .execute(json!({ + "title": "Doc title", + "source_id": "doc-1" + })) + .await + .expect_err("missing body should fail"); + assert!(err + .to_string() + .contains("ingest_document: missing required field `body`")); + } + + #[tokio::test] + async fn execute_rejects_missing_source_id_before_config_load() { + let tool = MemoryTreeIngestDocumentTool; + let err = tool + .execute(json!({ + "title": "Doc title", + "body": "Body text" + })) + .await + .expect_err("missing source_id should fail"); + assert!(err + .to_string() + .contains("ingest_document: missing required field `source_id`")); + } + + #[tokio::test] + async fn execute_rejects_blank_required_fields() { + let tool = MemoryTreeIngestDocumentTool; + let result = tool + .execute(json!({ + "title": " ", + "body": "Body text", + "source_id": "doc-1" + })) + .await + .expect("blank title should return ToolResult error, not anyhow failure"); + assert!(result.is_error); + assert_eq!( + result.text(), + "ingest_document: title, body, and source_id must be non-empty" + ); + + let result = tool + .execute(json!({ + "title": "Doc title", + "body": " ", + "source_id": "doc-1" + })) + .await + .expect("blank body should return ToolResult error"); + assert!(result.is_error); + + let result = tool + .execute(json!({ + "title": "Doc title", + "body": "Body text", + "source_id": " " + })) + .await + .expect("blank source_id should return ToolResult error"); + assert!(result.is_error); + } + + #[tokio::test] + async fn execute_success_path_roundtrips_document_chunk() { + let tmp = TempDir::new().expect("tempdir"); + let (_workspace, cfg) = isolated_config(&tmp).await; + let tool = MemoryTreeIngestDocumentTool; + let result = tool + .execute(json!({ + "title": "Doc title", + "body": "Body text with a memorable launch detail.", + "source_id": "doc-1", + "provider": "web", + "source_ref": "https://example.test/doc-1", + "owner": "owner-1" + })) + .await + .expect("valid request should succeed in the isolated test environment"); + assert!(!result.is_error); + let text = result.text(); + assert!( + text.contains("Ingested document \"Doc title\" as source_id=doc-1."), + "unexpected success payload: {text}" + ); + + let listed = rpc::list_chunks_rpc( + &cfg, + rpc::ListChunksRequest { + source_kind: Some("document".into()), + source_id: Some("doc-1".into()), + owner: Some("owner-1".into()), + limit: Some(10), + ..Default::default() + }, + ) + .await + .expect("list chunks after tool execute") + .value + .chunks; + assert_eq!(listed.len(), 1); + assert!( + listed[0] + .content + .contains("Body text with a memorable launch detail."), + "stored chunk missing document body: {}", + listed[0].content + ); + assert_eq!(listed[0].metadata.owner, "owner-1"); + assert_eq!( + listed[0].metadata.source_ref, + Some(SourceRef::new("https://example.test/doc-1")) + ); + } + + #[tokio::test] + async fn execute_duplicate_source_id_reports_zero_new_chunks() { + let tmp = TempDir::new().expect("tempdir"); + let (_workspace, cfg) = isolated_config(&tmp).await; + let tool = MemoryTreeIngestDocumentTool; + let args = json!({ + "title": "Doc title", + "body": "Body text", + "source_id": "doc-dup" + }); + + let first = tool.execute(args.clone()).await.expect("first execute"); + let second = tool.execute(args).await.expect("second execute"); + assert!(!first.is_error); + assert!(!second.is_error); + assert!(first.text().contains("1 chunks created and indexed.")); + assert!(second.text().contains("0 chunks created and indexed.")); + + let listed = rpc::list_chunks_rpc( + &cfg, + rpc::ListChunksRequest { + source_kind: Some("document".into()), + source_id: Some("doc-dup".into()), + limit: Some(10), + ..Default::default() + }, + ) + .await + .expect("list chunks after duplicate execute") + .value + .chunks; + assert_eq!( + listed.len(), + 1, + "duplicate source_id should not create extra chunks" + ); + } +} diff --git a/src/openhuman/memory_tree/tools/mod.rs b/src/openhuman/memory_tree/tools/mod.rs index 9a1c08e59..f6789cdaf 100644 --- a/src/openhuman/memory_tree/tools/mod.rs +++ b/src/openhuman/memory_tree/tools/mod.rs @@ -266,6 +266,37 @@ mod memory_tree_dispatcher_tests { assert!(result.is_err()); } + #[tokio::test] + async fn memory_tree_query_global_mode_dispatches_successfully() { + let result = MemoryTreeTool + .execute(json!({ + "mode": "query_global", + "time_window_days": 7 + })) + .await + .expect("query_global mode should dispatch successfully"); + assert!(!result.is_error); + let parsed: serde_json::Value = + serde_json::from_str(&result.text()).expect("result should be valid json"); + assert!(parsed.get("hits").is_some()); + assert!(parsed.get("total").is_some()); + } + + #[tokio::test] + async fn memory_tree_fetch_leaves_mode_dispatches_successfully() { + let result = MemoryTreeTool + .execute(json!({ + "mode": "fetch_leaves", + "chunk_ids": ["chunk-does-not-exist"] + })) + .await + .expect("fetch_leaves mode should dispatch successfully"); + assert!(!result.is_error); + let parsed: serde_json::Value = + serde_json::from_str(&result.text()).expect("result should be valid json"); + assert!(parsed.is_array()); + } + #[test] fn translate_query_global_args_renames_time_window_days_to_window_days() { // Per-issue #2252: the consolidated schema advertises `time_window_days` diff --git a/src/openhuman/memory_tree/tools/query_global.rs b/src/openhuman/memory_tree/tools/query_global.rs index b85cc4f81..be48246a4 100644 --- a/src/openhuman/memory_tree/tools/query_global.rs +++ b/src/openhuman/memory_tree/tools/query_global.rs @@ -1,6 +1,6 @@ use crate::openhuman::config::rpc as config_rpc; -use crate::openhuman::memory_tree::retrieval; -use crate::openhuman::memory_tree::retrieval::rpc::QueryGlobalRequest; +use crate::openhuman::memory::retrieval; +use crate::openhuman::memory::retrieval::rpc::QueryGlobalRequest; use crate::openhuman::tools::traits::{Tool, ToolResult}; use async_trait::async_trait; use serde_json::json; @@ -51,3 +51,125 @@ impl Tool for MemoryTreeQueryGlobalTool { Ok(ToolResult::success(json)) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::OsString; + + use tempfile::TempDir; + + use crate::openhuman::config::{Config, TEST_ENV_LOCK}; + use crate::openhuman::tools::traits::Tool; + use serde_json::json; + + struct WorkspaceEnvGuard { + _lock: std::sync::MutexGuard<'static, ()>, + previous: Option, + } + + impl WorkspaceEnvGuard { + fn set(path: &std::path::Path) -> Self { + let lock = TEST_ENV_LOCK.lock().unwrap_or_else(|err| err.into_inner()); + let previous = std::env::var_os("OPENHUMAN_WORKSPACE"); + std::env::set_var("OPENHUMAN_WORKSPACE", path); + Self { + _lock: lock, + previous, + } + } + } + + impl Drop for WorkspaceEnvGuard { + fn drop(&mut self) { + if let Some(previous) = self.previous.as_ref() { + std::env::set_var("OPENHUMAN_WORKSPACE", previous); + } else { + std::env::remove_var("OPENHUMAN_WORKSPACE"); + } + } + } + + async fn isolated_config(tmp: &TempDir) -> (WorkspaceEnvGuard, Config) { + let guard = WorkspaceEnvGuard::set(tmp.path()); + let config = Config::load_or_init().await.expect("load config"); + (guard, config) + } + + #[test] + fn parameters_schema_requires_time_window_days() { + let tool = MemoryTreeQueryGlobalTool; + let schema = tool.parameters_schema(); + assert_eq!(schema["required"], json!(["time_window_days"])); + assert_eq!(schema["properties"]["time_window_days"]["minimum"], 1); + } + + #[tokio::test] + async fn execute_rejects_missing_time_window_days() { + let tool = MemoryTreeQueryGlobalTool; + let err = tool + .execute(json!({})) + .await + .expect_err("missing time_window_days should fail"); + assert!(err + .to_string() + .contains("invalid arguments for memory_tree_query_global")); + } + + #[tokio::test] + async fn execute_rejects_wrong_type_for_time_window_days() { + let tool = MemoryTreeQueryGlobalTool; + let err = tool + .execute(json!({"time_window_days": "seven"})) + .await + .expect_err("wrong type should fail"); + assert!(err + .to_string() + .contains("invalid arguments for memory_tree_query_global")); + } + + #[tokio::test] + async fn execute_accepts_window_days_alias() { + let tool = MemoryTreeQueryGlobalTool; + let req: QueryGlobalRequest = + serde_json::from_value(json!({"window_days": 7})).expect("alias should deserialize"); + assert_eq!(req.time_window_days, 7); + + let result = tool + .execute(json!({"window_days": 7})) + .await + .expect("window_days alias should succeed"); + assert!(!result.is_error); + } + + #[tokio::test] + async fn execute_success_path_returns_empty_payload_for_isolated_workspace() { + let tmp = TempDir::new().expect("tempdir"); + let (_workspace, cfg) = isolated_config(&tmp).await; + let tool = MemoryTreeQueryGlobalTool; + let result = tool + .execute(json!({"time_window_days": 7})) + .await + .expect("valid query_global should succeed in isolated workspace"); + assert!(!result.is_error); + let payload = result.text(); + let parsed: serde_json::Value = + serde_json::from_str(&payload).expect("result should be valid json"); + assert!( + parsed.get("hits").is_some(), + "payload should include hits array" + ); + assert!( + parsed.get("total").is_some(), + "payload should include total count" + ); + assert_eq!(parsed["total"], json!(0)); + assert_eq!(parsed["hits"], json!([])); + + let direct = retrieval::query_global(&cfg, 7) + .await + .expect("direct query_global on empty workspace"); + assert_eq!(direct.total, 0); + assert!(direct.hits.is_empty()); + } +} diff --git a/src/openhuman/memory_tree/tools/query_source.rs b/src/openhuman/memory_tree/tools/query_source.rs index 6b433eccd..067d4fed1 100644 --- a/src/openhuman/memory_tree/tools/query_source.rs +++ b/src/openhuman/memory_tree/tools/query_source.rs @@ -1,7 +1,7 @@ use crate::openhuman::config::rpc as config_rpc; -use crate::openhuman::memory_tree::retrieval; -use crate::openhuman::memory_tree::retrieval::rpc::QuerySourceRequest; -use crate::openhuman::memory_tree::types::SourceKind; +use crate::openhuman::memory::retrieval; +use crate::openhuman::memory::retrieval::rpc::QuerySourceRequest; +use crate::openhuman::memory_store::chunks::types::SourceKind; use crate::openhuman::tools::traits::{Tool, ToolResult}; use async_trait::async_trait; use serde_json::json; @@ -85,3 +85,132 @@ impl Tool for MemoryTreeQuerySourceTool { Ok(ToolResult::success(json)) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::OsString; + + use tempfile::TempDir; + + use crate::openhuman::config::{Config, TEST_ENV_LOCK}; + use crate::openhuman::tools::traits::Tool; + use serde_json::json; + + struct WorkspaceEnvGuard { + _lock: std::sync::MutexGuard<'static, ()>, + previous: Option, + } + + impl WorkspaceEnvGuard { + fn set(path: &std::path::Path) -> Self { + let lock = TEST_ENV_LOCK.lock().unwrap_or_else(|err| err.into_inner()); + let previous = std::env::var_os("OPENHUMAN_WORKSPACE"); + std::env::set_var("OPENHUMAN_WORKSPACE", path); + Self { + _lock: lock, + previous, + } + } + } + + impl Drop for WorkspaceEnvGuard { + fn drop(&mut self) { + if let Some(previous) = self.previous.as_ref() { + std::env::set_var("OPENHUMAN_WORKSPACE", previous); + } else { + std::env::remove_var("OPENHUMAN_WORKSPACE"); + } + } + } + + async fn isolated_config(tmp: &TempDir) -> (WorkspaceEnvGuard, Config) { + let guard = WorkspaceEnvGuard::set(tmp.path()); + let config = Config::load_or_init().await.expect("load config"); + (guard, config) + } + + #[test] + fn parameters_schema_exposes_supported_source_filters() { + let tool = MemoryTreeQuerySourceTool; + let schema = tool.parameters_schema(); + assert_eq!(schema["type"], "object"); + assert_eq!( + schema["properties"]["source_kind"]["enum"], + json!(["chat", "email", "document"]) + ); + assert_eq!(schema["properties"]["time_window_days"]["minimum"], 0); + } + + #[tokio::test] + async fn execute_rejects_invalid_source_kind() { + let tool = MemoryTreeQuerySourceTool; + let err = tool + .execute(json!({ + "source_kind": "not-real" + })) + .await + .expect_err("invalid source kind should fail"); + assert!(err.to_string().contains("memory_tree_query_source:")); + } + + #[tokio::test] + async fn execute_rejects_wrong_type_for_limit() { + let tool = MemoryTreeQuerySourceTool; + let err = tool + .execute(json!({ + "limit": "five" + })) + .await + .expect_err("wrong limit type should fail"); + assert!(err + .to_string() + .contains("invalid arguments for memory_tree_query_source")); + } + + #[tokio::test] + async fn execute_success_path_returns_empty_payload_for_isolated_workspace() { + let tmp = TempDir::new().expect("tempdir"); + let (_workspace, cfg) = isolated_config(&tmp).await; + let tool = MemoryTreeQuerySourceTool; + let result = tool + .execute(json!({ + "source_kind": "document", + "limit": 2 + })) + .await + .expect("valid query_source should succeed in isolated workspace"); + assert!(!result.is_error); + let payload = result.text(); + let parsed: serde_json::Value = + serde_json::from_str(&payload).expect("result should be valid json"); + assert!(parsed.get("hits").is_some(), "payload should include hits"); + assert!( + parsed.get("total").is_some(), + "payload should include total" + ); + assert_eq!(parsed["hits"], json!([])); + assert_eq!(parsed["total"], json!(0)); + + let direct = retrieval::query_source(&cfg, None, Some(SourceKind::Document), None, None, 2) + .await + .expect("direct query_source on empty workspace"); + assert!(direct.hits.is_empty()); + assert_eq!(direct.total, 0); + } + + #[tokio::test] + async fn execute_accepts_exact_source_id_without_source_kind() { + let tmp = TempDir::new().expect("tempdir"); + let (_workspace, _cfg) = isolated_config(&tmp).await; + let tool = MemoryTreeQuerySourceTool; + let result = tool + .execute(json!({ + "source_id": "slack:#eng", + "limit": 1 + })) + .await + .expect("source_id-only query should succeed"); + assert!(!result.is_error); + } +} diff --git a/src/openhuman/memory_tree/tools/query_topic.rs b/src/openhuman/memory_tree/tools/query_topic.rs index e45115d0c..733ebd0aa 100644 --- a/src/openhuman/memory_tree/tools/query_topic.rs +++ b/src/openhuman/memory_tree/tools/query_topic.rs @@ -1,6 +1,6 @@ use crate::openhuman::config::rpc as config_rpc; -use crate::openhuman::memory_tree::retrieval; -use crate::openhuman::memory_tree::retrieval::rpc::QueryTopicRequest; +use crate::openhuman::memory::retrieval; +use crate::openhuman::memory::retrieval::rpc::QueryTopicRequest; use crate::openhuman::tools::traits::{Tool, ToolResult}; use async_trait::async_trait; use serde_json::json; @@ -72,3 +72,126 @@ impl Tool for MemoryTreeQueryTopicTool { Ok(ToolResult::success(json)) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::OsString; + + use tempfile::TempDir; + + use crate::openhuman::config::{Config, TEST_ENV_LOCK}; + use crate::openhuman::tools::traits::Tool; + use serde_json::json; + + struct WorkspaceEnvGuard { + _lock: std::sync::MutexGuard<'static, ()>, + previous: Option, + } + + impl WorkspaceEnvGuard { + fn set(path: &std::path::Path) -> Self { + let lock = TEST_ENV_LOCK.lock().unwrap_or_else(|err| err.into_inner()); + let previous = std::env::var_os("OPENHUMAN_WORKSPACE"); + std::env::set_var("OPENHUMAN_WORKSPACE", path); + Self { + _lock: lock, + previous, + } + } + } + + impl Drop for WorkspaceEnvGuard { + fn drop(&mut self) { + if let Some(previous) = self.previous.as_ref() { + std::env::set_var("OPENHUMAN_WORKSPACE", previous); + } else { + std::env::remove_var("OPENHUMAN_WORKSPACE"); + } + } + } + + async fn isolated_config(tmp: &TempDir) -> (WorkspaceEnvGuard, Config) { + let guard = WorkspaceEnvGuard::set(tmp.path()); + let config = Config::load_or_init().await.expect("load config"); + (guard, config) + } + + #[test] + fn parameters_schema_requires_entity_id() { + let tool = MemoryTreeQueryTopicTool; + let schema = tool.parameters_schema(); + assert_eq!(schema["required"], json!(["entity_id"])); + assert_eq!(schema["properties"]["time_window_days"]["minimum"], 0); + } + + #[tokio::test] + async fn execute_rejects_missing_entity_id() { + let tool = MemoryTreeQueryTopicTool; + let err = tool + .execute(json!({})) + .await + .expect_err("missing entity_id should fail"); + assert!(err + .to_string() + .contains("invalid arguments for memory_tree_query_topic")); + } + + #[tokio::test] + async fn execute_rejects_wrong_type_for_entity_id() { + let tool = MemoryTreeQueryTopicTool; + let err = tool + .execute(json!({"entity_id": 42})) + .await + .expect_err("wrong type should fail"); + assert!(err + .to_string() + .contains("invalid arguments for memory_tree_query_topic")); + } + + #[tokio::test] + async fn execute_success_path_returns_empty_payload_for_isolated_workspace() { + let tmp = TempDir::new().expect("tempdir"); + let (_workspace, cfg) = isolated_config(&tmp).await; + let tool = MemoryTreeQueryTopicTool; + let result = tool + .execute(json!({ + "entity_id": "topic:phoenix", + "limit": 2 + })) + .await + .expect("valid query_topic should succeed in isolated workspace"); + assert!(!result.is_error); + let payload = result.text(); + let parsed: serde_json::Value = + serde_json::from_str(&payload).expect("result should be valid json"); + assert!(parsed.get("hits").is_some(), "payload should include hits"); + assert!( + parsed.get("total").is_some(), + "payload should include total" + ); + assert_eq!(parsed["hits"], json!([])); + assert_eq!(parsed["total"], json!(0)); + + let direct = retrieval::query_topic(&cfg, "topic:phoenix", None, None, 2) + .await + .expect("direct query_topic on empty workspace"); + assert!(direct.hits.is_empty()); + assert_eq!(direct.total, 0); + } + + #[tokio::test] + async fn execute_accepts_time_window_without_query() { + let tmp = TempDir::new().expect("tempdir"); + let (_workspace, _cfg) = isolated_config(&tmp).await; + let tool = MemoryTreeQueryTopicTool; + let result = tool + .execute(json!({ + "entity_id": "email:alice@example.com", + "time_window_days": 7 + })) + .await + .expect("time-window-only topic query should succeed"); + assert!(!result.is_error); + } +} diff --git a/src/openhuman/memory_tree/tools/search_entities.rs b/src/openhuman/memory_tree/tools/search_entities.rs index e280d689a..f8823ff4b 100644 --- a/src/openhuman/memory_tree/tools/search_entities.rs +++ b/src/openhuman/memory_tree/tools/search_entities.rs @@ -1,7 +1,7 @@ use crate::openhuman::config::rpc as config_rpc; -use crate::openhuman::memory_tree::retrieval; -use crate::openhuman::memory_tree::retrieval::rpc::SearchEntitiesRequest; -use crate::openhuman::memory_tree::score::extract::EntityKind; +use crate::openhuman::memory::retrieval; +use crate::openhuman::memory::retrieval::rpc::SearchEntitiesRequest; +use crate::openhuman::memory::score::extract::EntityKind; use crate::openhuman::tools::traits::{Tool, ToolResult}; use async_trait::async_trait; use serde_json::json; @@ -79,3 +79,145 @@ impl Tool for MemoryTreeSearchEntitiesTool { Ok(ToolResult::success(json)) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::OsString; + + use tempfile::TempDir; + + use crate::openhuman::config::{Config, TEST_ENV_LOCK}; + use crate::openhuman::tools::traits::Tool; + use serde_json::json; + + struct WorkspaceEnvGuard { + _lock: std::sync::MutexGuard<'static, ()>, + previous: Option, + } + + impl WorkspaceEnvGuard { + fn set(path: &std::path::Path) -> Self { + let lock = TEST_ENV_LOCK.lock().unwrap_or_else(|err| err.into_inner()); + let previous = std::env::var_os("OPENHUMAN_WORKSPACE"); + std::env::set_var("OPENHUMAN_WORKSPACE", path); + Self { + _lock: lock, + previous, + } + } + } + + impl Drop for WorkspaceEnvGuard { + fn drop(&mut self) { + if let Some(previous) = self.previous.as_ref() { + std::env::set_var("OPENHUMAN_WORKSPACE", previous); + } else { + std::env::remove_var("OPENHUMAN_WORKSPACE"); + } + } + } + + async fn isolated_config(tmp: &TempDir) -> (WorkspaceEnvGuard, Config) { + let guard = WorkspaceEnvGuard::set(tmp.path()); + let config = Config::load_or_init().await.expect("load config"); + (guard, config) + } + + #[test] + fn parameters_schema_requires_query() { + let tool = MemoryTreeSearchEntitiesTool; + let schema = tool.parameters_schema(); + assert_eq!(schema["required"], json!(["query"])); + assert_eq!( + schema["properties"]["limit"]["description"].is_string(), + true + ); + } + + #[test] + fn kind_enum_contains_expected_memory_entity_kinds() { + let tool = MemoryTreeSearchEntitiesTool; + let schema = tool.parameters_schema(); + let kinds = schema["properties"]["kinds"]["items"]["enum"] + .as_array() + .unwrap(); + for required in ["email", "person", "organization", "topic"] { + assert!( + kinds.iter().any(|v| v == required), + "missing kind {required}" + ); + } + } + + #[tokio::test] + async fn execute_rejects_missing_query() { + let tool = MemoryTreeSearchEntitiesTool; + let err = tool + .execute(json!({})) + .await + .expect_err("missing query should fail"); + assert!(err + .to_string() + .contains("invalid arguments for memory_tree_search_entities")); + } + + #[tokio::test] + async fn execute_rejects_invalid_kind_after_validation() { + let tool = MemoryTreeSearchEntitiesTool; + let err = tool + .execute(json!({ + "query": "alice", + "kinds": ["not-a-real-kind"] + })) + .await + .expect_err("invalid kind should fail"); + assert!(err + .to_string() + .contains("memory_tree_search_entities: invalid kind:")); + } + + #[tokio::test] + async fn execute_success_path_returns_empty_json_array_for_isolated_workspace() { + let tmp = TempDir::new().expect("tempdir"); + let (_workspace, cfg) = isolated_config(&tmp).await; + let tool = MemoryTreeSearchEntitiesTool; + let result = tool + .execute(json!({ + "query": "alice", + "limit": 3 + })) + .await + .expect("valid search_entities request should succeed in isolated workspace"); + assert!(!result.is_error); + let payload = result.text(); + let parsed: serde_json::Value = + serde_json::from_str(&payload).expect("result should be valid json"); + assert!( + parsed.is_array(), + "search_entities should serialize a JSON array" + ); + assert_eq!(parsed, json!([])); + + let direct = retrieval::search_entities(&cfg, "alice", None, 3) + .await + .expect("direct search_entities on empty workspace"); + assert!(direct.is_empty()); + } + + #[tokio::test] + async fn execute_accepts_kind_filter_and_clamps_large_limit() { + let tmp = TempDir::new().expect("tempdir"); + let (_workspace, _cfg) = isolated_config(&tmp).await; + let tool = MemoryTreeSearchEntitiesTool; + let result = tool + .execute(json!({ + "query": "alice", + "kinds": ["email", "person"], + "limit": 999 + })) + .await + .expect("filtered search_entities request should succeed"); + assert!(!result.is_error); + } +} diff --git a/src/openhuman/memory_tree/tools/walk.rs b/src/openhuman/memory_tree/tools/walk.rs index cfe75eea6..32e509dfa 100644 --- a/src/openhuman/memory_tree/tools/walk.rs +++ b/src/openhuman/memory_tree/tools/walk.rs @@ -18,10 +18,10 @@ use crate::openhuman::config::rpc as config_rpc; use crate::openhuman::config::Config; use crate::openhuman::inference::provider::traits::{ChatMessage, Provider}; -use crate::openhuman::memory_tree::chat::{build_chat_provider, ChatConsumer, ChatPrompt}; -use crate::openhuman::memory_tree::retrieval; -use crate::openhuman::memory_tree::retrieval::fetch::fetch_leaves as do_fetch_leaves; -use crate::openhuman::memory_tree::summarizer::store::{read_children, read_node}; +use crate::openhuman::memory::chat::{build_chat_provider, ChatPrompt}; +use crate::openhuman::memory::retrieval; +use crate::openhuman::memory::retrieval::fetch::fetch_leaves as do_fetch_leaves; +use crate::openhuman::memory_tree::tree_runtime::store::{read_children, read_node}; use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolCategory, ToolResult}; use async_trait::async_trait; use serde_json::json; @@ -179,7 +179,7 @@ impl Tool for MemoryTreeWalkTool { // Build a chat provider from config (same path used by the summariser) // and wrap it in the thin `ChatProviderAdapter` that satisfies `Provider`. - let chat_provider = build_chat_provider(&cfg, ChatConsumer::Summarise) + let chat_provider = build_chat_provider(&cfg) .map_err(|e| anyhow::anyhow!("memory_tree_walk: build chat provider failed: {e}"))?; let adapter = ChatProviderAdapter { inner: chat_provider, @@ -372,7 +372,7 @@ pub async fn run_walk( // unit-test stubs that implement `Provider` directly. struct ChatProviderAdapter { - inner: std::sync::Arc, + inner: std::sync::Arc, } #[async_trait] @@ -740,8 +740,8 @@ mod tests { use super::*; use crate::openhuman::config::Config; use crate::openhuman::inference::provider::traits::ChatMessage; - use crate::openhuman::memory_tree::summarizer::store::write_node; - use crate::openhuman::memory_tree::summarizer::types::{NodeLevel, TreeNode}; + use crate::openhuman::memory_tree::tree_runtime::store::write_node; + use crate::openhuman::memory_tree::tree_runtime::types::{NodeLevel, TreeNode}; use async_trait::async_trait; use chrono::Utc; use std::sync::Mutex; @@ -803,8 +803,9 @@ mod tests { } fn make_node(namespace: &str, node_id: &str, summary: &str, child_count: u32) -> TreeNode { - let level = crate::openhuman::memory_tree::summarizer::types::level_from_node_id(node_id); - let parent_id = crate::openhuman::memory_tree::summarizer::types::derive_parent_id(node_id); + let level = crate::openhuman::memory_tree::tree_runtime::types::level_from_node_id(node_id); + let parent_id = + crate::openhuman::memory_tree::tree_runtime::types::derive_parent_id(node_id); let ts = Utc::now(); TreeNode { node_id: node_id.to_string(), @@ -812,7 +813,9 @@ mod tests { level, parent_id, summary: summary.to_string(), - token_count: crate::openhuman::memory_tree::summarizer::types::estimate_tokens(summary), + token_count: crate::openhuman::memory_tree::tree_runtime::types::estimate_tokens( + summary, + ), child_count, created_at: ts, updated_at: ts, diff --git a/src/openhuman/memory_tree/tree_topic/README.md b/src/openhuman/memory_tree/topic/README.md similarity index 100% rename from src/openhuman/memory_tree/tree_topic/README.md rename to src/openhuman/memory_tree/topic/README.md diff --git a/src/openhuman/memory_tree/tree_topic/backfill.rs b/src/openhuman/memory_tree/topic/backfill.rs similarity index 81% rename from src/openhuman/memory_tree/tree_topic/backfill.rs rename to src/openhuman/memory_tree/topic/backfill.rs index 9172aea47..6c904a042 100644 --- a/src/openhuman/memory_tree/tree_topic/backfill.rs +++ b/src/openhuman/memory_tree/topic/backfill.rs @@ -27,14 +27,11 @@ use anyhow::{Context, Result}; use chrono::Utc; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::score::store::lookup_entity; -use crate::openhuman::memory_tree::store::get_chunk; -use crate::openhuman::memory_tree::tree_source::bucket_seal::{ - append_leaf, LabelStrategy, LeafRef, -}; -use crate::openhuman::memory_tree::tree_source::summariser::Summariser; -use crate::openhuman::memory_tree::tree_source::types::Tree; -use crate::openhuman::memory_tree::util::redact::redact; +use crate::openhuman::memory::score::store::lookup_entity; +use crate::openhuman::memory::util::redact::redact; +use crate::openhuman::memory_store::chunks::store::get_chunk; +use crate::openhuman::memory_store::trees::types::Tree; +use crate::openhuman::memory_tree::tree::bucket_seal::{append_leaf, LabelStrategy, LeafRef}; /// Max leaves to pull from the entity index during backfill. A hard cap /// keeps initial spawn latency bounded even for very active entities. @@ -51,20 +48,8 @@ const DAY_MS: i64 = 24 * 60 * 60 * 1_000; /// to `tree`. Returns the number of leaves appended (NOT the number of /// summaries sealed). Idempotent: `append_leaf` itself is a no-op when a /// leaf is already in the buffer, so re-running backfill is safe. -pub async fn backfill_topic_tree( - config: &Config, - tree: &Tree, - entity_id: &str, - summariser: &dyn Summariser, -) -> Result { - backfill_topic_tree_at( - config, - tree, - entity_id, - summariser, - Utc::now().timestamp_millis(), - ) - .await +pub async fn backfill_topic_tree(config: &Config, tree: &Tree, entity_id: &str) -> Result { + backfill_topic_tree_at(config, tree, entity_id, Utc::now().timestamp_millis()).await } /// Deterministic variant — backfill against a caller-supplied `now_ms` @@ -74,7 +59,6 @@ pub async fn backfill_topic_tree_at( config: &Config, tree: &Tree, entity_id: &str, - summariser: &dyn Summariser, now_ms: i64, ) -> Result { let cutoff_ms = now_ms.saturating_sub(BACKFILL_WINDOW_DAYS.saturating_mul(DAY_MS)); @@ -165,7 +149,7 @@ pub async fn backfill_topic_tree_at( // Topic-tree backfill: empty labels for sealed summaries — the // tree's scope already pins the canonical id, so cross-pollinating // descendants' entities would noise the index. See LabelStrategy. - append_leaf(config, tree, &leaf, summariser, &LabelStrategy::Empty) + append_leaf(config, tree, &leaf, &LabelStrategy::Empty) .await .with_context(|| { format!( @@ -191,15 +175,18 @@ pub async fn backfill_topic_tree_at( #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory_tree::score::extract::EntityKind; - use crate::openhuman::memory_tree::score::resolver::CanonicalEntity; - use crate::openhuman::memory_tree::score::store::index_entity; - use crate::openhuman::memory_tree::store::upsert_chunks; - use crate::openhuman::memory_tree::tree_source::store as src_store; - use crate::openhuman::memory_tree::tree_source::summariser::inert::InertSummariser; - use crate::openhuman::memory_tree::tree_topic::registry::get_or_create_topic_tree; - use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; + use crate::openhuman::memory::chat::{test_override, ChatProvider, StaticChatProvider}; + use crate::openhuman::memory::score::extract::EntityKind; + use crate::openhuman::memory::score::resolver::CanonicalEntity; + use crate::openhuman::memory::score::store::index_entity; + use crate::openhuman::memory_store::chunks::store::upsert_chunks; + use crate::openhuman::memory_store::chunks::types::{ + chunk_id, Chunk, Metadata, SourceKind, SourceRef, + }; + use crate::openhuman::memory_store::trees::registry::get_or_create_topic_tree; + use crate::openhuman::memory_tree::tree::store as src_store; use chrono::{TimeZone, Utc}; + use std::sync::Arc; use tempfile::TempDir; fn test_config() -> (TempDir, Config) { @@ -289,16 +276,14 @@ mod tests { .unwrap(); let tree = get_or_create_topic_tree(&cfg, "email:alice@example.com").unwrap(); - let summariser = InertSummariser::new(); - let n = backfill_topic_tree_at( - &cfg, - &tree, - "email:alice@example.com", - &summariser, - TEST_NOW_MS, - ) - .await - .unwrap(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); + let n = test_override::with_provider(provider, async { + backfill_topic_tree_at(&cfg, &tree, "email:alice@example.com", TEST_NOW_MS) + .await + .unwrap() + }) + .await; assert_eq!(n, 3); // L0 buffer should hold all three leaves (combined tokens well @@ -326,16 +311,14 @@ mod tests { index_entity(&cfg, &e, &c_new.id, "leaf", new_ts, Some("source:slack")).unwrap(); let tree = get_or_create_topic_tree(&cfg, "email:alice@example.com").unwrap(); - let summariser = InertSummariser::new(); - let n = backfill_topic_tree_at( - &cfg, - &tree, - "email:alice@example.com", - &summariser, - TEST_NOW_MS, - ) - .await - .unwrap(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); + let n = test_override::with_provider(provider, async { + backfill_topic_tree_at(&cfg, &tree, "email:alice@example.com", TEST_NOW_MS) + .await + .unwrap() + }) + .await; assert_eq!(n, 1, "only the in-window leaf should be appended"); let buf = src_store::get_buffer(&cfg, &tree.id, 0).unwrap(); assert_eq!(buf.item_ids.len(), 1); @@ -362,16 +345,14 @@ mod tests { .unwrap(); let tree = get_or_create_topic_tree(&cfg, "email:alice@example.com").unwrap(); - let summariser = InertSummariser::new(); - let n = backfill_topic_tree_at( - &cfg, - &tree, - "email:alice@example.com", - &summariser, - TEST_NOW_MS, - ) - .await - .unwrap(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); + let n = test_override::with_provider(provider, async { + backfill_topic_tree_at(&cfg, &tree, "email:alice@example.com", TEST_NOW_MS) + .await + .unwrap() + }) + .await; assert_eq!(n, 1, "only the existing chunk should be appended"); let buf = src_store::get_buffer(&cfg, &tree.id, 0).unwrap(); assert_eq!(buf.item_ids.len(), 1); @@ -394,25 +375,17 @@ mod tests { .unwrap(); let tree = get_or_create_topic_tree(&cfg, "email:alice@example.com").unwrap(); - let summariser = InertSummariser::new(); - backfill_topic_tree_at( - &cfg, - &tree, - "email:alice@example.com", - &summariser, - TEST_NOW_MS, - ) - .await - .unwrap(); - backfill_topic_tree_at( - &cfg, - &tree, - "email:alice@example.com", - &summariser, - TEST_NOW_MS, - ) - .await - .unwrap(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); + test_override::with_provider(provider, async { + backfill_topic_tree_at(&cfg, &tree, "email:alice@example.com", TEST_NOW_MS) + .await + .unwrap(); + backfill_topic_tree_at(&cfg, &tree, "email:alice@example.com", TEST_NOW_MS) + .await + .unwrap(); + }) + .await; // append_leaf is idempotent so the buffer still has exactly one row. let buf = src_store::get_buffer(&cfg, &tree.id, 0).unwrap(); assert_eq!(buf.item_ids.len(), 1); @@ -433,16 +406,14 @@ mod tests { ) .unwrap(); let tree = get_or_create_topic_tree(&cfg, "email:alice@example.com").unwrap(); - let summariser = InertSummariser::new(); - let n = backfill_topic_tree_at( - &cfg, - &tree, - "email:alice@example.com", - &summariser, - TEST_NOW_MS, - ) - .await - .unwrap(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); + let n = test_override::with_provider(provider, async { + backfill_topic_tree_at(&cfg, &tree, "email:alice@example.com", TEST_NOW_MS) + .await + .unwrap() + }) + .await; assert_eq!(n, 0); } } diff --git a/src/openhuman/memory_tree/tree_topic/curator.rs b/src/openhuman/memory_tree/topic/curator.rs similarity index 75% rename from src/openhuman/memory_tree/tree_topic/curator.rs rename to src/openhuman/memory_tree/topic/curator.rs index 45acac45c..220ea6cfb 100644 --- a/src/openhuman/memory_tree/tree_topic/curator.rs +++ b/src/openhuman/memory_tree/topic/curator.rs @@ -19,18 +19,15 @@ use anyhow::Result; use chrono::Utc; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::tree_source::store as src_store; -use crate::openhuman::memory_tree::tree_source::summariser::Summariser; -use crate::openhuman::memory_tree::tree_source::types::{Tree, TreeKind}; -use crate::openhuman::memory_tree::tree_topic::backfill::backfill_topic_tree; -use crate::openhuman::memory_tree::tree_topic::hotness::hotness_at; -use crate::openhuman::memory_tree::tree_topic::registry::get_or_create_topic_tree; -use crate::openhuman::memory_tree::tree_topic::store::{ - distinct_sources_for, get_or_fresh, upsert, -}; -use crate::openhuman::memory_tree::tree_topic::types::{ +use crate::openhuman::memory_store::trees::hotness::{distinct_sources_for, get_or_fresh, upsert}; +use crate::openhuman::memory_store::trees::registry::get_or_create_topic_tree; +use crate::openhuman::memory_store::trees::types::{ HotnessCounters, TOPIC_CREATION_THRESHOLD, TOPIC_RECHECK_EVERY, }; +use crate::openhuman::memory_store::trees::types::{Tree, TreeKind}; +use crate::openhuman::memory_tree::topic::backfill::backfill_topic_tree; +use crate::openhuman::memory_tree::topic::hotness::hotness_at; +use crate::openhuman::memory_tree::tree::store as src_store; /// Outcome of one curator invocation. Surfaced so the caller (typically /// the routing layer) can log / emit metrics. @@ -52,15 +49,7 @@ pub enum SpawnOutcome { /// Record an ingest touching `entity_id` and, when the recheck cadence /// fires, consider spawning a topic tree. -/// -/// `summariser` is used only when a spawn + backfill happens; passing an -/// [`InertSummariser`](crate::openhuman::memory_tree::tree_source::summariser::inert::InertSummariser) -/// is fine for Phase 3c. -pub async fn maybe_spawn_topic_tree( - config: &Config, - entity_id: &str, - summariser: &dyn Summariser, -) -> Result { +pub async fn maybe_spawn_topic_tree(config: &Config, entity_id: &str) -> Result { let now_ms = Utc::now().timestamp_millis(); // 1. Read existing counters (fresh row if first sighting). @@ -85,21 +74,17 @@ pub async fn maybe_spawn_topic_tree( } // 4. Full recompute. - run_full_recompute(config, entity_id, &mut counters, now_ms, summariser).await + run_full_recompute(config, entity_id, &mut counters, now_ms).await } /// Admin path: force a recompute + spawn-if-hot regardless of the /// [`TOPIC_RECHECK_EVERY`] cadence. Used by (future) RPCs that want to /// prod the curator without waiting for the next bump cycle. -pub async fn force_recompute( - config: &Config, - entity_id: &str, - summariser: &dyn Summariser, -) -> Result { +pub async fn force_recompute(config: &Config, entity_id: &str) -> Result { let now_ms = Utc::now().timestamp_millis(); let mut counters = get_or_fresh(config, entity_id)?; counters.last_updated_ms = now_ms; - run_full_recompute(config, entity_id, &mut counters, now_ms, summariser).await + run_full_recompute(config, entity_id, &mut counters, now_ms).await } async fn run_full_recompute( @@ -107,7 +92,6 @@ async fn run_full_recompute( entity_id: &str, counters: &mut HotnessCounters, now_ms: i64, - summariser: &dyn Summariser, ) -> Result { // Refresh distinct_sources from the entity index — the authoritative // source of cross-tree coverage. @@ -148,7 +132,7 @@ async fn run_full_recompute( h ); let tree = get_or_create_topic_tree(config, entity_id)?; - let backfilled = backfill_topic_tree(config, &tree, entity_id, summariser).await?; + let backfilled = backfill_topic_tree(config, &tree, entity_id).await?; SpawnOutcome::Spawned { hotness: h, tree_id: tree.id, @@ -168,14 +152,17 @@ fn existing_topic_tree(config: &Config, entity_id: &str) -> Result> #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory_tree::score::extract::EntityKind; - use crate::openhuman::memory_tree::score::resolver::CanonicalEntity; - use crate::openhuman::memory_tree::score::store::index_entity; - use crate::openhuman::memory_tree::store::upsert_chunks; - use crate::openhuman::memory_tree::tree_source::summariser::inert::InertSummariser; - use crate::openhuman::memory_tree::tree_topic::store::get; - use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; + use crate::openhuman::memory::chat::{test_override, ChatProvider, StaticChatProvider}; + use crate::openhuman::memory::score::extract::EntityKind; + use crate::openhuman::memory::score::resolver::CanonicalEntity; + use crate::openhuman::memory::score::store::index_entity; + use crate::openhuman::memory_store::chunks::store::upsert_chunks; + use crate::openhuman::memory_store::chunks::types::{ + chunk_id, Chunk, Metadata, SourceKind, SourceRef, + }; + use crate::openhuman::memory_store::trees::hotness::get; use chrono::{TimeZone, Utc}; + use std::sync::Arc; use tempfile::TempDir; fn test_config() -> (TempDir, Config) { @@ -224,10 +211,14 @@ mod tests { #[tokio::test] async fn first_ingest_just_bumps_counters() { let (_tmp, cfg) = test_config(); - let summariser = InertSummariser::new(); - let out = maybe_spawn_topic_tree(&cfg, "email:alice@example.com", &summariser) - .await - .unwrap(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); + let out = test_override::with_provider(provider, async { + maybe_spawn_topic_tree(&cfg, "email:alice@example.com") + .await + .unwrap() + }) + .await; assert_eq!(out, SpawnOutcome::CountersBumped); let c = get(&cfg, "email:alice@example.com").unwrap().unwrap(); assert_eq!(c.mention_count_30d, 1); @@ -238,12 +229,16 @@ mod tests { #[tokio::test] async fn no_spawn_below_threshold_on_recompute() { let (_tmp, cfg) = test_config(); - let summariser = InertSummariser::new(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); // Force a recompute on the very first call — but with no index data // the hotness comes out well below threshold. - let out = force_recompute(&cfg, "email:alice@example.com", &summariser) - .await - .unwrap(); + let out = test_override::with_provider(provider, async { + force_recompute(&cfg, "email:alice@example.com") + .await + .unwrap() + }) + .await; match out { SpawnOutcome::BelowThreshold { hotness } => { assert!(hotness < TOPIC_CREATION_THRESHOLD); @@ -258,7 +253,8 @@ mod tests { #[tokio::test] async fn spawn_fires_exactly_once_when_threshold_crossed() { let (_tmp, cfg) = test_config(); - let summariser = InertSummariser::new(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); // Seed substantial activity across several sources so hotness is // well above threshold. let mut counters = HotnessCounters::fresh("email:alice@example.com", 0); @@ -275,9 +271,18 @@ mod tests { seed_leaf_for_entity(&cfg, "email:alice@example.com", "gmail:alice", i); } - let out = force_recompute(&cfg, "email:alice@example.com", &summariser) - .await - .unwrap(); + let (out, out2) = test_override::with_provider(provider, async { + let o1 = force_recompute(&cfg, "email:alice@example.com") + .await + .unwrap(); + // Re-running should report TreeExists, NOT a second spawn. + let o2 = force_recompute(&cfg, "email:alice@example.com") + .await + .unwrap(); + (o1, o2) + }) + .await; + match out { SpawnOutcome::Spawned { hotness, @@ -290,11 +295,6 @@ mod tests { } other => panic!("expected Spawned, got {other:?}"), } - - // Re-running should report TreeExists, NOT a second spawn. - let out2 = force_recompute(&cfg, "email:alice@example.com", &summariser) - .await - .unwrap(); match out2 { SpawnOutcome::TreeExists { tree_id, .. } => { assert!(tree_id.starts_with("topic:")); @@ -306,7 +306,8 @@ mod tests { #[tokio::test] async fn recompute_refreshes_distinct_sources_from_entity_index() { let (_tmp, cfg) = test_config(); - let summariser = InertSummariser::new(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); // Counter says 0 distinct sources but the index has 3. let mut counters = HotnessCounters::fresh("email:alice@example.com", 0); counters.mention_count_30d = 1; @@ -316,9 +317,12 @@ mod tests { seed_leaf_for_entity(&cfg, "email:alice@example.com", "gmail:alice", 0); seed_leaf_for_entity(&cfg, "email:alice@example.com", "notion:abc", 0); - force_recompute(&cfg, "email:alice@example.com", &summariser) - .await - .unwrap(); + test_override::with_provider(provider, async { + force_recompute(&cfg, "email:alice@example.com") + .await + .unwrap(); + }) + .await; let c = get(&cfg, "email:alice@example.com").unwrap().unwrap(); assert_eq!(c.distinct_sources, 3); // ingests_since_check should also reset. @@ -328,7 +332,8 @@ mod tests { #[tokio::test] async fn cadence_only_recomputes_every_n_ingests() { let (_tmp, cfg) = test_config(); - let summariser = InertSummariser::new(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); // Pre-seed entity index with enough cross-source signal that the // recompute (which refreshes `distinct_sources` from the index) will // still produce a hotness above threshold. @@ -353,20 +358,26 @@ mod tests { counters.ingests_since_check = TOPIC_RECHECK_EVERY - 2; upsert(&cfg, &counters).unwrap(); - let out = maybe_spawn_topic_tree(&cfg, "email:alice@example.com", &summariser) - .await - .unwrap(); - assert_eq!(out, SpawnOutcome::CountersBumped); - // No tree yet — cadence not crossed. - assert!(existing_topic_tree(&cfg, "email:alice@example.com") - .unwrap() - .is_none()); + let out2 = test_override::with_provider(provider, async { + let o1 = maybe_spawn_topic_tree(&cfg, "email:alice@example.com") + .await + .unwrap(); + assert_eq!(o1, SpawnOutcome::CountersBumped); + // No tree yet — cadence not crossed after first call. + assert!( + existing_topic_tree(&cfg, "email:alice@example.com") + .unwrap() + .is_none(), + "no topic tree should exist before cadence is crossed" + ); + // One more bump — now ingests_since_check == TOPIC_RECHECK_EVERY + // and the recompute fires. + maybe_spawn_topic_tree(&cfg, "email:alice@example.com") + .await + .unwrap() + }) + .await; - // One more bump — now ingests_since_check == TOPIC_RECHECK_EVERY - // and the recompute fires. - let out2 = maybe_spawn_topic_tree(&cfg, "email:alice@example.com", &summariser) - .await - .unwrap(); match out2 { SpawnOutcome::Spawned { .. } | SpawnOutcome::TreeExists { .. } => {} other => panic!("expected Spawn/TreeExists after cadence, got {other:?}"), diff --git a/src/openhuman/memory_tree/tree_topic/hotness.rs b/src/openhuman/memory_tree/topic/hotness.rs similarity index 97% rename from src/openhuman/memory_tree/tree_topic/hotness.rs rename to src/openhuman/memory_tree/topic/hotness.rs index 43241f124..140bb1408 100644 --- a/src/openhuman/memory_tree/tree_topic/hotness.rs +++ b/src/openhuman/memory_tree/topic/hotness.rs @@ -23,7 +23,7 @@ use chrono::Utc; -use crate::openhuman::memory_tree::tree_topic::types::EntityIndexStats; +use crate::openhuman::memory_store::trees::types::EntityIndexStats; /// Pure hotness function — no I/O, no clocks unless the caller passes one. /// @@ -110,7 +110,7 @@ mod tests { #[test] fn spike_of_mentions_pushes_over_creation_threshold() { - use crate::openhuman::memory_tree::tree_topic::types::TOPIC_CREATION_THRESHOLD; + use crate::openhuman::memory_store::trees::types::TOPIC_CREATION_THRESHOLD; let now_ms = 1_700_000_000_000; // 100 mentions across 5 sources, 3 recent query hits, seen today. let s = EntityIndexStats { diff --git a/src/openhuman/memory_tree/tree_topic/mod.rs b/src/openhuman/memory_tree/topic/mod.rs similarity index 76% rename from src/openhuman/memory_tree/tree_topic/mod.rs rename to src/openhuman/memory_tree/topic/mod.rs index 7e28a3c77..756fba4bc 100644 --- a/src/openhuman/memory_tree/tree_topic/mod.rs +++ b/src/openhuman/memory_tree/topic/mod.rs @@ -22,25 +22,31 @@ //! easy to unit-test. //! //! Tree mechanics (buffer, seal, cascade) are **not reimplemented** here -//! — `append_leaf` from [`super::tree_source::bucket_seal`] takes a +//! — `append_leaf` from [`super::super::tree::bucket_seal`] takes a //! `&Tree` so it works for any `TreeKind`. The Phase 3c code only adds //! the hotness layer and the per-entity fan-out. +//! +//! Persistence (store, types, registry) has moved to +//! `memory_store::trees`. pub mod backfill; pub mod curator; pub mod hotness; -pub mod registry; pub mod routing; -pub mod store; -pub mod types; -pub use curator::{maybe_spawn_topic_tree, SpawnOutcome}; -pub use hotness::{hotness, recency_decay}; -pub use registry::{ +// Re-export persistence submodules from memory_store so callers using +// tree_topic::store/types/registry still work. +pub use crate::openhuman::memory_store::trees::hotness as store; +pub use crate::openhuman::memory_store::trees::registry; +pub use crate::openhuman::memory_store::trees::types; + +pub use crate::openhuman::memory_store::trees::{ archive_topic_tree, force_create_topic_tree, get_or_create_topic_tree, list_topic_trees, }; -pub use routing::route_leaf_to_topic_trees; -pub use types::{ +pub use crate::openhuman::memory_store::trees::{ EntityIndexStats, HotnessCounters, TOPIC_ARCHIVE_THRESHOLD, TOPIC_CREATION_THRESHOLD, TOPIC_RECHECK_EVERY, }; +pub use curator::{maybe_spawn_topic_tree, SpawnOutcome}; +pub use hotness::{hotness, recency_decay}; +pub use routing::route_leaf_to_topic_trees; diff --git a/src/openhuman/memory_tree/tree_topic/routing.rs b/src/openhuman/memory_tree/topic/routing.rs similarity index 80% rename from src/openhuman/memory_tree/tree_topic/routing.rs rename to src/openhuman/memory_tree/topic/routing.rs index 3591e3f11..6e02dc3ee 100644 --- a/src/openhuman/memory_tree/tree_topic/routing.rs +++ b/src/openhuman/memory_tree/topic/routing.rs @@ -21,13 +21,10 @@ use anyhow::Result; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::tree_source::bucket_seal::{ - append_leaf, LabelStrategy, LeafRef, -}; -use crate::openhuman::memory_tree::tree_source::store as src_store; -use crate::openhuman::memory_tree::tree_source::summariser::Summariser; -use crate::openhuman::memory_tree::tree_source::types::{TreeKind, TreeStatus}; -use crate::openhuman::memory_tree::tree_topic::curator::maybe_spawn_topic_tree; +use crate::openhuman::memory_store::trees::types::{TreeKind, TreeStatus}; +use crate::openhuman::memory_tree::topic::curator::maybe_spawn_topic_tree; +use crate::openhuman::memory_tree::tree::bucket_seal::{append_leaf, LabelStrategy, LeafRef}; +use crate::openhuman::memory_tree::tree::store as src_store; /// Route `leaf` into every active topic tree matching one of /// `canonical_entities`. Also ticks the curator for each entity so the @@ -40,7 +37,6 @@ pub async fn route_leaf_to_topic_trees( config: &Config, leaf: &LeafRef, canonical_entities: &[String], - summariser: &dyn Summariser, ) -> Result<()> { if canonical_entities.is_empty() { return Ok(()); @@ -53,7 +49,7 @@ pub async fn route_leaf_to_topic_trees( ); for entity_id in canonical_entities { - if let Err(e) = route_one_entity(config, leaf, entity_id, summariser).await { + if let Err(e) = route_one_entity(config, leaf, entity_id).await { let entity_kind = entity_id .split_once(':') .map(|(k, _)| k) @@ -69,12 +65,7 @@ pub async fn route_leaf_to_topic_trees( Ok(()) } -async fn route_one_entity( - config: &Config, - leaf: &LeafRef, - entity_id: &str, - summariser: &dyn Summariser, -) -> Result<()> { +async fn route_one_entity(config: &Config, leaf: &LeafRef, entity_id: &str) -> Result<()> { // Step 1: if a topic tree already exists and is active, append the leaf. // We intentionally do this BEFORE asking the curator to spawn — a // same-call spawn would also include this leaf via backfill @@ -98,14 +89,7 @@ async fn route_one_entity( }; // Topic-tree seals leave entities/topics empty: the tree's // scope already pins the canonical id this tree represents. - append_leaf( - config, - &tree, - &topic_leaf, - summariser, - &LabelStrategy::Empty, - ) - .await?; + append_leaf(config, &tree, &topic_leaf, &LabelStrategy::Empty).await?; } else { let entity_kind = entity_id .split_once(':') @@ -120,7 +104,7 @@ async fn route_one_entity( } // Step 2: curator tick — may spawn a new tree on cadence. - maybe_spawn_topic_tree(config, entity_id, summariser).await?; + maybe_spawn_topic_tree(config, entity_id).await?; Ok(()) } @@ -128,17 +112,20 @@ async fn route_one_entity( #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory_tree::score::extract::EntityKind; - use crate::openhuman::memory_tree::score::resolver::CanonicalEntity; - use crate::openhuman::memory_tree::score::store::index_entity; - use crate::openhuman::memory_tree::store::upsert_chunks; - use crate::openhuman::memory_tree::tree_source::summariser::inert::InertSummariser; - use crate::openhuman::memory_tree::tree_topic::registry::{ + use crate::openhuman::memory::chat::{test_override, ChatProvider, StaticChatProvider}; + use crate::openhuman::memory::score::extract::EntityKind; + use crate::openhuman::memory::score::resolver::CanonicalEntity; + use crate::openhuman::memory::score::store::index_entity; + use crate::openhuman::memory_store::chunks::store::upsert_chunks; + use crate::openhuman::memory_store::chunks::types::{ + chunk_id, Chunk, Metadata, SourceKind, SourceRef, + }; + use crate::openhuman::memory_store::trees::hotness::get as get_hotness; + use crate::openhuman::memory_store::trees::registry::{ archive_topic_tree, get_or_create_topic_tree, }; - use crate::openhuman::memory_tree::tree_topic::store::get as get_hotness; - use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; use chrono::{TimeZone, Utc}; + use std::sync::Arc; use tempfile::TempDir; fn test_config() -> (TempDir, Config) { @@ -191,14 +178,11 @@ mod tests { #[tokio::test] async fn empty_entities_is_noop() { let (_tmp, cfg) = test_config(); - let summariser = InertSummariser::new(); let leaf = mk_leaf("c1", 10, 1_700_000_000_000); - route_leaf_to_topic_trees(&cfg, &leaf, &[], &summariser) - .await - .unwrap(); + route_leaf_to_topic_trees(&cfg, &leaf, &[]).await.unwrap(); // No hotness rows were created. assert_eq!( - crate::openhuman::memory_tree::tree_topic::store::count(&cfg).unwrap(), + crate::openhuman::memory_store::trees::hotness::count(&cfg).unwrap(), 0 ); } @@ -206,21 +190,20 @@ mod tests { #[tokio::test] async fn appends_to_existing_topic_tree() { let (_tmp, cfg) = test_config(); - let summariser = InertSummariser::new(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); // Pre-create the topic tree so the hot-path append fires. let tree = get_or_create_topic_tree(&cfg, "email:alice@example.com").unwrap(); // Persist the backing chunk so hydrate can read it on seal. let chunk_id_s = persist_chunk(&cfg, "slack:#eng", 0, 1_700_000_000_000, 100); let leaf = mk_leaf(&chunk_id_s, 100, 1_700_000_000_000); - route_leaf_to_topic_trees( - &cfg, - &leaf, - &["email:alice@example.com".to_string()], - &summariser, - ) - .await - .unwrap(); + test_override::with_provider(provider, async { + route_leaf_to_topic_trees(&cfg, &leaf, &["email:alice@example.com".to_string()]) + .await + .unwrap() + }) + .await; let buf = src_store::get_buffer(&cfg, &tree.id, 0).unwrap(); assert_eq!(buf.item_ids.len(), 1); @@ -235,20 +218,14 @@ mod tests { #[tokio::test] async fn archived_topic_tree_does_not_receive_new_leaves() { let (_tmp, cfg) = test_config(); - let summariser = InertSummariser::new(); let tree = get_or_create_topic_tree(&cfg, "email:alice@example.com").unwrap(); archive_topic_tree(&cfg, &tree.id).unwrap(); let chunk_id_s = persist_chunk(&cfg, "slack:#eng", 0, 1_700_000_000_000, 100); let leaf = mk_leaf(&chunk_id_s, 100, 1_700_000_000_000); - route_leaf_to_topic_trees( - &cfg, - &leaf, - &["email:alice@example.com".to_string()], - &summariser, - ) - .await - .unwrap(); + route_leaf_to_topic_trees(&cfg, &leaf, &["email:alice@example.com".to_string()]) + .await + .unwrap(); let buf = src_store::get_buffer(&cfg, &tree.id, 0).unwrap(); assert!( @@ -265,23 +242,26 @@ mod tests { #[tokio::test] async fn one_leaf_multiple_entities_fans_out() { let (_tmp, cfg) = test_config(); - let summariser = InertSummariser::new(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); let t1 = get_or_create_topic_tree(&cfg, "email:alice@example.com").unwrap(); let t2 = get_or_create_topic_tree(&cfg, "hashtag:launch").unwrap(); let chunk_id_s = persist_chunk(&cfg, "slack:#eng", 0, 1_700_000_000_000, 100); let leaf = mk_leaf(&chunk_id_s, 100, 1_700_000_000_000); - route_leaf_to_topic_trees( - &cfg, - &leaf, - &[ - "email:alice@example.com".to_string(), - "hashtag:launch".to_string(), - ], - &summariser, - ) - .await - .unwrap(); + test_override::with_provider(provider, async { + route_leaf_to_topic_trees( + &cfg, + &leaf, + &[ + "email:alice@example.com".to_string(), + "hashtag:launch".to_string(), + ], + ) + .await + .unwrap() + }) + .await; // Both topic trees' L0 buffers hold the leaf. let b1 = src_store::get_buffer(&cfg, &t1.id, 0).unwrap(); @@ -297,7 +277,8 @@ mod tests { // new Alice-mentioning leaf routes into both the source tree AND // the topic tree. let (_tmp, cfg) = test_config(); - let summariser = InertSummariser::new(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); let entity_id = "email:alice@example.com"; // Pre-seed counters / index so the next call crosses threshold. @@ -306,14 +287,14 @@ mod tests { // to keep hotness above `TOPIC_CREATION_THRESHOLD` once the index // is queried (two indexed sources below → distinct_sources → 2). let mut counters = - crate::openhuman::memory_tree::tree_topic::types::HotnessCounters::fresh(entity_id, 0); + crate::openhuman::memory_store::trees::types::HotnessCounters::fresh(entity_id, 0); counters.mention_count_30d = 1_000; counters.distinct_sources = 2; counters.last_seen_ms = Some(Utc::now().timestamp_millis()); counters.query_hits_30d = 5; counters.ingests_since_check = - crate::openhuman::memory_tree::tree_topic::types::TOPIC_RECHECK_EVERY - 1; - crate::openhuman::memory_tree::tree_topic::store::upsert(&cfg, &counters).unwrap(); + crate::openhuman::memory_store::trees::types::TOPIC_RECHECK_EVERY - 1; + crate::openhuman::memory_store::trees::hotness::upsert(&cfg, &counters).unwrap(); // Seed leaves in slack and gmail referencing Alice. Anchor the // timestamps to "now" so the 30-day backfill window @@ -348,9 +329,12 @@ mod tests { score: 0.5, }; - route_leaf_to_topic_trees(&cfg, &leaf, &[entity_id.to_string()], &summariser) - .await - .unwrap(); + test_override::with_provider(provider, async { + route_leaf_to_topic_trees(&cfg, &leaf, &[entity_id.to_string()]) + .await + .unwrap() + }) + .await; // Topic tree now exists. let tree = src_store::get_tree_by_scope(&cfg, TreeKind::Topic, entity_id) diff --git a/src/openhuman/memory_tree/tree_source/bucket_seal.rs b/src/openhuman/memory_tree/tree/bucket_seal.rs similarity index 88% rename from src/openhuman/memory_tree/tree_source/bucket_seal.rs rename to src/openhuman/memory_tree/tree/bucket_seal.rs index 093454e47..18c82bd1a 100644 --- a/src/openhuman/memory_tree/tree_source/bucket_seal.rs +++ b/src/openhuman/memory_tree/tree/bucket_seal.rs @@ -39,21 +39,21 @@ use chrono::{DateTime, Utc}; use rusqlite::Transaction; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::content_store::{ +use crate::openhuman::memory::score::embed::build_embedder_from_config; +use crate::openhuman::memory::score::extract::EntityExtractor; +use crate::openhuman::memory::score::resolver::canonicalise; +use crate::openhuman::memory_store::chunks::store::with_connection; +use crate::openhuman::memory_store::content::{ atomic::stage_summary, paths::slugify_source_id, SummaryComposeInput, SummaryTreeKind, }; -use crate::openhuman::memory_tree::score::embed::build_embedder_from_config; -use crate::openhuman::memory_tree::score::extract::EntityExtractor; -use crate::openhuman::memory_tree::score::resolver::canonicalise; -use crate::openhuman::memory_tree::store::with_connection; -use crate::openhuman::memory_tree::tree_source::registry::new_summary_id; -use crate::openhuman::memory_tree::tree_source::store; -use crate::openhuman::memory_tree::tree_source::summariser::{ - Summariser, SummaryContext, SummaryInput, -}; -use crate::openhuman::memory_tree::tree_source::types::{ +use crate::openhuman::memory_store::trees::types::{ Buffer, SummaryNode, Tree, TreeKind, INPUT_TOKEN_BUDGET, OUTPUT_TOKEN_BUDGET, SUMMARY_FANOUT, }; +use crate::openhuman::memory_tree::summarise::{ + fallback_summary, summarise, SummaryContext, SummaryInput, +}; +use crate::openhuman::memory_tree::tree::registry::new_summary_id; +use crate::openhuman::memory_tree::tree::store; /// Hard cap on cascade depth — prevents runaway loops if token accounting /// ever slips. 32 levels at even a 2x fan-in is more than enough for any @@ -163,11 +163,10 @@ pub async fn append_leaf( config: &Config, tree: &Tree, leaf: &LeafRef, - summariser: &dyn Summariser, strategy: &LabelStrategy, ) -> Result> { log::debug!( - "[tree_source::bucket_seal] append_leaf tree_id={} leaf_id={} tokens={} strategy={:?}", + "[tree::bucket_seal] append_leaf tree_id={} leaf_id={} tokens={} strategy={:?}", tree.id, leaf.chunk_id, leaf.token_count, @@ -185,7 +184,7 @@ pub async fn append_leaf( )?; // 2. Cascade seals upward until a level stays under budget. - cascade_seals(config, tree, summariser, strategy).await + cascade_seals(config, tree, strategy).await } /// Queue-oriented variant of [`append_leaf`]. @@ -223,7 +222,7 @@ fn append_to_buffer( // stays on first-seen. if buf.item_ids.iter().any(|existing| existing == item_id) { log::debug!( - "[tree_source::bucket_seal] append_to_buffer: {item_id} already in buffer \ + "[tree::bucket_seal] append_to_buffer: {item_id} already in buffer \ tree_id={tree_id} level={level} — no-op" ); return Ok(()); @@ -243,10 +242,9 @@ fn append_to_buffer( async fn cascade_seals( config: &Config, tree: &Tree, - summariser: &dyn Summariser, strategy: &LabelStrategy, ) -> Result> { - cascade_all_from(config, tree, 0, summariser, None, strategy).await + cascade_all_from(config, tree, 0, None, strategy).await } /// Seal buffers starting at `start_level` and cascade upward. When @@ -260,7 +258,6 @@ pub async fn cascade_all_from( config: &Config, tree: &Tree, start_level: u32, - summariser: &dyn Summariser, force_now: Option>, strategy: &LabelStrategy, ) -> Result> { @@ -275,7 +272,7 @@ pub async fn cascade_all_from( if !forced && !should_seal(&buf) { log::debug!( - "[tree_source::bucket_seal] cascade done tree_id={} stop_level={} token_sum={}", + "[tree::bucket_seal] cascade done tree_id={} stop_level={} token_sum={}", tree.id, level, buf.token_sum @@ -284,7 +281,7 @@ pub async fn cascade_all_from( } if buf.is_empty() { log::debug!( - "[tree_source::bucket_seal] cascade hit empty buffer tree_id={} level={} — stopping", + "[tree::bucket_seal] cascade hit empty buffer tree_id={} level={} — stopping", tree.id, level ); @@ -293,7 +290,7 @@ pub async fn cascade_all_from( // Sync cascade — drives the level walk itself; doesn't need the // queue follow-ups (we'll hit `seal_one_level` again next iter). - let summary_id = seal_one_level(config, tree, &buf, summariser, strategy, false).await?; + let summary_id = seal_one_level(config, tree, &buf, strategy, false).await?; sealed_ids.push(summary_id); level += 1; } @@ -312,7 +309,7 @@ pub async fn cascade_all_from( /// /// Time-based sealing for low-volume sources is handled separately /// by `flush_stale_buffers` (see [`crate::openhuman::memory_tree:: -/// tree_source::flush::flush_stale_buffers`]), which filters buffers +/// tree::flush::flush_stale_buffers`]), which filters buffers /// by `oldest_at` before calling the cascade. Keeping the time gate /// out of `should_seal` avoids prematurely sealing buffers during /// normal `append_leaf` calls when test/restored data carries older @@ -358,7 +355,6 @@ pub(crate) async fn seal_one_level( config: &Config, tree: &Tree, buf: &Buffer, - summariser: &dyn Summariser, strategy: &LabelStrategy, enqueue_follow_ups: bool, ) -> Result { @@ -369,7 +365,7 @@ pub(crate) async fn seal_one_level( let inputs = hydrate_inputs(config, level, &buf.item_ids)?; if inputs.is_empty() { anyhow::bail!( - "[tree_source::bucket_seal] refused to seal empty buffer tree_id={} level={}", + "[tree::bucket_seal] refused to seal empty buffer tree_id={} level={}", tree.id, level ); @@ -399,10 +395,16 @@ pub(crate) async fn seal_one_level( target_level, token_budget: OUTPUT_TOKEN_BUDGET, }; - let output = summariser - .summarise(&inputs, &ctx) - .await - .context("summariser failed during seal")?; + let output = match summarise(config, &inputs, &ctx).await { + Ok(o) => o, + Err(e) => { + log::warn!( + "[memory_tree::seal] summarise failed for tree_id={} level={}: {e:#} — using fallback", + ctx.tree_id, ctx.target_level, + ); + fallback_summary(&inputs, ctx.token_budget) + } + }; // Resolve labels (entities/topics) for the new summary node according // to the chosen strategy. Done before the write tx so an extractor @@ -429,7 +431,7 @@ pub(crate) async fn seal_one_level( // even at 4× tokenizer ratio. let embed_input = truncate_for_embed(&output.content, 1_000); log::info!( - "[tree_source::bucket_seal] embed input: original_chars={} truncated_chars={}", + "[tree::bucket_seal] embed input: original_chars={} truncated_chars={}", output.content.len(), embed_input.len() ); @@ -440,7 +442,7 @@ pub(crate) async fn seal_one_level( ) })?; log::debug!( - "[tree_source::bucket_seal] embedded summary tree_id={} level={}→{} bytes={} provider={}", + "[tree::bucket_seal] embedded summary tree_id={} level={}→{} bytes={} provider={}", tree.id, level, target_level, @@ -537,7 +539,9 @@ pub(crate) async fn seal_one_level( // We still yield `None` (so `compose_summary_md` // takes the sanitised-id fallback) but a warn log // makes the SQL error visible for diagnosis. - match crate::openhuman::memory_tree::store::get_chunk_raw_refs(config, chunk_id) { + match crate::openhuman::memory_store::chunks::store::get_chunk_raw_refs( + config, chunk_id, + ) { Ok(Some(refs)) if !refs.is_empty() => { // RawRef::path is a forward-slash relative path // under content_root, e.g. @@ -556,14 +560,14 @@ pub(crate) async fn seal_one_level( // Obsidian link. Log so the silent-degradation // path stays visible during diagnosis. log::debug!( - "[tree_source::bucket_seal] no raw_refs for chunk_id={chunk_id} \ + "[tree::bucket_seal] no raw_refs for chunk_id={chunk_id} \ — wikilink will fall back to sanitised chunk id" ); None } Err(e) => { log::warn!( - "[tree_source::bucket_seal] get_chunk_raw_refs failed \ + "[tree::bucket_seal] get_chunk_raw_refs failed \ chunk_id={chunk_id} err={e:#} — falling back to \ chunk_id-based wikilink" ); @@ -600,12 +604,10 @@ pub(crate) async fn seal_one_level( // without manual configuration. Best-effort and idempotent — never // overwrites an existing file. if let Err(err) = - crate::openhuman::memory_tree::content_store::obsidian::ensure_obsidian_defaults( - &content_root, - ) + crate::openhuman::memory_store::content::obsidian::ensure_obsidian_defaults(&content_root) { log::warn!( - "[tree_source::bucket_seal] ensure_obsidian_defaults failed: {err:#} — \ + "[tree::bucket_seal] ensure_obsidian_defaults failed: {err:#} — \ continuing seal without vault defaults" ); } @@ -617,7 +619,7 @@ pub(crate) async fn seal_one_level( ) })?; log::debug!( - "[tree_source::bucket_seal] staged summary {} → {}", + "[tree::bucket_seal] staged summary {} → {}", node.id, staged.content_path ); @@ -648,15 +650,15 @@ pub(crate) async fn seal_one_level( &tx, &node, Some(&staged), - &crate::openhuman::memory_tree::store::tree_active_signature(config), + &crate::openhuman::memory_store::chunks::store::tree_active_signature(config), )?; // Forward-compat: index any entities the summariser emitted into // `mem_tree_entity_index` so Phase 4 retrieval can resolve // "summaries mentioning Alice" via the same inverted index as - // leaves. No-op under InertSummariser (entities is empty by - // design — see summariser/inert.rs doc); becomes active once the - // Ollama summariser lands and emits curated canonical ids. - crate::openhuman::memory_tree::score::store::index_summary_entity_ids_tx( + // leaves. No-op when entities is empty (the current summarise() + // always emits empty — entity extraction is the learning domain's job); + // becomes active once the summariser or a post-seal extractor emits canonical ids. + crate::openhuman::memory::score::store::index_summary_entity_ids_tx( &tx, &node.entities, &node.id, @@ -709,8 +711,8 @@ pub(crate) async fn seal_one_level( // `seal:{tree_id}:{parent_level}` prevents duplicates if a // parallel path already queued it. if should_seal(&parent) { - use crate::openhuman::memory_tree::jobs::store::enqueue_tx as enqueue_job_tx; - use crate::openhuman::memory_tree::jobs::types::{NewJob, SealPayload}; + use crate::openhuman::memory::jobs::store::enqueue_tx as enqueue_job_tx; + use crate::openhuman::memory::jobs::types::{NewJob, SealPayload}; let parent_seal = SealPayload { tree_id: tree_id.clone(), level: target_level_for_closure, @@ -722,10 +724,8 @@ pub(crate) async fn seal_one_level( // entities back into the topic-tree spawn pipeline. Topic // and global trees are sinks — no fan-out from their seals. if matches!(tree_kind, TreeKind::Source) { - use crate::openhuman::memory_tree::jobs::store::enqueue_tx as enqueue_job_tx; - use crate::openhuman::memory_tree::jobs::types::{ - NewJob, NodeRef, TopicRoutePayload, - }; + use crate::openhuman::memory::jobs::store::enqueue_tx as enqueue_job_tx; + use crate::openhuman::memory::jobs::types::{NewJob, NodeRef, TopicRoutePayload}; let route = TopicRoutePayload { node: NodeRef::Summary { summary_id: summary_id_for_closure.clone(), @@ -756,7 +756,7 @@ pub(crate) async fn seal_one_level( })?; log::info!( - "[tree_source::bucket_seal] sealed tree_id={} level={}→{} summary_id={} children={}", + "[tree::bucket_seal] sealed tree_id={} level={}→{} summary_id={} children={}", tree.id, level, target_level, @@ -774,7 +774,7 @@ pub(crate) async fn seal_one_level( /// HTTP 500 from Ollama rather than auto-truncating, which would /// abort the seal transaction. fn truncate_for_embed(text: &str, max_tokens: u32) -> String { - let approx = crate::openhuman::memory_tree::types::approx_token_count(text); + let approx = crate::openhuman::memory_store::chunks::types::approx_token_count(text); if approx <= max_tokens { return text.to_string(); } @@ -807,9 +807,9 @@ fn hydrate_inputs(config: &Config, level: u32, item_ids: &[String]) -> Result Result> { - use crate::openhuman::memory_tree::content_store::read as content_read; - use crate::openhuman::memory_tree::score::store::{get_score, list_entity_ids_for_node}; - use crate::openhuman::memory_tree::store::get_chunk; + use crate::openhuman::memory::score::store::{get_score, list_entity_ids_for_node}; + use crate::openhuman::memory_store::chunks::store::get_chunk; + use crate::openhuman::memory_store::content::read as content_read; let mut out: Vec = Vec::with_capacity(chunk_ids.len()); for id in chunk_ids { @@ -817,7 +817,7 @@ fn hydrate_leaf_inputs(config: &Config, chunk_ids: &[String]) -> Result c, None => { log::warn!( - "[tree_source::bucket_seal] hydrate_leaf_inputs: missing chunk {id} — skipping" + "[tree::bucket_seal] hydrate_leaf_inputs: missing chunk {id} — skipping" ); continue; } @@ -838,7 +838,7 @@ fn hydrate_leaf_inputs(config: &Config, chunk_ids: &[String]) -> Result Result Result> { - use crate::openhuman::memory_tree::content_store::read as content_read; + use crate::openhuman::memory_store::content::read as content_read; let mut out: Vec = Vec::with_capacity(summary_ids.len()); for id in summary_ids { @@ -863,7 +863,7 @@ fn hydrate_summary_inputs(config: &Config, summary_ids: &[String]) -> Result n, None => { log::warn!( - "[tree_source::bucket_seal] hydrate_summary_inputs: missing summary {id} — skipping" + "[tree::bucket_seal] hydrate_summary_inputs: missing summary {id} — skipping" ); continue; } @@ -872,7 +872,7 @@ fn hydrate_summary_inputs(config: &Config, summary_ids: &[String]) -> Result LeafRef { async fn append_below_budget_does_not_seal() { let (_tmp, cfg) = test_config(); let tree = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); - let summariser = InertSummariser::new(); + let provider: Arc = Arc::new(StaticChatProvider::new("test summary content")); // Chunks don't exist in DB — we're only exercising the buffer // accounting, which doesn't require leaf rows until a seal fires. let leaf = mk_leaf("leaf-1", 100, 1_700_000_000_000); - let sealed = append_leaf(&cfg, &tree, &leaf, &summariser, &LabelStrategy::Empty) - .await - .unwrap(); + let sealed = test_override::with_provider(provider, async { + append_leaf(&cfg, &tree, &leaf, &LabelStrategy::Empty) + .await + .unwrap() + }) + .await; assert!(sealed.is_empty()); let buf = store::get_buffer(&cfg, &tree.id, 0).unwrap(); @@ -74,13 +81,15 @@ async fn append_below_budget_does_not_seal() { #[tokio::test] async fn crossing_budget_triggers_seal() { - use crate::openhuman::memory_tree::store::upsert_chunks; - use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; + use crate::openhuman::memory_store::chunks::store::upsert_chunks; + use crate::openhuman::memory_store::chunks::types::{ + chunk_id, Chunk, Metadata, SourceKind, SourceRef, + }; use chrono::TimeZone; let (_tmp, cfg) = test_config(); let tree = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); - let summariser = InertSummariser::new(); + let provider: Arc = Arc::new(StaticChatProvider::new("test summary content")); // Persist two chunks that the hydrator can load during seal. let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(); @@ -130,14 +139,20 @@ async fn crossing_budget_triggers_seal() { score: 0.5, }; - let first = append_leaf(&cfg, &tree, &leaf1, &summariser, &LabelStrategy::Empty) - .await - .unwrap(); + let first = test_override::with_provider(Arc::clone(&provider), async { + append_leaf(&cfg, &tree, &leaf1, &LabelStrategy::Empty) + .await + .unwrap() + }) + .await; assert!(first.is_empty(), "first append below budget — no seal"); - let second = append_leaf(&cfg, &tree, &leaf2, &summariser, &LabelStrategy::Empty) - .await - .unwrap(); + let second = test_override::with_provider(Arc::clone(&provider), async { + append_leaf(&cfg, &tree, &leaf2, &LabelStrategy::Empty) + .await + .unwrap() + }) + .await; assert_eq!(second.len(), 1, "second append crosses budget — one seal"); let summary_id = &second[0]; @@ -159,7 +174,7 @@ async fn crossing_budget_triggers_seal() { assert!(t.last_sealed_at.is_some()); // Leaf → parent backlink populated for both children. - use crate::openhuman::memory_tree::store::with_connection; + use crate::openhuman::memory_store::chunks::store::with_connection; let parent: Option = with_connection(&cfg, |conn| { let p: Option = conn .query_row( @@ -176,14 +191,16 @@ async fn crossing_budget_triggers_seal() { #[tokio::test] async fn fanout_at_l1_triggers_l2_seal() { - use crate::openhuman::memory_tree::store::upsert_chunks; - use crate::openhuman::memory_tree::tree_source::types::SUMMARY_FANOUT; - use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; + use crate::openhuman::memory_store::chunks::store::upsert_chunks; + use crate::openhuman::memory_store::chunks::types::{ + chunk_id, Chunk, Metadata, SourceKind, SourceRef, + }; + use crate::openhuman::memory_store::trees::types::SUMMARY_FANOUT; use chrono::TimeZone; let (_tmp, cfg) = test_config(); let tree = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); - let summariser = InertSummariser::new(); + let provider: Arc = Arc::new(StaticChatProvider::new("test summary content")); let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(); let mk_chunk = |seq: u32| { @@ -226,9 +243,12 @@ async fn fanout_at_l1_triggers_l2_seal() { topics: vec![], score: 0.5, }; - let sealed = append_leaf(&cfg, &tree, &leaf, &summariser, &LabelStrategy::Empty) - .await - .unwrap(); + let sealed = test_override::with_provider(Arc::clone(&provider), async { + append_leaf(&cfg, &tree, &leaf, &LabelStrategy::Empty) + .await + .unwrap() + }) + .await; all_sealed.extend(sealed); } @@ -265,14 +285,16 @@ async fn fanout_at_l1_triggers_l2_seal() { #[tokio::test] async fn upper_level_does_not_seal_below_fanout() { - use crate::openhuman::memory_tree::store::upsert_chunks; - use crate::openhuman::memory_tree::tree_source::types::SUMMARY_FANOUT; - use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; + use crate::openhuman::memory_store::chunks::store::upsert_chunks; + use crate::openhuman::memory_store::chunks::types::{ + chunk_id, Chunk, Metadata, SourceKind, SourceRef, + }; + use crate::openhuman::memory_store::trees::types::SUMMARY_FANOUT; use chrono::TimeZone; let (_tmp, cfg) = test_config(); let tree = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); - let summariser = InertSummariser::new(); + let provider: Arc = Arc::new(StaticChatProvider::new("test summary content")); let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(); // Emit (fanout - 1) L1 summaries — should leave the L1 buffer @@ -309,9 +331,12 @@ async fn upper_level_does_not_seal_below_fanout() { topics: vec![], score: 0.5, }; - let _ = append_leaf(&cfg, &tree, &leaf, &summariser, &LabelStrategy::Empty) - .await - .unwrap(); + let _ = test_override::with_provider(Arc::clone(&provider), async { + append_leaf(&cfg, &tree, &leaf, &LabelStrategy::Empty) + .await + .unwrap() + }) + .await; } let t = store::get_tree(&cfg, &tree.id).unwrap().unwrap(); @@ -348,11 +373,13 @@ fn seed_leaf( entities: Vec, topics: Vec, ) -> LeafRef { - use crate::openhuman::memory_tree::score::extract::EntityKind; - use crate::openhuman::memory_tree::score::resolver::CanonicalEntity; - use crate::openhuman::memory_tree::score::store::index_entity; - use crate::openhuman::memory_tree::store::upsert_chunks; - use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; + use crate::openhuman::memory::score::extract::EntityKind; + use crate::openhuman::memory::score::resolver::CanonicalEntity; + use crate::openhuman::memory::score::store::index_entity; + use crate::openhuman::memory_store::chunks::store::upsert_chunks; + use crate::openhuman::memory_store::chunks::types::{ + chunk_id, Chunk, Metadata, SourceKind, SourceRef, + }; use chrono::TimeZone; let ts = Utc .timestamp_millis_opt(1_700_000_000_000 + seq as i64) @@ -413,17 +440,16 @@ fn seed_leaf( #[tokio::test] async fn seal_with_extract_strategy_populates_entities_and_topics() { - use crate::openhuman::memory_tree::score::extract::{CompositeExtractor, EntityExtractor}; - use std::sync::Arc; + use crate::openhuman::memory::score::extract::{CompositeExtractor, EntityExtractor}; let (_tmp, cfg) = test_config(); let tree = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); - let summariser = InertSummariser::new(); + let provider: Arc = Arc::new(StaticChatProvider::new( + "alice@example.com is leading the #launch sprint this week.", + )); // Content the regex extractor can find: an email and a hashtag. The - // inert summariser concatenates leaf content into the L1 summary, so - // these tokens survive into the summary text and the extractor finds - // them when run on the summary content. + // StaticChatProvider returns content that the extractor finds. let leaf = seed_leaf( &cfg, 0, @@ -435,9 +461,10 @@ async fn seal_with_extract_strategy_populates_entities_and_topics() { let extractor: Arc = Arc::new(CompositeExtractor::regex_only()); let strategy = LabelStrategy::ExtractFromContent(extractor); - let sealed = append_leaf(&cfg, &tree, &leaf, &summariser, &strategy) - .await - .unwrap(); + let sealed = test_override::with_provider(provider, async { + append_leaf(&cfg, &tree, &leaf, &strategy).await.unwrap() + }) + .await; assert_eq!(sealed.len(), 1, "single 10k-token leaf should seal L0→L1"); let summary = store::get_summary(&cfg, &sealed[0]).unwrap().unwrap(); @@ -460,7 +487,7 @@ async fn seal_with_extract_strategy_populates_entities_and_topics() { async fn seal_with_union_strategy_inherits_labels_from_children() { let (_tmp, cfg) = test_config(); let tree = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); - let summariser = InertSummariser::new(); + let provider: Arc = Arc::new(StaticChatProvider::new("test summary content")); // Two leaves with overlapping + distinct labels. Union should // dedup-merge them into the parent. @@ -501,26 +528,20 @@ async fn seal_with_union_strategy_inherits_labels_from_children() { }; // First leaf: under budget, no seal. - let sealed_1 = append_leaf( - &cfg, - &tree, - &leaf1, - &summariser, - &LabelStrategy::UnionFromChildren, - ) - .await - .unwrap(); + let sealed_1 = test_override::with_provider(Arc::clone(&provider), async { + append_leaf(&cfg, &tree, &leaf1, &LabelStrategy::UnionFromChildren) + .await + .unwrap() + }) + .await; assert!(sealed_1.is_empty()); // Second leaf: crosses budget → one seal covering both leaves. - let sealed_2 = append_leaf( - &cfg, - &tree, - &leaf2, - &summariser, - &LabelStrategy::UnionFromChildren, - ) - .await - .unwrap(); + let sealed_2 = test_override::with_provider(Arc::clone(&provider), async { + append_leaf(&cfg, &tree, &leaf2, &LabelStrategy::UnionFromChildren) + .await + .unwrap() + }) + .await; assert_eq!(sealed_2.len(), 1); let summary = store::get_summary(&cfg, &sealed_2[0]).unwrap().unwrap(); @@ -546,7 +567,7 @@ async fn seal_with_union_strategy_inherits_labels_from_children() { async fn seal_with_empty_strategy_leaves_labels_empty() { let (_tmp, cfg) = test_config(); let tree = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); - let summariser = InertSummariser::new(); + let provider: Arc = Arc::new(StaticChatProvider::new("test summary content")); // Leaf carries labels — Empty strategy should ignore them. let leaf = seed_leaf( @@ -557,9 +578,12 @@ async fn seal_with_empty_strategy_leaves_labels_empty() { vec!["launch".into()], ); - let sealed = append_leaf(&cfg, &tree, &leaf, &summariser, &LabelStrategy::Empty) - .await - .unwrap(); + let sealed = test_override::with_provider(provider, async { + append_leaf(&cfg, &tree, &leaf, &LabelStrategy::Empty) + .await + .unwrap() + }) + .await; assert_eq!(sealed.len(), 1); let summary = store::get_summary(&cfg, &sealed[0]).unwrap().unwrap(); @@ -577,7 +601,7 @@ async fn seal_with_empty_strategy_leaves_labels_empty() { #[tokio::test] async fn topic_tree_seal_persists_topic_kind_not_source() { - use crate::openhuman::memory_tree::tree_source::types::TreeStatus; + use crate::openhuman::memory_store::trees::types::TreeStatus; let (_tmp, cfg) = test_config(); // Build a topic tree directly — `seal_one_level` runs for both @@ -595,12 +619,15 @@ async fn topic_tree_seal_persists_topic_kind_not_source() { }; store::insert_tree(&cfg, &tree).unwrap(); - let summariser = InertSummariser::new(); + let provider: Arc = Arc::new(StaticChatProvider::new("test summary content")); let leaf = seed_leaf(&cfg, 0, "topic content", vec![], vec![]); - let sealed = append_leaf(&cfg, &tree, &leaf, &summariser, &LabelStrategy::Empty) - .await - .unwrap(); + let sealed = test_override::with_provider(provider, async { + append_leaf(&cfg, &tree, &leaf, &LabelStrategy::Empty) + .await + .unwrap() + }) + .await; assert_eq!(sealed.len(), 1); let summary = store::get_summary(&cfg, &sealed[0]).unwrap().unwrap(); @@ -616,7 +643,7 @@ fn scope_slug_non_gmail_uses_full_scope() { // slack:#eng and discord:#eng must NOT produce the same scope slug. // Previously, stripping everything before ':' made both → "eng". // After Fix K, only gmail: strips the prefix — others use the full string. - use crate::openhuman::memory_tree::content_store::paths::slugify_source_id; + use crate::openhuman::memory_store::content::paths::slugify_source_id; // Verify that the slug logic produces distinct values for different platforms. let slack_slug = slugify_source_id("slack:#eng"); diff --git a/src/openhuman/memory_tree/tree_source/flush.rs b/src/openhuman/memory_tree/tree/flush.rs similarity index 73% rename from src/openhuman/memory_tree/tree_source/flush.rs rename to src/openhuman/memory_tree/tree/flush.rs index a6470c279..6a5954a2b 100644 --- a/src/openhuman/memory_tree/tree_source/flush.rs +++ b/src/openhuman/memory_tree/tree/flush.rs @@ -15,10 +15,9 @@ use anyhow::Result; use chrono::{DateTime, Duration, Utc}; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::tree_source::bucket_seal::{cascade_all_from, LabelStrategy}; -use crate::openhuman::memory_tree::tree_source::store; -use crate::openhuman::memory_tree::tree_source::summariser::Summariser; -use crate::openhuman::memory_tree::tree_source::types::DEFAULT_FLUSH_AGE_SECS; +use crate::openhuman::memory_store::trees::types::DEFAULT_FLUSH_AGE_SECS; +use crate::openhuman::memory_tree::tree::bucket_seal::{cascade_all_from, LabelStrategy}; +use crate::openhuman::memory_tree::tree::store; /// Seal every buffer whose oldest item is older than `max_age`. Returns /// the number of individual seal calls (not trees) that fired. When the @@ -26,14 +25,13 @@ use crate::openhuman::memory_tree::tree_source::types::DEFAULT_FLUSH_AGE_SECS; pub async fn flush_stale_buffers( config: &Config, max_age: Duration, - summariser: &dyn Summariser, strategy: &LabelStrategy, ) -> Result { let now = Utc::now(); let cutoff = now - max_age; let stale = store::list_stale_buffers(config, cutoff)?; log::info!( - "[tree_source::flush] found {} stale buffers (max_age={:?})", + "[tree::flush] found {} stale buffers (max_age={:?})", stale.len(), max_age ); @@ -44,15 +42,14 @@ pub async fn flush_stale_buffers( Some(t) => t, None => { log::warn!( - "[tree_source::flush] orphan buffer tree_id={} level={}", + "[tree::flush] orphan buffer tree_id={} level={}", buf.tree_id, buf.level ); continue; } }; - let sealed = - cascade_all_from(config, &tree, buf.level, summariser, Some(now), strategy).await?; + let sealed = cascade_all_from(config, &tree, buf.level, Some(now), strategy).await?; seals += sealed.len(); } Ok(seals) @@ -61,16 +58,9 @@ pub async fn flush_stale_buffers( /// Convenience wrapper that uses [`DEFAULT_FLUSH_AGE_SECS`]. pub async fn flush_stale_buffers_default( config: &Config, - summariser: &dyn Summariser, strategy: &LabelStrategy, ) -> Result { - flush_stale_buffers( - config, - Duration::seconds(DEFAULT_FLUSH_AGE_SECS), - summariser, - strategy, - ) - .await + flush_stale_buffers(config, Duration::seconds(DEFAULT_FLUSH_AGE_SECS), strategy).await } /// Helper exposed for callers that want a single explicit force-seal (e.g. @@ -78,24 +68,26 @@ pub async fn flush_stale_buffers_default( pub async fn force_flush_tree( config: &Config, tree_id: &str, - summariser: &dyn Summariser, now: Option>, strategy: &LabelStrategy, ) -> Result> { let tree = store::get_tree(config, tree_id)? .ok_or_else(|| anyhow::anyhow!("no tree with id {tree_id}"))?; - cascade_all_from(config, &tree, 0, summariser, now, strategy).await + cascade_all_from(config, &tree, 0, now, strategy).await } #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory_tree::content_store; - use crate::openhuman::memory_tree::store::upsert_chunks; - use crate::openhuman::memory_tree::tree_source::bucket_seal::{append_leaf, LeafRef}; - use crate::openhuman::memory_tree::tree_source::registry::get_or_create_source_tree; - use crate::openhuman::memory_tree::tree_source::summariser::inert::InertSummariser; - use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; + use crate::openhuman::memory::chat::{test_override, ChatProvider, StaticChatProvider}; + use crate::openhuman::memory_store::chunks::store::upsert_chunks; + use crate::openhuman::memory_store::chunks::types::{ + chunk_id, Chunk, Metadata, SourceKind, SourceRef, + }; + use crate::openhuman::memory_store::content as content_store; + use crate::openhuman::memory_tree::sources::registry::get_or_create_source_tree; + use crate::openhuman::memory_tree::tree::bucket_seal::{append_leaf, LeafRef}; + use std::sync::Arc; use tempfile::TempDir; fn stage_test_chunks(cfg: &Config, chunks: &[Chunk]) { @@ -103,9 +95,9 @@ mod tests { std::fs::create_dir_all(&content_root).expect("create content_root for test"); let staged = content_store::stage_chunks(&content_root, chunks) .expect("stage_chunks for test chunks"); - crate::openhuman::memory_tree::store::with_connection(cfg, |conn| { + crate::openhuman::memory_store::chunks::store::with_connection(cfg, |conn| { let tx = conn.unchecked_transaction()?; - crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?; + crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(&tx, &staged)?; tx.commit()?; Ok(()) }) @@ -127,7 +119,8 @@ mod tests { async fn flush_seals_old_buffer_even_under_budget() { let (_tmp, cfg) = test_config(); let tree = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); - let summariser = InertSummariser::new(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); // Persist one chunk with an old timestamp (10 days ago). let old_ts = Utc::now() - Duration::days(10); @@ -160,15 +153,20 @@ mod tests { topics: vec![], score: 0.5, }; - append_leaf(&cfg, &tree, &leaf, &summariser, &LabelStrategy::Empty) - .await - .unwrap(); - assert_eq!(store::count_summaries(&cfg, &tree.id).unwrap(), 0); - - let seals = - flush_stale_buffers(&cfg, Duration::days(7), &summariser, &LabelStrategy::Empty) + test_override::with_provider(Arc::clone(&provider), async { + append_leaf(&cfg, &tree, &leaf, &LabelStrategy::Empty) .await .unwrap(); + }) + .await; + assert_eq!(store::count_summaries(&cfg, &tree.id).unwrap(), 0); + + let seals = test_override::with_provider(Arc::clone(&provider), async { + flush_stale_buffers(&cfg, Duration::days(7), &LabelStrategy::Empty) + .await + .unwrap() + }) + .await; assert_eq!(seals, 1); assert_eq!(store::count_summaries(&cfg, &tree.id).unwrap(), 1); @@ -187,16 +185,17 @@ mod tests { // on `SUMMARY_FANOUT` naturally. let (_tmp, cfg) = test_config(); let tree = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); - let summariser = InertSummariser::new(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); // Plant a stale L1 buffer holding a single (synthetic) child id. // No L0 chunks — the only thing flush could touch is the L1 buffer. let old_ts = Utc::now() - Duration::days(10); - crate::openhuman::memory_tree::store::with_connection(&cfg, |conn| { + crate::openhuman::memory_store::chunks::store::with_connection(&cfg, |conn| { let tx = conn.unchecked_transaction()?; - crate::openhuman::memory_tree::tree_source::store::upsert_buffer_tx( + crate::openhuman::memory_store::trees::store::upsert_buffer_tx( &tx, - &crate::openhuman::memory_tree::tree_source::types::Buffer { + &crate::openhuman::memory_store::trees::types::Buffer { tree_id: tree.id.clone(), level: 1, item_ids: vec!["fake-l1-child".into()], @@ -209,10 +208,12 @@ mod tests { }) .unwrap(); - let seals = - flush_stale_buffers(&cfg, Duration::days(7), &summariser, &LabelStrategy::Empty) + let seals = test_override::with_provider(provider, async { + flush_stale_buffers(&cfg, Duration::days(7), &LabelStrategy::Empty) .await - .unwrap(); + .unwrap() + }) + .await; assert_eq!(seals, 0, "L1 stale buffer must not be force-sealed"); assert_eq!(store::count_summaries(&cfg, &tree.id).unwrap(), 0); @@ -225,7 +226,8 @@ mod tests { async fn flush_noop_when_buffer_is_recent() { let (_tmp, cfg) = test_config(); let tree = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); - let summariser = InertSummariser::new(); + let provider: Arc = + Arc::new(StaticChatProvider::new("test summary content")); // Persist a leaf stamped now so it's NOT stale. let now = Utc::now(); @@ -256,14 +258,19 @@ mod tests { topics: vec![], score: 0.5, }; - append_leaf(&cfg, &tree, &leaf, &summariser, &LabelStrategy::Empty) - .await - .unwrap(); - - let seals = - flush_stale_buffers(&cfg, Duration::days(7), &summariser, &LabelStrategy::Empty) + test_override::with_provider(Arc::clone(&provider), async { + append_leaf(&cfg, &tree, &leaf, &LabelStrategy::Empty) .await .unwrap(); + }) + .await; + + let seals = test_override::with_provider(provider, async { + flush_stale_buffers(&cfg, Duration::days(7), &LabelStrategy::Empty) + .await + .unwrap() + }) + .await; assert_eq!(seals, 0); assert_eq!(store::count_summaries(&cfg, &tree.id).unwrap(), 0); } diff --git a/src/openhuman/memory_tree/tree/mod.rs b/src/openhuman/memory_tree/tree/mod.rs new file mode 100644 index 000000000..41ca80bee --- /dev/null +++ b/src/openhuman/memory_tree/tree/mod.rs @@ -0,0 +1,31 @@ +//! Generic summary-tree mechanics shared by all three tree flavors: +//! Source (per ingest source), Global (cross-source digest), and Topic +//! (per-entity). Covers storage, buffer management, bucket-seal cascade, +//! time-based flush, and the get-or-create registry primitive. +//! +//! Source-specific policy (the `_source.md` on-disk mirror, the +//! `get_or_create_source_tree` wrapper) lives in the sibling +//! [`crate::openhuman::memory_tree::sources`] module. +//! +//! Global and topic policies (scope constants, hotness gates, curator) +//! live in [`crate::openhuman::memory_tree::global`] and +//! [`crate::openhuman::memory_tree::topic`] respectively; both +//! import generic primitives from this module. +//! +//! Persistence (store + types) has moved to `memory_store::trees`. + +pub mod bucket_seal; +pub mod flush; +pub mod registry; + +// Re-export persistence from memory_store so callers using tree::store / tree::types still work. +pub use crate::openhuman::memory_store::trees::store; +pub use crate::openhuman::memory_store::trees::types; + +pub use crate::openhuman::memory_store::trees::{get_summary_embedding, set_summary_embedding}; +pub use crate::openhuman::memory_store::trees::{ + Buffer, SummaryNode, Tree, TreeKind, TreeStatus, INPUT_TOKEN_BUDGET, OUTPUT_TOKEN_BUDGET, + SUMMARY_FANOUT, +}; +pub use bucket_seal::{append_leaf, append_leaf_deferred, LabelStrategy, LeafRef}; +pub use registry::{get_or_create_tree, new_summary_id, new_tree_id}; diff --git a/src/openhuman/memory_tree/tree_source/registry.rs b/src/openhuman/memory_tree/tree/registry.rs similarity index 54% rename from src/openhuman/memory_tree/tree_source/registry.rs rename to src/openhuman/memory_tree/tree/registry.rs index f507a523d..c8be7a199 100644 --- a/src/openhuman/memory_tree/tree_source/registry.rs +++ b/src/openhuman/memory_tree/tree/registry.rs @@ -1,43 +1,39 @@ -//! Tree registry — get-or-create for source trees (#709). +//! Generic tree registry — get-or-create for any tree kind (#709). //! -//! The registry is the entry point for the ingest path to look up the -//! tree for a given (kind, scope). Phase 3a only touches source trees; -//! topic / global trees will reuse the same `(kind, scope)` convention -//! in Phases 3b / 3c. +//! All three tree flavors (Source, Global, Topic) share `UNIQUE(kind, scope)` +//! and the same race-recovery dance — there is no reason for three copies. +//! Source-specific side-effects (writing the `_source.md` mirror) live in +//! the `sources::registry` wrapper rather than here. use anyhow::Result; use chrono::Utc; use uuid::Uuid; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::tree_source::source_file::write_source_file; -use crate::openhuman::memory_tree::tree_source::store; -use crate::openhuman::memory_tree::tree_source::types::{Tree, TreeKind, TreeStatus}; +use crate::openhuman::memory_store::trees::types::{Tree, TreeKind, TreeStatus}; +use crate::openhuman::memory_tree::tree::store; -/// Look up the source tree for `scope`, or create a new one. +/// Generic get-or-create. All three tree flavors (Source, Global, Topic) +/// share UNIQUE(kind, scope) and the same race-recovery dance — there's +/// no reason for three copies. /// -/// Scope format convention (Phase 3a): use the ingested chunk's -/// `metadata.source_id` verbatim, so re-ingesting the same Slack channel -/// or Gmail account keeps appending to the same tree. -pub fn get_or_create_source_tree(config: &Config, scope: &str) -> Result { - if let Some(existing) = store::get_tree_by_scope(config, TreeKind::Source, scope)? { +/// Source-specific side-effects (writing the `_source.md` on-disk mirror) +/// are NOT performed here; callers that need them should go through +/// [`crate::openhuman::memory_tree::sources::registry::get_or_create_source_tree`]. +pub fn get_or_create_tree(config: &Config, kind: TreeKind, scope: &str) -> Result { + if let Some(existing) = store::get_tree_by_scope(config, kind, scope)? { log::debug!( - "[tree_source::registry] found tree id={} scope={}", + "[tree::registry] found tree id={} kind={} scope={}", existing.id, + kind.as_str(), scope ); - // Refresh the `_source.md` mirror — cheap idempotent rewrite, - // keeps the on-disk view current even if a previous run wrote - // the row before this file existed (or the file was deleted). - if let Err(e) = write_source_file(config, &existing) { - log::warn!("[tree_source::registry] write_source_file failed scope={scope} err={e:#}"); - } return Ok(existing); } let tree = Tree { - id: new_tree_id(TreeKind::Source), - kind: TreeKind::Source, + id: new_tree_id(kind), + kind, scope: scope.to_string(), root_id: None, max_level: 0, @@ -48,28 +44,27 @@ pub fn get_or_create_source_tree(config: &Config, scope: &str) -> Result { match store::insert_tree(config, &tree) { Ok(()) => { log::info!( - "[tree_source::registry] created source tree id={} scope={}", + "[tree::registry] created tree id={} kind={} scope={}", tree.id, + kind.as_str(), scope ); - if let Err(e) = write_source_file(config, &tree) { - log::warn!( - "[tree_source::registry] write_source_file failed scope={scope} err={e:#}" - ); - } Ok(tree) } Err(err) if is_unique_violation(&err) => { - // Race: another caller created a tree for the same scope - // between our initial lookup and this insert. UNIQUE(kind, - // scope) rejected our row; re-query and return the winner. + // Race: another caller created a tree for the same (kind, scope) + // between our initial lookup and this insert. UNIQUE(kind, scope) + // rejected our row; re-query and return the winner. log::debug!( - "[tree_source::registry] UNIQUE race for scope={} — re-querying", + "[tree::registry] UNIQUE race for kind={} scope={} — re-querying", + kind.as_str(), scope ); - store::get_tree_by_scope(config, TreeKind::Source, scope)?.ok_or_else(|| { + store::get_tree_by_scope(config, kind, scope)?.ok_or_else(|| { anyhow::anyhow!( - "UNIQUE violation on insert but no row found on re-query for scope {scope}" + "UNIQUE violation on insert but no row found on re-query for kind={} scope={}", + kind.as_str(), + scope ) }) } @@ -80,7 +75,7 @@ pub fn get_or_create_source_tree(config: &Config, scope: &str) -> Result { /// Return true if `err` represents a SQLite UNIQUE constraint violation. /// Matches both the anyhow-wrapped rusqlite error text and the raw SQLite /// error codes in case the wrapping chain is shorter. -fn is_unique_violation(err: &anyhow::Error) -> bool { +pub fn is_unique_violation(err: &anyhow::Error) -> bool { if let Some(rusqlite_err) = err.downcast_ref::() { if let rusqlite::Error::SqliteFailure(sqlite_err, _) = rusqlite_err { return sqlite_err.code == rusqlite::ErrorCode::ConstraintViolation; @@ -91,7 +86,8 @@ fn is_unique_violation(err: &anyhow::Error) -> bool { msg.contains("UNIQUE constraint failed") } -fn new_tree_id(kind: TreeKind) -> String { +/// Generate a stable id for a new tree row, prefixed with the kind discriminator. +pub fn new_tree_id(kind: TreeKind) -> String { format!("{}:{}", kind.as_str(), Uuid::new_v4()) } @@ -127,8 +123,8 @@ mod tests { #[test] fn get_or_create_is_idempotent_on_scope() { let (_tmp, cfg) = test_config(); - let first = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); - let second = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); + let first = get_or_create_tree(&cfg, TreeKind::Source, "slack:#eng").unwrap(); + let second = get_or_create_tree(&cfg, TreeKind::Source, "slack:#eng").unwrap(); assert_eq!(first.id, second.id); assert_eq!(first.kind, TreeKind::Source); assert_eq!(first.status, TreeStatus::Active); @@ -137,35 +133,49 @@ mod tests { #[test] fn different_scopes_yield_different_trees() { let (_tmp, cfg) = test_config(); - let a = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); - let b = get_or_create_source_tree(&cfg, "gmail:user@example.com").unwrap(); + let a = get_or_create_tree(&cfg, TreeKind::Source, "slack:#eng").unwrap(); + let b = get_or_create_tree(&cfg, TreeKind::Source, "gmail:user@example.com").unwrap(); assert_ne!(a.id, b.id); assert_ne!(a.scope, b.scope); } + #[test] + fn different_kinds_same_scope_yield_different_trees() { + let (_tmp, cfg) = test_config(); + let source = get_or_create_tree(&cfg, TreeKind::Source, "shared:scope").unwrap(); + let topic = get_or_create_tree(&cfg, TreeKind::Topic, "shared:scope").unwrap(); + assert_ne!(source.id, topic.id); + assert_eq!(source.kind, TreeKind::Source); + assert_eq!(topic.kind, TreeKind::Topic); + } + + #[test] + fn global_tree_is_singleton() { + let (_tmp, cfg) = test_config(); + let first = get_or_create_tree(&cfg, TreeKind::Global, "global").unwrap(); + let second = get_or_create_tree(&cfg, TreeKind::Global, "global").unwrap(); + assert_eq!(first.id, second.id); + assert_eq!(first.kind, TreeKind::Global); + } + #[test] fn tree_id_has_expected_prefix() { - let id = new_tree_id(TreeKind::Source); - assert!(id.starts_with("source:")); + let source_id = new_tree_id(TreeKind::Source); + assert!(source_id.starts_with("source:")); + let topic_id = new_tree_id(TreeKind::Topic); + assert!(topic_id.starts_with("topic:")); + let global_id = new_tree_id(TreeKind::Global); + assert!(global_id.starts_with("global:")); + let sum_id = new_summary_id(3); assert!(sum_id.starts_with("summary:")); - // Time-first layout: the segment after `summary:` is a 13-digit - // zero-padded ms timestamp, then `:L-<8hex>`. assert!(sum_id.contains(":L3-"), "expected level suffix in {sum_id}"); } #[test] fn summary_id_format_is_lexicographically_chronological() { - // The prefix `summary:` is identical across all ids, so the - // first character that differs is in the 13-digit ms field. - // Comparing two synthesised ids built around the same ms +/- a - // step proves the format sorts by time without depending on - // wall-clock granularity in the test runner. We verify the - // generator's _format_ (the contract), not the system clock. let earlier_ms: u64 = 1_700_000_000_000; let later_ms: u64 = 1_700_000_000_001; - // Use a max-tail rand for the earlier id to prove the - // millisecond field dominates over the random suffix. let earlier = format!("summary:{:013}:L1-{:08x}", earlier_ms, u32::MAX); let later = format!("summary:{:013}:L9-{:08x}", later_ms, 0u32); assert!( @@ -173,9 +183,6 @@ mod tests { "expected {earlier} < {later} (ms must outrank level + tail)" ); - // Sanity: a real id from the live generator parses with the - // same prefix shape so the contract above maps onto runtime - // values, not just synthesised strings. let live = new_summary_id(2); assert!(live.starts_with("summary:"), "live: {live}"); let rest = &live["summary:".len()..]; @@ -189,9 +196,6 @@ mod tests { #[test] fn get_or_create_recovers_from_unique_race() { - // Simulate the race by pre-inserting a tree under the same scope - // with a different id. `get_or_create` must re-query and return - // the pre-existing row, not bubble the UNIQUE error. let (_tmp, cfg) = test_config(); let pre_existing = Tree { id: "source:preexisting".into(), @@ -205,18 +209,9 @@ mod tests { }; store::insert_tree(&cfg, &pre_existing).unwrap(); - // First call finds it via get_tree_by_scope (happy path — no race - // triggered here). To hit the race branch we need a caller that - // skips the lookup and goes straight to insert with a fresh id. - // Simplest proxy: call get_or_create twice from this test thread; - // the first creates, the second's UNIQUE would fire if the - // lookup was ever elided. Instead we cover the race path directly - // via `is_unique_violation` on a synthesised insert failure below. - let got = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); + let got = get_or_create_tree(&cfg, TreeKind::Source, "slack:#eng").unwrap(); assert_eq!(got.id, "source:preexisting"); - // Direct coverage: a second insert with a different id for the - // same scope must surface as UNIQUE and be detected. let dup = Tree { id: "source:would-collide".into(), ..pre_existing.clone() diff --git a/src/openhuman/memory_tree/tree_global/registry.rs b/src/openhuman/memory_tree/tree_global/registry.rs deleted file mode 100644 index ad91013e3..000000000 --- a/src/openhuman/memory_tree/tree_global/registry.rs +++ /dev/null @@ -1,138 +0,0 @@ -//! Singleton registry for the global activity digest tree (#709, Phase 3b). -//! -//! Unlike source trees (one per `source_id`) the global tree is a true -//! singleton per workspace — scope is the literal string `"global"`. The -//! lookup and race-recovery pattern otherwise mirrors -//! `tree_source::registry::get_or_create_source_tree`. - -use anyhow::Result; -use chrono::Utc; -use uuid::Uuid; - -use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::tree_global::GLOBAL_SCOPE; -use crate::openhuman::memory_tree::tree_source::store; -use crate::openhuman::memory_tree::tree_source::types::{Tree, TreeKind, TreeStatus}; - -/// Return the workspace's singleton global tree, creating it lazily on -/// first call. Safe to call on every ingest; subsequent calls short-circuit -/// to the existing row. -pub fn get_or_create_global_tree(config: &Config) -> Result { - if let Some(existing) = store::get_tree_by_scope(config, TreeKind::Global, GLOBAL_SCOPE)? { - log::debug!( - "[tree_global::registry] found global tree id={}", - existing.id - ); - return Ok(existing); - } - - let tree = Tree { - id: new_global_tree_id(), - kind: TreeKind::Global, - scope: GLOBAL_SCOPE.to_string(), - root_id: None, - max_level: 0, - status: TreeStatus::Active, - created_at: Utc::now(), - last_sealed_at: None, - }; - match store::insert_tree(config, &tree) { - Ok(()) => { - log::info!("[tree_global::registry] created global tree id={}", tree.id); - Ok(tree) - } - Err(err) if is_unique_violation(&err) => { - // Another caller beat us to it between our initial lookup and - // the insert. The UNIQUE(kind, scope) index caught it — - // re-query and return the winner. - log::debug!("[tree_global::registry] UNIQUE race for global tree — re-querying"); - store::get_tree_by_scope(config, TreeKind::Global, GLOBAL_SCOPE)?.ok_or_else(|| { - anyhow::anyhow!( - "UNIQUE violation on global-tree insert but no row found on re-query" - ) - }) - } - Err(err) => Err(err), - } -} - -/// True when `err` wraps a SQLite UNIQUE constraint violation. Duplicated -/// from `tree_source::registry` to keep this module self-contained; the -/// two copies are ~5 lines and have the same shape. -fn is_unique_violation(err: &anyhow::Error) -> bool { - if let Some(rusqlite::Error::SqliteFailure(sqlite_err, _)) = - err.downcast_ref::() - { - return sqlite_err.code == rusqlite::ErrorCode::ConstraintViolation; - } - let msg = format!("{err:#}"); - msg.contains("UNIQUE constraint failed") -} - -fn new_global_tree_id() -> String { - format!("{}:{}", TreeKind::Global.as_str(), Uuid::new_v4()) -} - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::TempDir; - - fn test_config() -> (TempDir, Config) { - let tmp = TempDir::new().unwrap(); - let mut cfg = Config::default(); - cfg.workspace_dir = tmp.path().to_path_buf(); - (tmp, cfg) - } - - #[test] - fn get_or_create_is_idempotent() { - let (_tmp, cfg) = test_config(); - let first = get_or_create_global_tree(&cfg).unwrap(); - let second = get_or_create_global_tree(&cfg).unwrap(); - assert_eq!(first.id, second.id); - assert_eq!(first.kind, TreeKind::Global); - assert_eq!(first.scope, GLOBAL_SCOPE); - assert_eq!(first.status, TreeStatus::Active); - } - - #[test] - fn global_tree_has_expected_id_prefix() { - let id = new_global_tree_id(); - assert!(id.starts_with("global:")); - } - - #[test] - fn race_recovery_returns_existing_row() { - // Pre-seed a global tree so the second `get_or_create` path exercises - // the normal lookup branch; the UNIQUE-race branch is covered by the - // shared `is_unique_violation` contract in `tree_source::registry`. - let (_tmp, cfg) = test_config(); - let pre_existing = Tree { - id: "global:preexisting".into(), - kind: TreeKind::Global, - scope: GLOBAL_SCOPE.into(), - root_id: None, - max_level: 0, - status: TreeStatus::Active, - created_at: Utc::now(), - last_sealed_at: None, - }; - store::insert_tree(&cfg, &pre_existing).unwrap(); - - let got = get_or_create_global_tree(&cfg).unwrap(); - assert_eq!(got.id, "global:preexisting"); - - // And a direct duplicate insert must fire UNIQUE, covering the - // detector path this module depends on for race recovery. - let dup = Tree { - id: "global:would-collide".into(), - ..pre_existing.clone() - }; - let err = store::insert_tree(&cfg, &dup).unwrap_err(); - assert!( - is_unique_violation(&err), - "expected UNIQUE violation, got {err:#}" - ); - } -} diff --git a/src/openhuman/memory_tree/summarizer/bus.rs b/src/openhuman/memory_tree/tree_runtime/bus.rs similarity index 100% rename from src/openhuman/memory_tree/summarizer/bus.rs rename to src/openhuman/memory_tree/tree_runtime/bus.rs diff --git a/src/openhuman/memory_tree/summarizer/cli.rs b/src/openhuman/memory_tree/tree_runtime/cli.rs similarity index 53% rename from src/openhuman/memory_tree/summarizer/cli.rs rename to src/openhuman/memory_tree/tree_runtime/cli.rs index 228999c39..d58bb7457 100644 --- a/src/openhuman/memory_tree/summarizer/cli.rs +++ b/src/openhuman/memory_tree/tree_runtime/cli.rs @@ -108,7 +108,9 @@ fn run_ingest(args: &[String]) -> Result<()> { let (opts, rest) = parse_opts(args)?; if rest.iter().any(|a| is_help(a)) || rest.is_empty() { - println!("Usage: openhuman tree-summarizer ingest [--content ] [--file ] [-v]"); + println!( + "Usage: openhuman tree-summarizer ingest [--content ] [--file ] [-v]" + ); println!(); println!("Append content to the summarization buffer for a namespace."); println!(); @@ -152,7 +154,7 @@ fn run_ingest(args: &[String]) -> Result<()> { let rt = build_runtime()?; rt.block_on(async { let config = load_config().await?; - let outcome = crate::openhuman::memory_tree::summarizer::rpc::tree_summarizer_ingest( + let outcome = crate::openhuman::memory_tree::tree_runtime::rpc::tree_summarizer_ingest( &config, namespace, &content, None, None, ) .await @@ -188,10 +190,11 @@ fn run_summarize(args: &[String]) -> Result<()> { let rt = build_runtime()?; rt.block_on(async { let config = load_config().await?; - let outcome = - crate::openhuman::memory_tree::summarizer::rpc::tree_summarizer_run(&config, namespace) - .await - .map_err(anyhow::Error::msg)?; + let outcome = crate::openhuman::memory_tree::tree_runtime::rpc::tree_summarizer_run( + &config, namespace, + ) + .await + .map_err(anyhow::Error::msg)?; println!( "{}", @@ -238,7 +241,7 @@ fn run_query(args: &[String]) -> Result<()> { let rt = build_runtime()?; rt.block_on(async { let config = load_config().await?; - let outcome = crate::openhuman::memory_tree::summarizer::rpc::tree_summarizer_query( + let outcome = crate::openhuman::memory_tree::tree_runtime::rpc::tree_summarizer_query( &config, namespace, node_id, ) .await @@ -273,7 +276,7 @@ fn run_status(args: &[String]) -> Result<()> { let rt = build_runtime()?; rt.block_on(async { let config = load_config().await?; - let outcome = crate::openhuman::memory_tree::summarizer::rpc::tree_summarizer_status( + let outcome = crate::openhuman::memory_tree::tree_runtime::rpc::tree_summarizer_status( &config, namespace, ) .await @@ -311,7 +314,7 @@ fn run_rebuild(args: &[String]) -> Result<()> { let rt = build_runtime()?; rt.block_on(async { let config = load_config().await?; - let outcome = crate::openhuman::memory_tree::summarizer::rpc::tree_summarizer_rebuild( + let outcome = crate::openhuman::memory_tree::tree_runtime::rpc::tree_summarizer_rebuild( &config, namespace, ) .await @@ -386,3 +389,306 @@ fn print_help() { println!(" openhuman tree-summarizer query my-ns 2024/03/15"); println!(" openhuman tree-summarizer status my-ns"); } + +#[cfg(test)] +mod tests { + use std::ffi::OsString; + use std::path::PathBuf; + + use tempfile::TempDir; + + use crate::openhuman::config::TEST_ENV_LOCK; + + use super::*; + + fn lock_env() -> std::sync::MutexGuard<'static, ()> { + TEST_ENV_LOCK.lock().unwrap_or_else(|err| err.into_inner()) + } + + struct WorkspaceEnvGuard { + _lock: std::sync::MutexGuard<'static, ()>, + previous: Option, + } + + impl WorkspaceEnvGuard { + fn set(path: &std::path::Path) -> Self { + let lock = lock_env(); + let previous = std::env::var_os("OPENHUMAN_WORKSPACE"); + std::env::set_var("OPENHUMAN_WORKSPACE", path); + Self { + _lock: lock, + previous, + } + } + } + + impl Drop for WorkspaceEnvGuard { + fn drop(&mut self) { + if let Some(previous) = self.previous.as_ref() { + std::env::set_var("OPENHUMAN_WORKSPACE", previous); + } else { + std::env::remove_var("OPENHUMAN_WORKSPACE"); + } + } + } + + struct EnvVarGuard { + key: &'static str, + previous: Option, + } + + impl EnvVarGuard { + fn set(key: &'static str, value: impl AsRef) -> Self { + let previous = std::env::var_os(key); + std::env::set_var(key, value); + Self { key, previous } + } + + fn remove(key: &'static str) -> Self { + let previous = std::env::var_os(key); + std::env::remove_var(key); + Self { key, previous } + } + } + + impl Drop for EnvVarGuard { + fn drop(&mut self) { + if let Some(previous) = self.previous.as_ref() { + std::env::set_var(self.key, previous); + } else { + std::env::remove_var(self.key); + } + } + } + + #[test] + fn is_help_matches_supported_aliases() { + assert!(is_help("-h")); + assert!(is_help("--help")); + assert!(is_help("help")); + assert!(!is_help("run")); + } + + #[test] + fn parse_opts_collects_known_flags_and_rest_args() { + let args = vec![ + "--content".to_string(), + "hello".to_string(), + "--file".to_string(), + "notes.md".to_string(), + "--node-id".to_string(), + "2024/03/15".to_string(), + "--verbose".to_string(), + "namespace".to_string(), + ]; + let (opts, rest) = parse_opts(&args).unwrap(); + assert!(opts.verbose); + assert_eq!(opts.content.as_deref(), Some("hello")); + assert_eq!(opts.file.as_deref(), Some("notes.md")); + assert_eq!(opts.node_id.as_deref(), Some("2024/03/15")); + assert_eq!(rest, vec!["namespace".to_string()]); + } + + #[test] + fn parse_opts_errors_when_flag_value_is_missing() { + let err = match parse_opts(&["--content".to_string()]) { + Ok(_) => panic!("missing --content value should fail"), + Err(err) => err, + }; + assert!(err.to_string().contains("missing value for --content")); + + let err = match parse_opts(&["--file".to_string()]) { + Ok(_) => panic!("missing --file value should fail"), + Err(err) => err, + }; + assert!(err.to_string().contains("missing value for --file")); + + let err = match parse_opts(&["--node-id".to_string()]) { + Ok(_) => panic!("missing --node-id value should fail"), + Err(err) => err, + }; + assert!(err.to_string().contains("missing value for --node-id")); + } + + #[test] + fn top_level_command_help_and_unknown_subcommand_behave() { + assert!(run_tree_summarizer_command(&[]).is_ok()); + assert!(run_tree_summarizer_command(&["--help".to_string()]).is_ok()); + + let err = run_tree_summarizer_command(&["bogus".to_string()]) + .expect_err("unknown subcommand should fail"); + assert!(err + .to_string() + .contains("unknown tree-summarizer subcommand")); + } + + #[test] + fn subcommand_argument_validation_errors_without_running_runtime() { + let err = run_ingest(&["ns".to_string()]) + .expect_err("ingest without content or file should fail"); + assert!(err + .to_string() + .contains("either --content or --file is required")); + + let err = run_ingest(&["ns".to_string(), "--content".to_string(), " ".to_string()]) + .expect_err("blank content should fail"); + assert!(err.to_string().contains("content is empty")); + } + + #[test] + fn help_paths_for_subcommands_return_ok() { + assert!(run_ingest(&["--help".to_string()]).is_ok()); + assert!(run_summarize(&["--help".to_string()]).is_ok()); + assert!(run_query(&["--help".to_string()]).is_ok()); + assert!(run_status(&["--help".to_string()]).is_ok()); + assert!(run_rebuild(&["--help".to_string()]).is_ok()); + } + + #[test] + fn ingest_status_and_query_run_against_isolated_workspace() { + let tmp = TempDir::new().unwrap(); + let _workspace = WorkspaceEnvGuard::set(tmp.path()); + + assert!(run_ingest(&[ + "ns".to_string(), + "--content".to_string(), + "hello world".to_string() + ]) + .is_ok()); + assert!(run_status(&["ns".to_string()]).is_ok()); + let err = run_query(&["ns".to_string(), "root".to_string()]) + .expect_err("root query should fail before a summarization run creates nodes"); + assert!(err.to_string().contains("not found")); + } + + #[test] + fn ingest_reads_from_file_path() { + let tmp = TempDir::new().unwrap(); + let _workspace = WorkspaceEnvGuard::set(tmp.path()); + let input = tmp.path().join("input.txt"); + std::fs::write(&input, "from file").unwrap(); + + let args = vec![ + "ns".to_string(), + "--file".to_string(), + input.display().to_string(), + ]; + assert!(run_ingest(&args).is_ok()); + } + + #[test] + fn ingest_prefers_file_input_and_surfaces_read_errors() { + let tmp = TempDir::new().unwrap(); + let _workspace = WorkspaceEnvGuard::set(tmp.path()); + let missing = tmp.path().join("missing.txt"); + + let args = vec![ + "ns".to_string(), + "--content".to_string(), + "fallback text".to_string(), + "--file".to_string(), + missing.display().to_string(), + ]; + let err = run_ingest(&args).expect_err("missing file should win over inline content"); + assert!(err.to_string().contains("failed to read")); + assert!(err.to_string().contains("missing.txt")); + } + + #[test] + fn run_summarize_surfaces_local_ai_requirement_before_empty_buffer_skip() { + let tmp = TempDir::new().unwrap(); + let _workspace = WorkspaceEnvGuard::set(tmp.path()); + + let err = run_summarize(&["fresh-ns".to_string()]) + .expect_err("run should still surface the local ai runtime requirement"); + assert!( + err.to_string() + .contains("tree summarizer requires local_ai to be enabled in config"), + "unexpected run_summarize error: {err:#}" + ); + } + + #[test] + fn query_prefers_explicit_node_flag_over_positional_node() { + let tmp = TempDir::new().unwrap(); + let _workspace = WorkspaceEnvGuard::set(tmp.path()); + + let err = run_query(&[ + "ns".to_string(), + "2024/03/15".to_string(), + "--node-id".to_string(), + "2024/03/16".to_string(), + ]) + .expect_err("missing node should fail"); + + assert!(err + .to_string() + .contains("node '2024/03/16' not found in namespace 'ns'")); + } + + #[test] + fn load_config_uses_isolated_workspace_and_env_overrides() { + let tmp = TempDir::new().unwrap(); + let _workspace = WorkspaceEnvGuard::set(tmp.path()); + let _model = EnvVarGuard::set("OPENHUMAN_MODEL", "custom-model"); + let _language = EnvVarGuard::set("OPENHUMAN_OUTPUT_LANGUAGE", "fr-CA"); + + let runtime = build_runtime().expect("runtime"); + let config = runtime.block_on(load_config()).expect("config"); + + let expected_config_path: PathBuf = tmp.path().join("config.toml"); + assert_eq!(config.config_path, expected_config_path); + assert_eq!(config.workspace_dir, tmp.path().join("workspace")); + assert_eq!(config.default_model.as_deref(), Some("custom-model")); + assert_eq!(config.output_language.as_deref(), Some("fr-CA")); + } + + #[test] + fn init_logging_sets_default_rust_log_only_when_needed() { + let _lock = lock_env(); + + { + let _rust_log = EnvVarGuard::remove("RUST_LOG"); + init_logging(false); + assert_eq!(std::env::var("RUST_LOG").ok().as_deref(), Some("warn")); + } + + { + let _rust_log = EnvVarGuard::remove("RUST_LOG"); + init_logging(true); + assert!(std::env::var_os("RUST_LOG").is_none()); + } + + { + let _rust_log = EnvVarGuard::set("RUST_LOG", "debug"); + init_logging(false); + assert_eq!(std::env::var("RUST_LOG").ok().as_deref(), Some("debug")); + } + } + + #[test] + fn run_and_rebuild_surface_local_ai_runtime_requirement() { + let tmp = TempDir::new().unwrap(); + let _workspace = WorkspaceEnvGuard::set(tmp.path()); + + // Seed a namespace so the commands go through the runtime path + // rather than failing argument validation. + assert!(run_ingest(&[ + "ns".to_string(), + "--content".to_string(), + "seed".to_string() + ]) + .is_ok()); + + let run_err = run_summarize(&["ns".to_string()]).expect_err("run should require local ai"); + assert!(run_err + .to_string() + .contains("requires local_ai to be enabled")); + + let rebuild_err = + run_rebuild(&["ns".to_string()]).expect_err("rebuild should require local ai"); + assert!(rebuild_err + .to_string() + .contains("requires local_ai to be enabled")); + } +} diff --git a/src/openhuman/memory_tree/summarizer/engine.rs b/src/openhuman/memory_tree/tree_runtime/engine.rs similarity index 99% rename from src/openhuman/memory_tree/summarizer/engine.rs rename to src/openhuman/memory_tree/tree_runtime/engine.rs index c20a5110d..3b3f1946a 100644 --- a/src/openhuman/memory_tree/summarizer/engine.rs +++ b/src/openhuman/memory_tree/tree_runtime/engine.rs @@ -8,8 +8,8 @@ use std::collections::BTreeMap; use crate::core::event_bus::{publish_global, DomainEvent}; use crate::openhuman::config::Config; use crate::openhuman::inference::provider::traits::Provider; -use crate::openhuman::memory_tree::summarizer::store; -use crate::openhuman::memory_tree::summarizer::types::{ +use crate::openhuman::memory_tree::tree_runtime::store; +use crate::openhuman::memory_tree::tree_runtime::types::{ derive_node_ids, derive_parent_id, estimate_tokens, level_from_node_id, NodeLevel, TreeNode, TreeStatus, }; @@ -512,7 +512,7 @@ fn collect_hour_leaves_recursive( if level == NodeLevel::Hour { let raw = std::fs::read_to_string(entry.path())?; let node = - crate::openhuman::memory_tree::summarizer::store::parse_node_markdown_pub( + crate::openhuman::memory_tree::tree_runtime::store::parse_node_markdown_pub( &raw, namespace, &node_id, ) .with_context(|| format!("failed to parse hour leaf '{node_id}'"))?; diff --git a/src/openhuman/memory_tree/summarizer/engine_tests.rs b/src/openhuman/memory_tree/tree_runtime/engine_tests.rs similarity index 75% rename from src/openhuman/memory_tree/summarizer/engine_tests.rs rename to src/openhuman/memory_tree/tree_runtime/engine_tests.rs index 407c1de6a..1b3bd05e8 100644 --- a/src/openhuman/memory_tree/summarizer/engine_tests.rs +++ b/src/openhuman/memory_tree/tree_runtime/engine_tests.rs @@ -446,3 +446,168 @@ async fn run_summarization_multi_hour_groups_produce_multiple_hour_leaves() { // Buffer must be empty. assert!(store::buffer_read(&cfg, ns).unwrap().is_empty()); } + +#[tokio::test] +async fn rebuild_tree_restores_buffer_and_rewrites_ancestors() { + let tmp = TempDir::new().unwrap(); + let cfg = test_config(&tmp); + std::fs::create_dir_all(&cfg.workspace_dir).unwrap(); + let ns = "rebuild-test"; + let ts = Utc.with_ymd_and_hms(2024, 3, 15, 10, 0, 0).unwrap(); + + let make_hour = |id: &str, text: &str| TreeNode { + node_id: id.to_string(), + namespace: ns.to_string(), + level: NodeLevel::Hour, + parent_id: derive_parent_id(id), + summary: text.to_string(), + token_count: estimate_tokens(text), + child_count: 0, + created_at: ts, + updated_at: ts, + metadata: None, + }; + + // Seed the tree with hour leaves and an outdated ancestor/root. + store::write_node(&cfg, &make_hour("2024/03/15/10", "hour ten")).unwrap(); + store::write_node(&cfg, &make_hour("2024/03/15/11", "hour eleven")).unwrap(); + store::write_node( + &cfg, + &TreeNode { + node_id: "2024/03/15".into(), + namespace: ns.into(), + level: NodeLevel::Day, + parent_id: Some("2024/03".into()), + summary: "stale day".into(), + token_count: 2, + child_count: 1, + created_at: ts, + updated_at: ts, + metadata: None, + }, + ) + .unwrap(); + store::write_node( + &cfg, + &TreeNode { + node_id: "root".into(), + namespace: ns.into(), + level: NodeLevel::Root, + parent_id: None, + summary: "stale root".into(), + token_count: 2, + child_count: 1, + created_at: ts, + updated_at: ts, + metadata: None, + }, + ) + .unwrap(); + + // Preserve unsummarized buffer content across rebuild. + store::buffer_write(&cfg, ns, "pending buffer item", &ts, None).unwrap(); + let provider = StubProvider::with_reply("rebuilt summary"); + + let status = rebuild_tree(&cfg, &provider, ns).await.unwrap(); + assert!(status.total_nodes >= 5, "expected leaf + ancestor chain"); + + let restored_buffer = store::buffer_read(&cfg, ns).unwrap(); + assert_eq!( + restored_buffer.len(), + 1, + "buffer entries must survive rebuild" + ); + + let day = store::read_node(&cfg, ns, "2024/03/15").unwrap().unwrap(); + assert!( + day.summary.contains("hour ten") || day.summary.contains("rebuilt summary"), + "day node should be regenerated from hour leaves" + ); + + let root = store::read_node(&cfg, ns, "root").unwrap().unwrap(); + assert!( + root.summary.contains("rebuilt summary") || root.summary.contains("hour ten"), + "root node should be regenerated during rebuild" + ); +} + +#[tokio::test] +async fn rebuild_tree_on_empty_namespace_is_noop() { + let tmp = TempDir::new().unwrap(); + let cfg = test_config(&tmp); + std::fs::create_dir_all(&cfg.workspace_dir).unwrap(); + + let provider = StubProvider::with_reply("unused"); + let status = rebuild_tree(&cfg, &provider, "empty-rebuild") + .await + .unwrap(); + assert_eq!(status.total_nodes, 0); + assert_eq!(status.depth, 0); +} + +#[tokio::test] +async fn summarize_to_limit_truncates_overlong_provider_output() { + let provider = StubProvider::with_reply("x".repeat(MAX_SUMMARY_CHARS + 50)); + let summary = summarize_to_limit(&provider, "short input", 10, "day", "2024/03/15", "m") + .await + .unwrap(); + + assert_eq!(summary.len(), 40, "max_tokens=10 should clamp to 40 chars"); + assert!(summary.chars().all(|c| c == 'x')); +} + +#[test] +fn hour_id_from_buffer_filename_parses_and_rejects_invalid_inputs() { + let parsed = hour_id_from_buffer_filename("1711958400000_uuid.md").unwrap(); + assert_eq!(parsed, "2024/04/01/08"); + + assert!(hour_id_from_buffer_filename("not-a-timestamp.md").is_none()); + assert!(hour_id_from_buffer_filename("abc_123.md").is_none()); +} + +#[test] +fn derive_node_ids_from_hour_id_falls_back_for_non_hour_ids() { + let ids = derive_node_ids_from_hour_id("2024/03/15"); + assert_eq!( + ids, + ( + "2024/03/15".to_string(), + "unknown".to_string(), + "unknown".to_string(), + "unknown".to_string(), + "root".to_string(), + ) + ); +} + +#[test] +fn discover_active_namespaces_requires_markdown_entries_in_buffer() { + let tmp = TempDir::new().unwrap(); + let cfg = test_config(&tmp); + std::fs::create_dir_all(&cfg.workspace_dir).unwrap(); + + let base = cfg.workspace_dir.join("memory").join("namespaces"); + std::fs::create_dir_all(base.join("alpha").join("tree").join("buffer")).unwrap(); + std::fs::create_dir_all(base.join("beta").join("tree").join("buffer")).unwrap(); + std::fs::create_dir_all(base.join("gamma").join("tree")).unwrap(); + + std::fs::write( + base.join("alpha") + .join("tree") + .join("buffer") + .join("entry.md"), + "alpha", + ) + .unwrap(); + std::fs::write( + base.join("beta") + .join("tree") + .join("buffer") + .join("entry.txt"), + "beta", + ) + .unwrap(); + + let active = discover_active_namespaces(&cfg); + assert_eq!(active, vec!["alpha".to_string()]); +} diff --git a/src/openhuman/memory_tree/summarizer/mod.rs b/src/openhuman/memory_tree/tree_runtime/mod.rs similarity index 72% rename from src/openhuman/memory_tree/summarizer/mod.rs rename to src/openhuman/memory_tree/tree_runtime/mod.rs index 986b201df..081b17789 100644 --- a/src/openhuman/memory_tree/summarizer/mod.rs +++ b/src/openhuman/memory_tree/tree_runtime/mod.rs @@ -4,6 +4,11 @@ //! Each hour, a background job drains buffered raw content, summarizes it into //! the hour leaf, and propagates updated summaries upward through the tree. //! Stored as markdown files in `memory/namespaces/{ns}/tree/`. +//! +//! This module was renamed from `memory::summarizer` to +//! `memory_tree::tree_runtime` so it no longer collides conceptually with +//! [`crate::openhuman::memory_tree::summarise`], which is only the single-call +//! LLM fold primitive used during seals. pub mod bus; pub(crate) mod cli; diff --git a/src/openhuman/memory_tree/tree_runtime/ops.rs b/src/openhuman/memory_tree/tree_runtime/ops.rs new file mode 100644 index 000000000..204246018 --- /dev/null +++ b/src/openhuman/memory_tree/tree_runtime/ops.rs @@ -0,0 +1,420 @@ +//! RPC operation wrappers for the tree summarizer. + +use chrono::{DateTime, Utc}; +use serde_json::{json, Value}; + +use crate::openhuman::config::Config; +use crate::openhuman::memory_tree::tree_runtime::{engine, store, types::*}; +use crate::rpc::RpcOutcome; + +/// Append raw content to the ingestion buffer. +pub async fn tree_summarizer_ingest( + config: &Config, + namespace: &str, + content: &str, + timestamp: Option>, + metadata: Option<&Value>, +) -> Result, String> { + store::validate_namespace(namespace)?; + if content.trim().is_empty() { + return Err("content must not be empty".to_string()); + } + + let ts = timestamp.unwrap_or_else(Utc::now); + let path = store::buffer_write(config, namespace.trim(), content, &ts, metadata) + .map_err(|e| format!("buffer write failed: {e}"))?; + + Ok(RpcOutcome::single_log( + json!({ + "buffered": true, + "namespace": namespace.trim(), + "timestamp": ts.to_rfc3339(), + "tokens": estimate_tokens(content), + "path": path.display().to_string(), + "has_metadata": metadata.is_some(), + }), + format!("content buffered for namespace '{}'", namespace.trim()), + )) +} + +/// Trigger the summarization job for a namespace (drain buffer + summarize + propagate). +pub async fn tree_summarizer_run( + config: &Config, + namespace: &str, +) -> Result, String> { + store::validate_namespace(namespace)?; + + let provider = create_provider(config)?; + let ts = Utc::now(); + + match engine::run_summarization(config, provider.as_ref(), namespace.trim(), ts).await { + Ok(Some(node)) => Ok(RpcOutcome::single_log( + serde_json::to_value(&node).map_err(|e| e.to_string())?, + format!( + "summarization completed for '{}': node {} ({} tokens)", + namespace.trim(), + node.node_id, + node.token_count + ), + )), + Ok(None) => Ok(RpcOutcome::single_log( + json!({ "skipped": true, "reason": "no buffered data" }), + format!( + "summarization skipped for '{}': no buffered data", + namespace.trim() + ), + )), + Err(e) => Err(format!("summarization failed: {e:#}")), + } +} + +/// Query the tree at a specific node or level. +pub async fn tree_summarizer_query( + config: &Config, + namespace: &str, + node_id: Option<&str>, +) -> Result, String> { + store::validate_namespace(namespace)?; + + let target_id = node_id.unwrap_or("root"); + store::validate_node_id(target_id)?; + + let node = store::read_node(config, namespace.trim(), target_id) + .map_err(|e| format!("read node: {e}"))? + .ok_or_else(|| { + format!( + "node '{}' not found in namespace '{}'", + target_id, + namespace.trim() + ) + })?; + + let children = store::read_children(config, namespace.trim(), target_id) + .map_err(|e| format!("read children: {e}"))?; + + let result = QueryResult { node, children }; + Ok(RpcOutcome::single_log( + serde_json::to_value(&result).map_err(|e| e.to_string())?, + format!( + "queried node '{}' in namespace '{}'", + target_id, + namespace.trim() + ), + )) +} + +/// Get tree status/metadata for a namespace. +pub async fn tree_summarizer_status( + config: &Config, + namespace: &str, +) -> Result, String> { + store::validate_namespace(namespace)?; + + let status = + store::get_tree_status(config, namespace.trim()).map_err(|e| format!("get status: {e}"))?; + + Ok(RpcOutcome::single_log( + serde_json::to_value(&status).map_err(|e| e.to_string())?, + format!("tree status for namespace '{}'", namespace.trim()), + )) +} + +/// Rebuild the entire tree from hour leaves (background task). +pub async fn tree_summarizer_rebuild( + config: &Config, + namespace: &str, +) -> Result, String> { + store::validate_namespace(namespace)?; + + let provider = create_provider(config)?; + + let status = engine::rebuild_tree(config, provider.as_ref(), namespace.trim()) + .await + .map_err(|e| format!("rebuild failed: {e:#}"))?; + + Ok(RpcOutcome::single_log( + serde_json::to_value(&status).map_err(|e| e.to_string())?, + format!( + "tree rebuilt for '{}': {} nodes", + namespace.trim(), + status.total_nodes + ), + )) +} + +// ── Helper ───────────────────────────────────────────────────────────── + +fn create_provider( + config: &Config, +) -> Result, String> { + // Tree summarization runs exclusively on local AI to keep memory + // processing private and offline — no backend calls. + if !config.local_ai.runtime_enabled { + return Err("tree summarizer requires local_ai to be enabled in config".to_string()); + } + create_local_ai_provider(config) +} + +/// Create a provider backed by the local Ollama instance for summarization, +/// wrapped in `ReliableProvider` for retry/backoff on transient failures. +fn create_local_ai_provider( + config: &Config, +) -> Result, String> { + use crate::openhuman::inference::local::OLLAMA_BASE_URL; + use crate::openhuman::inference::provider::compatible::{AuthStyle, OpenAiCompatibleProvider}; + use crate::openhuman::inference::provider::reliable::ReliableProvider; + + let base_url = format!("{}/v1", OLLAMA_BASE_URL); + let inner = OpenAiCompatibleProvider::new_no_responses_fallback( + "ollama-local", + &base_url, + None, + AuthStyle::None, + ); + + let providers: Vec<( + String, + Box, + )> = vec![("ollama-local".to_string(), Box::new(inner))]; + let reliable = ReliableProvider::new( + providers, + config.reliability.provider_retries, + config.reliability.provider_backoff_ms, + ); + + tracing::debug!( + "[tree_summarizer] using local Ollama provider at {} with model '{}'", + base_url, + config.local_ai.chat_model_id + ); + + Ok(Box::new(reliable)) +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::TimeZone; + use tempfile::TempDir; + + fn rfc3339_z(ts: DateTime) -> String { + ts.to_rfc3339_opts(chrono::SecondsFormat::Secs, true) + } + + fn config_in_tempdir() -> (TempDir, Config) { + let tmp = TempDir::new().expect("tempdir"); + let mut cfg = Config::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + (tmp, cfg) + } + + fn test_node( + namespace: &str, + node_id: &str, + summary: &str, + created_at: DateTime, + child_count: u32, + ) -> TreeNode { + TreeNode { + node_id: node_id.to_string(), + namespace: namespace.to_string(), + level: level_from_node_id(node_id), + parent_id: derive_parent_id(node_id), + summary: summary.to_string(), + token_count: estimate_tokens(summary), + child_count, + created_at, + updated_at: created_at, + metadata: None, + } + } + + #[test] + fn create_provider_requires_local_ai_runtime() { + let mut cfg = Config::default(); + cfg.local_ai.runtime_enabled = false; + let err = match create_provider(&cfg) { + Ok(_) => panic!("runtime-disabled config should fail"), + Err(err) => err, + }; + assert!(err.contains("requires local_ai to be enabled")); + } + + #[test] + fn create_local_ai_provider_uses_ollama_local_label() { + let mut cfg = Config::default(); + cfg.local_ai.runtime_enabled = true; + let provider = create_local_ai_provider(&cfg).expect("provider"); + let _ = provider; + } + + #[tokio::test] + async fn tree_summarizer_ingest_rejects_blank_content() { + let (_tmp, cfg) = config_in_tempdir(); + let err = tree_summarizer_ingest(&cfg, "team", " ", None, None) + .await + .expect_err("blank content should be rejected"); + assert!(err.contains("content must not be empty")); + } + + #[tokio::test] + async fn tree_summarizer_ingest_writes_buffer_and_reports_metadata() { + let (_tmp, cfg) = config_in_tempdir(); + let ts = chrono::Utc + .with_ymd_and_hms(2026, 5, 24, 12, 30, 0) + .unwrap(); + let meta = json!({"source": "unit-test"}); + let outcome = + tree_summarizer_ingest(&cfg, "Team / Notes", "hello world", Some(ts), Some(&meta)) + .await + .expect("ingest should succeed"); + + assert_eq!( + outcome.logs, + vec!["content buffered for namespace 'Team / Notes'".to_string()] + ); + assert_eq!(outcome.value["buffered"], true); + assert_eq!(outcome.value["namespace"], "Team / Notes"); + assert_eq!( + outcome.value["tokens"], + json!(estimate_tokens("hello world")) + ); + assert_eq!(outcome.value["has_metadata"], true); + + let path = outcome.value["path"] + .as_str() + .expect("path string in response"); + let written = std::fs::read_to_string(path).expect("buffer file should exist"); + assert!(written.contains("hello world")); + assert!(written.contains("\"source\":\"unit-test\"")); + } + + #[tokio::test] + async fn tree_summarizer_status_reports_empty_tree_defaults() { + let (_tmp, cfg) = config_in_tempdir(); + let outcome = tree_summarizer_status(&cfg, "fresh-ns") + .await + .expect("status on fresh namespace"); + assert_eq!( + outcome.logs, + vec!["tree status for namespace 'fresh-ns'".to_string()] + ); + assert_eq!(outcome.value["namespace"], "fresh-ns"); + assert_eq!(outcome.value["total_nodes"], 0); + assert_eq!(outcome.value["depth"], 0); + } + + #[tokio::test] + async fn tree_summarizer_query_errors_when_node_is_missing() { + let (_tmp, cfg) = config_in_tempdir(); + let err = tree_summarizer_query(&cfg, "fresh-ns", Some("root")) + .await + .expect_err("missing node should error"); + assert!(err.contains("node 'root' not found in namespace 'fresh-ns'")); + } + + #[tokio::test] + async fn tree_summarizer_query_returns_node_and_children() { + let (_tmp, cfg) = config_in_tempdir(); + let ts = chrono::Utc + .with_ymd_and_hms(2026, 5, 24, 12, 30, 0) + .unwrap(); + let root = test_node("team", "root", "root summary", ts, 1); + let year = test_node("team", "2026", "year summary", ts, 1); + store::write_node(&cfg, &root).expect("write root"); + store::write_node(&cfg, &year).expect("write year"); + + let outcome = tree_summarizer_query(&cfg, "team", None) + .await + .expect("query should succeed"); + + assert_eq!( + outcome.logs, + vec!["queried node 'root' in namespace 'team'"] + ); + assert_eq!(outcome.value["node"]["node_id"], "root"); + assert_eq!(outcome.value["node"]["summary"], "root summary"); + assert_eq!( + outcome.value["children"], + json!([{ + "node_id": "2026", + "namespace": "team", + "level": "year", + "parent_id": "root", + "summary": "year summary", + "token_count": estimate_tokens("year summary"), + "child_count": 1, + "created_at": rfc3339_z(ts), + "updated_at": rfc3339_z(ts) + }]) + ); + } + + #[tokio::test] + async fn tree_summarizer_status_reports_populated_tree_details() { + let (_tmp, cfg) = config_in_tempdir(); + let early = chrono::Utc.with_ymd_and_hms(2026, 5, 24, 8, 0, 0).unwrap(); + let late = chrono::Utc.with_ymd_and_hms(2026, 5, 24, 17, 0, 0).unwrap(); + for node in [ + test_node("team", "root", "root summary", early, 1), + test_node("team", "2026", "year summary", early, 1), + test_node("team", "2026/05", "month summary", early, 1), + test_node("team", "2026/05/24", "day summary", early, 2), + test_node("team", "2026/05/24/08", "hour one", early, 0), + test_node("team", "2026/05/24/17", "hour two", late, 0), + ] { + store::write_node(&cfg, &node).expect("write test node"); + } + + let outcome = tree_summarizer_status(&cfg, "team") + .await + .expect("status should succeed"); + + assert_eq!(outcome.logs, vec!["tree status for namespace 'team'"]); + assert_eq!(outcome.value["namespace"], "team"); + assert_eq!(outcome.value["total_nodes"], 6); + assert_eq!(outcome.value["depth"], 5); + assert_eq!(outcome.value["oldest_entry"], rfc3339_z(early)); + assert_eq!(outcome.value["newest_entry"], rfc3339_z(late)); + assert_eq!(outcome.value["last_run_at"], Value::Null); + } + + #[tokio::test] + async fn tree_summarizer_run_skips_when_buffer_is_empty() { + let (_tmp, mut cfg) = config_in_tempdir(); + cfg.local_ai.runtime_enabled = true; + + let outcome = tree_summarizer_run(&cfg, "team") + .await + .expect("empty buffer should skip"); + + assert_eq!( + outcome.logs, + vec!["summarization skipped for 'team': no buffered data"] + ); + assert_eq!( + outcome.value, + json!({ "skipped": true, "reason": "no buffered data" }) + ); + assert!( + !store::buffer_dir(&cfg, "team").exists(), + "skip path should not create a buffer directory" + ); + } + + #[tokio::test] + async fn tree_summarizer_run_and_rebuild_require_local_ai() { + let (_tmp, mut cfg) = config_in_tempdir(); + cfg.local_ai.runtime_enabled = false; + + let run_err = tree_summarizer_run(&cfg, "team") + .await + .expect_err("run should require local ai"); + assert!(run_err.contains("requires local_ai to be enabled")); + + let rebuild_err = tree_summarizer_rebuild(&cfg, "team") + .await + .expect_err("rebuild should require local ai"); + assert!(rebuild_err.contains("requires local_ai to be enabled")); + } +} diff --git a/src/openhuman/memory_tree/summarizer/schemas.rs b/src/openhuman/memory_tree/tree_runtime/schemas.rs similarity index 97% rename from src/openhuman/memory_tree/summarizer/schemas.rs rename to src/openhuman/memory_tree/tree_runtime/schemas.rs index 8c22b72a6..2726f2cdf 100644 --- a/src/openhuman/memory_tree/summarizer/schemas.rs +++ b/src/openhuman/memory_tree/tree_runtime/schemas.rs @@ -176,7 +176,7 @@ fn handle_ingest(params: Map) -> ControllerFuture { let timestamp = read_optional_timestamp(¶ms, "timestamp")?; let metadata = read_optional::(¶ms, "metadata")?; to_json( - crate::openhuman::memory_tree::summarizer::rpc::tree_summarizer_ingest( + crate::openhuman::memory_tree::tree_runtime::rpc::tree_summarizer_ingest( &config, &namespace, &content, @@ -193,7 +193,7 @@ fn handle_run(params: Map) -> ControllerFuture { let config = config_rpc::load_config_with_timeout().await?; let namespace = read_required::(¶ms, "namespace")?; to_json( - crate::openhuman::memory_tree::summarizer::rpc::tree_summarizer_run( + crate::openhuman::memory_tree::tree_runtime::rpc::tree_summarizer_run( &config, &namespace, ) .await?, @@ -207,7 +207,7 @@ fn handle_query(params: Map) -> ControllerFuture { let namespace = read_required::(¶ms, "namespace")?; let node_id = read_optional::(¶ms, "node_id")?; to_json( - crate::openhuman::memory_tree::summarizer::rpc::tree_summarizer_query( + crate::openhuman::memory_tree::tree_runtime::rpc::tree_summarizer_query( &config, &namespace, node_id.as_deref(), @@ -222,7 +222,7 @@ fn handle_status(params: Map) -> ControllerFuture { let config = config_rpc::load_config_with_timeout().await?; let namespace = read_required::(¶ms, "namespace")?; to_json( - crate::openhuman::memory_tree::summarizer::rpc::tree_summarizer_status( + crate::openhuman::memory_tree::tree_runtime::rpc::tree_summarizer_status( &config, &namespace, ) .await?, @@ -235,7 +235,7 @@ fn handle_rebuild(params: Map) -> ControllerFuture { let config = config_rpc::load_config_with_timeout().await?; let namespace = read_required::(¶ms, "namespace")?; to_json( - crate::openhuman::memory_tree::summarizer::rpc::tree_summarizer_rebuild( + crate::openhuman::memory_tree::tree_runtime::rpc::tree_summarizer_rebuild( &config, &namespace, ) .await?, diff --git a/src/openhuman/memory_tree/summarizer/store.rs b/src/openhuman/memory_tree/tree_runtime/store.rs similarity index 99% rename from src/openhuman/memory_tree/summarizer/store.rs rename to src/openhuman/memory_tree/tree_runtime/store.rs index 90a0c384d..9c536f64a 100644 --- a/src/openhuman/memory_tree/summarizer/store.rs +++ b/src/openhuman/memory_tree/tree_runtime/store.rs @@ -13,7 +13,7 @@ use serde_json::Value; use std::path::{Path, PathBuf}; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::summarizer::types::{ +use crate::openhuman::memory_tree::tree_runtime::types::{ derive_parent_id, estimate_tokens, level_from_node_id, node_id_to_path, NodeLevel, TreeNode, TreeStatus, }; diff --git a/src/openhuman/memory_tree/summarizer/store_tests.rs b/src/openhuman/memory_tree/tree_runtime/store_tests.rs similarity index 100% rename from src/openhuman/memory_tree/summarizer/store_tests.rs rename to src/openhuman/memory_tree/tree_runtime/store_tests.rs diff --git a/src/openhuman/memory_tree/summarizer/types.rs b/src/openhuman/memory_tree/tree_runtime/types.rs similarity index 100% rename from src/openhuman/memory_tree/summarizer/types.rs rename to src/openhuman/memory_tree/tree_runtime/types.rs diff --git a/src/openhuman/memory_tree/tree_source/README.md b/src/openhuman/memory_tree/tree_source/README.md deleted file mode 100644 index 161917935..000000000 --- a/src/openhuman/memory_tree/tree_source/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# Tree source - -Phase 3a (#709) — per-source summary trees with bucket-seal mechanics. One tree per ingest source (Slack channel, Gmail account, document corpus, ...). Time-aligned L0 buffers accumulate canonical chunks; once a buffer crosses its gate it seals into an L1 summary, and the cascade may continue upward (L1 → L2 → ...). Storage primitives are reused by the topic and global trees in Phases 3b / 3c. - -## Public surface - -- `pub fn get_or_create_source_tree` — `registry.rs` — idempotent tree lookup keyed by `(kind=source, scope)`. -- `pub fn append_leaf` / `pub fn append_leaf_deferred` / `pub struct LeafRef` / `pub enum LabelStrategy` — `bucket_seal.rs` — push a chunk into its tree and cascade-seal on budget. -- `pub fn flush_stale_buffers` / `pub fn flush_stale_buffers_default` / `pub fn force_flush_tree` — `flush.rs` — time-based seal of buffers that never cross the token gate. -- `pub fn build_summariser` / `pub trait Summariser` / `pub struct SummaryInput` / `pub struct SummaryContext` / `pub struct SummaryOutput` — `summariser/mod.rs` — folds N inputs into one summary. -- `pub struct InertSummariser` — `summariser/inert.rs` — deterministic dependency-free fallback. -- `pub struct LlmSummariser` / `pub struct LlmSummariserConfig` — `summariser/llm.rs` — Ollama-backed implementation with soft-fallback to inert. -- `pub struct Tree` / `pub struct SummaryNode` / `pub struct Buffer` / `pub enum TreeKind` / `pub enum TreeStatus` / `pub const INPUT_TOKEN_BUDGET` / `pub const OUTPUT_TOKEN_BUDGET` / `pub const SUMMARY_FANOUT` — `types.rs`. -- `pub fn get_summary_embedding` / `pub fn set_summary_embedding` / `pub fn insert_tree` / `pub fn get_tree_by_scope` / `pub fn get_tree` / `pub fn list_trees_by_kind` / `pub fn get_summary` / `pub fn list_summaries_at_level` / `pub fn count_summaries` / `pub fn get_buffer` / `pub fn list_stale_buffers` — `store.rs`. - -## Files - -- `mod.rs` — module surface and re-exports. -- `types.rs` — `Tree`, `SummaryNode`, `Buffer`, gating constants. -- `registry.rs` — get-or-create + UNIQUE-race recovery; `new_summary_id` helper. -- `store.rs` — SQLite persistence for `mem_tree_trees` / `mem_tree_summaries` / `mem_tree_buffers`, including embedding blob handling. -- `bucket_seal.rs` — `append_leaf`, level-aware seal gate, single-tx `seal_one_level` with atomic follow-up enqueue. -- `flush.rs` — time-based stale-buffer flush. -- `summariser/` — summariser trait and implementations (see `summariser/README.md`). -- `bucket_seal_tests.rs` / `store_tests.rs` — per-module unit tests, included via `#[path]`. diff --git a/src/openhuman/memory_tree/tree_source/mod.rs b/src/openhuman/memory_tree/tree_source/mod.rs deleted file mode 100644 index b13d254c9..000000000 --- a/src/openhuman/memory_tree/tree_source/mod.rs +++ /dev/null @@ -1,33 +0,0 @@ -//! Phase 3a — summary trees + bucket-seal mechanics (#709). -//! -//! A thin orchestration layer on top of Phase 1 chunks and Phase 2 scores -//! that lifts individual leaves into a hierarchy of sealed summary nodes, -//! one tree per ingest source. See `docs/MEMORY_ARCHITECTURE_LLD.md` for -//! the full design. The module is isolated from the legacy -//! `openhuman::memory` layer and only depends on sibling `tree::*` modules. -//! -//! Public surface at Phase 3a: -//! - [`registry::get_or_create_source_tree`] — idempotent tree lookup -//! - [`bucket_seal::append_leaf`] — push a chunk into its tree, cascade-seal on budget -//! - [`flush::flush_stale_buffers`] — time-based seal of buffers that never cross budget -//! - [`summariser::inert::InertSummariser`] — deterministic fallback summariser -//! -//! Phases 3b / 3c will add `global` and `topic` trees; both reuse the -//! storage and cascade primitives defined here. - -pub mod bucket_seal; -pub mod flush; -pub mod registry; -pub mod source_file; -pub mod store; -pub mod summariser; -pub mod types; - -pub use bucket_seal::{append_leaf, append_leaf_deferred, LabelStrategy, LeafRef}; -pub use registry::get_or_create_source_tree; -pub use store::{get_summary_embedding, set_summary_embedding}; -pub use summariser::{build_summariser, inert::InertSummariser, llm::LlmSummariser, Summariser}; -pub use types::{ - Buffer, SummaryNode, Tree, TreeKind, TreeStatus, INPUT_TOKEN_BUDGET, OUTPUT_TOKEN_BUDGET, - SUMMARY_FANOUT, -}; diff --git a/src/openhuman/memory_tree/tree_source/summariser/README.md b/src/openhuman/memory_tree/tree_source/summariser/README.md deleted file mode 100644 index e29cc3472..000000000 --- a/src/openhuman/memory_tree/tree_source/summariser/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Summariser - -Summariser trait and implementations used by the bucket-seal cascade. A summariser folds N buffered items into one sealed [`SummaryOutput`]; the seal machinery (bucket budgeting, persistence, label resolution) lives in [`super::bucket_seal`] and is unaffected by the choice of implementation. - -## Public surface - -- `pub trait Summariser` / `pub struct SummaryInput` / `pub struct SummaryContext` / `pub struct SummaryOutput` — `mod.rs` — async trait + IO types. -- `pub fn build_summariser` — `mod.rs` — picks the implementation based on `Config::memory_tree.llm_summariser_*`. Returns the LLM summariser when both endpoint and model are set, otherwise the inert fallback. -- `pub struct InertSummariser` — `inert.rs` — deterministic concat-and-truncate fallback. `entities` and `topics` are intentionally empty (an honest stub — derived labels are an LLM concern). -- `pub struct LlmSummariser` / `pub struct LlmSummariserConfig` — `llm.rs` — Ollama `/api/chat` peer of `score::extract::llm`. Soft-falls-back to inert on every error so seal cascades never abort. - -## Files - -- `mod.rs` — trait, IO types, and the `build_summariser` factory. -- `inert.rs` — deterministic fallback, used in tests and when no LLM is configured. -- `llm.rs` — Ollama-backed implementation with prompt construction, per-input clamping for `num_ctx` safety, and post-generation budget enforcement. diff --git a/src/openhuman/memory_tree/tree_source/summariser/inert.rs b/src/openhuman/memory_tree/tree_source/summariser/inert.rs deleted file mode 100644 index edf06a65b..000000000 --- a/src/openhuman/memory_tree/tree_source/summariser/inert.rs +++ /dev/null @@ -1,181 +0,0 @@ -//! Deterministic fallback summariser (#709). -//! -//! `InertSummariser` concatenates each input's content, separated by a -//! blank line, and hard-truncates to `ctx.token_budget`. Entities and -//! topics are **intentionally empty**: per design, summary-level entity / -//! topic metadata is derived by the LLM summariser from the summary's own -//! synthesised content (not by mechanically unioning children's labels). -//! Until the networked summariser lands, inert-sealed summaries have no -//! entity index rows — an honest stub. The goal of this fallback is not -//! metadata fidelity; it's a stable, dependency-free baseline so tree -//! mechanics (sealing, cascade, roots) can be tested without an LLM. - -use anyhow::Result; -use async_trait::async_trait; - -use crate::openhuman::memory_tree::tree_source::summariser::{ - Summariser, SummaryContext, SummaryInput, SummaryOutput, -}; -use crate::openhuman::memory_tree::types::approx_token_count; - -/// Default prefix applied to each contribution in the joined body. Keeps -/// provenance visible to a human reading the raw summary. -const PROVENANCE_PREFIX: &str = "— "; - -/// Deterministic, dependency-free [`Summariser`] implementation that -/// concatenates inputs and truncates to budget. See module docs for why -/// `entities` and `topics` are intentionally empty. -pub struct InertSummariser; - -impl InertSummariser { - /// Construct a fresh summariser. Stateless — multiple instances behave - /// identically. - pub fn new() -> Self { - Self - } -} - -impl Default for InertSummariser { - fn default() -> Self { - Self::new() - } -} - -#[async_trait] -impl Summariser for InertSummariser { - async fn summarise( - &self, - inputs: &[SummaryInput], - ctx: &SummaryContext<'_>, - ) -> Result { - let mut parts: Vec = Vec::with_capacity(inputs.len()); - for inp in inputs { - let trimmed = inp.content.trim(); - if trimmed.is_empty() { - continue; - } - parts.push(format!("{}{}", PROVENANCE_PREFIX, trimmed)); - } - let joined = parts.join("\n\n"); - - let (content, token_count) = truncate_to_budget(&joined, ctx.token_budget); - - log::debug!( - "[tree_source::summariser::inert] sealed tree_id={} level={} inputs={} tokens={} \ - entities=0 topics=0 (honest stub — LLM summariser derives these)", - ctx.tree_id, - ctx.target_level, - inputs.len(), - token_count - ); - - Ok(SummaryOutput { - content, - token_count, - entities: Vec::new(), - topics: Vec::new(), - }) - } -} - -/// Truncate `text` to fit within `budget` approximate tokens. Returns the -/// (possibly truncated) body and its recomputed token count. Truncation is -/// done on character boundaries — `approx_token_count` assumes ~4 chars -/// per token so we clamp character length to `budget * 4`. -fn truncate_to_budget(text: &str, budget: u32) -> (String, u32) { - let initial = approx_token_count(text); - if initial <= budget { - return (text.to_string(), initial); - } - // Character ceiling derived from the same ~4 chars/token heuristic. - let char_ceiling = (budget as usize).saturating_mul(4); - let truncated: String = text.chars().take(char_ceiling).collect(); - let tokens = approx_token_count(&truncated); - (truncated, tokens) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::openhuman::memory_tree::tree_source::types::TreeKind; - use chrono::Utc; - - fn sample_input(id: &str, content: &str, entities: &[&str]) -> SummaryInput { - let ts = Utc::now(); - SummaryInput { - id: id.to_string(), - content: content.to_string(), - token_count: approx_token_count(content), - entities: entities.iter().map(|s| s.to_string()).collect(), - topics: Vec::new(), - time_range_start: ts, - time_range_end: ts, - score: 0.5, - } - } - - fn test_ctx() -> SummaryContext<'static> { - SummaryContext { - tree_id: "tree-1", - tree_kind: TreeKind::Source, - target_level: 1, - token_budget: 10_000, - } - } - - #[tokio::test] - async fn concats_inputs_with_provenance_prefix() { - let s = InertSummariser::default(); - let inputs = vec![ - sample_input("a", "hello world", &[]), - sample_input("b", "second contribution", &[]), - ]; - let out = s.summarise(&inputs, &test_ctx()).await.unwrap(); - assert!(out.content.contains(PROVENANCE_PREFIX)); - assert!(out.content.contains("hello world")); - assert!(out.content.contains("second contribution")); - assert_eq!(out.token_count, approx_token_count(&out.content)); - } - - #[tokio::test] - async fn honest_stub_emits_no_entities_or_topics() { - // Per design: summary-level entities/topics are LLM-derived from - // the summary's own synthesised content. The inert fallback does - // not propagate children's labels — it emits empty vecs. The - // Ollama summariser (future) will fill them via real NER on its - // own output. - let s = InertSummariser::default(); - let inputs = vec![ - sample_input("a", "x", &["entity:alice", "entity:bob"]), - sample_input("b", "y", &["entity:bob", "entity:carol"]), - ]; - let out = s.summarise(&inputs, &test_ctx()).await.unwrap(); - assert!(out.entities.is_empty()); - assert!(out.topics.is_empty()); - } - - #[tokio::test] - async fn truncates_when_over_budget() { - let s = InertSummariser::default(); - let long_text = "a".repeat(100); - let inputs = vec![sample_input("a", &long_text, &[])]; - let mut ctx = test_ctx(); - ctx.token_budget = 5; // way under — should truncate hard - let out = s.summarise(&inputs, &ctx).await.unwrap(); - assert!(out.token_count <= ctx.token_budget + 1); - assert!(out.content.len() < long_text.len() + PROVENANCE_PREFIX.len()); - } - - #[tokio::test] - async fn skips_empty_contributions() { - let s = InertSummariser::default(); - let inputs = vec![ - sample_input("a", " ", &[]), - sample_input("b", "kept", &[]), - ]; - let out = s.summarise(&inputs, &test_ctx()).await.unwrap(); - assert!(out.content.contains("kept")); - // exactly one provenance prefix should appear - assert_eq!(out.content.matches(PROVENANCE_PREFIX).count(), 1); - } -} diff --git a/src/openhuman/memory_tree/tree_source/summariser/llm.rs b/src/openhuman/memory_tree/tree_source/summariser/llm.rs deleted file mode 100644 index ff2cbc766..000000000 --- a/src/openhuman/memory_tree/tree_source/summariser/llm.rs +++ /dev/null @@ -1,686 +0,0 @@ -//! LLM-backed summariser — peer of -//! [`crate::openhuman::memory_tree::score::extract::llm::LlmEntityExtractor`]. -//! -//! ## Responsibility -//! -//! When the source / topic / global tree's bucket-seal cascade decides to -//! fold N contributions (raw leaves at L0→L1, or lower-level summaries at -//! L_n→L_{n+1}), this summariser is asked to produce the parent node's -//! `content`. The seal machinery itself (bucket budgeting, level -//! promotion, `mem_tree_summaries` persistence) is unchanged — only the -//! text inside the summary row differs from [`super::inert::InertSummariser`]. -//! Entities and topics on `SummaryOutput` are always emitted empty by -//! this summariser; canonical entity ids are populated separately by the -//! entity extractor. -//! -//! ## Soft-fallback contract -//! -//! A summariser that returns `Err` would abort the seal cascade and leave -//! the tree in an inconsistent state — a half-sealed buffer with no -//! parent row. We therefore promise **never** to return `Err`: every -//! failure (transport, HTTP status, JSON shape) falls back to the same -//! deterministic concat-and-truncate behaviour as `InertSummariser` and -//! logs a warn. -//! -//! ## Prompt shape -//! -//! The system prompt commits the model to returning JSON with the shape -//! `{ summary }`. We pass `temperature: 0.0` for maximum determinism — -//! same knob the entity extractor already uses with success. -//! -//! ## Backend transparency -//! -//! Originally this summariser owned its own `reqwest::Client` and talked -//! directly to Ollama. After the cloud-default refactor, it accepts an -//! `Arc` instead — letting a single workspace pick -//! cloud (default) or local (opt-in) at runtime without changing this -//! file's prompt or parse logic. - -use anyhow::Result; -use async_trait::async_trait; -use std::sync::Arc; - -use super::inert::InertSummariser; -use super::{Summariser, SummaryContext, SummaryInput, SummaryOutput}; -use crate::openhuman::learning::extract::summary_facets::{self, StructuredSummary}; -use crate::openhuman::memory_tree::chat::{ChatPrompt, ChatProvider}; -use crate::openhuman::memory_tree::types::approx_token_count; - -/// Hard cap on summariser output length (in approximate tokens). -/// -/// Sized to fit the downstream embedder (`nomic-embed-text-v1.5`, -/// 8192-token input ceiling) with headroom for tokenizer drift between -/// our 4-chars/token heuristic and the embedder's real tokenizer. The -/// post-generation [`clamp_to_budget`] enforces this regardless of what -/// the model produces. -const MAX_SUMMARY_OUTPUT_TOKENS: u32 = 5_000; - -/// Context window assumed for the model. Sized for the cloud -/// summariser's 120k-token window with comfortable headroom — leaves -/// room for the joined L0 input batch (up to `INPUT_TOKEN_BUDGET = 50k`), -/// the requested output budget, the system prompt, and tokenizer drift. -/// Used as the divisor in the per-input clamp so the joined prompt body -/// stays under this even at upper-level seals where many children fold -/// together. -const NUM_CTX_TOKENS: u32 = 60_000; - -/// Tokens reserved for the system prompt, message-envelope overhead, -/// and tokenizer drift between our 4-chars/token heuristic and the -/// model's tokenizer. Trades a small loss of input capacity for a -/// guarantee that the prompt body + output budget never exceeds -/// `num_ctx`. -const OVERHEAD_RESERVE_TOKENS: u32 = 2_048; - -/// Configuration for [`LlmSummariser`]. Threaded down to the chat -/// provider for diagnostic logging — model selection at the wire level -/// happens inside the [`ChatProvider`]. -#[derive(Clone, Debug)] -pub struct LlmSummariserConfig { - /// Model identifier (e.g. `summarization-v1` for cloud, `qwen2.5:0.5b` - /// or `llama3.1:8b` for local Ollama). Diagnostic / log only. - pub model: String, - /// When `true` (the default), the summariser appends a structured facet - /// extraction request to the prompt and parses the resulting JSON block. - /// Discovered facets are routed to the learning candidate buffer. - /// Set to `false` to restore the plain-text-only behaviour for A/B testing - /// or debugging. - pub structured_facet_extraction: bool, - /// Optional configured output language for generated prose summaries. - pub output_language: Option, -} - -impl Default for LlmSummariserConfig { - fn default() -> Self { - Self { - model: "qwen2.5:0.5b".to_string(), - structured_facet_extraction: true, - output_language: None, - } - } -} - -/// LLM-backed summariser. Delegates to [`InertSummariser`] on any -/// failure so seal cascades never fail. -pub struct LlmSummariser { - cfg: LlmSummariserConfig, - provider: Arc, - fallback: InertSummariser, -} - -impl LlmSummariser { - /// Build a summariser with the supplied chat provider. Infallible — - /// the caller is responsible for provider construction. - pub fn new(cfg: LlmSummariserConfig, provider: Arc) -> Self { - Self { - cfg, - provider, - fallback: InertSummariser::new(), - } - } - - /// Build the chat prompt sent to the provider for a given seal. - /// - /// When `structured_facet_extraction` is enabled the system prompt includes - /// an instruction to emit a fenced `json` block after the prose summary. - fn build_prompt(&self, prompt_body: &str, budget: u32) -> ChatPrompt { - ChatPrompt { - system: system_prompt( - budget, - self.cfg.structured_facet_extraction, - self.cfg.output_language.as_deref(), - ), - user: prompt_body.to_string(), - temperature: 0.0, - kind: "memory_tree::summarise", - } - } -} - -#[async_trait] -impl Summariser for LlmSummariser { - async fn summarise( - &self, - inputs: &[SummaryInput], - ctx: &SummaryContext<'_>, - ) -> Result { - // Clamp the model-side output budget so the summary fits the - // downstream embedder. The seal-cascade hands us - // `ctx.token_budget = 10k` by default but `nomic-embed-text` - // only accepts ≤ 8k tokens of input. Producing a smaller - // summary upfront avoids the embed-fails-after-summary - // dead end. - let effective_budget = ctx.token_budget.min(MAX_SUMMARY_OUTPUT_TOKENS); - - // Per-input clamp scaled by fanout. Without this, an upper-level - // seal feeding `SUMMARY_FANOUT=4` children each near - // `MAX_SUMMARY_OUTPUT_TOKENS` would push the prompt body alone - // past `num_ctx` and Ollama would silently truncate (or error). - // Divide the input budget evenly across contributors. - let per_input_cap = if inputs.is_empty() { - 0 - } else { - NUM_CTX_TOKENS - .saturating_sub(effective_budget) - .saturating_sub(OVERHEAD_RESERVE_TOKENS) - / inputs.len() as u32 - }; - - // Assemble the user-side prompt. We prefix each contribution with - // its id so the model can weigh them and so log diffs are - // traceable to source rows if anything looks odd. - let body = build_user_prompt(inputs, per_input_cap); - if body.trim().is_empty() { - log::debug!( - "[tree_source::summariser::llm] empty prompt body (no non-blank inputs) \ - tree_id={} level={} — returning empty summary", - ctx.tree_id, - ctx.target_level - ); - return Ok(SummaryOutput { - content: String::new(), - token_count: 0, - entities: Vec::new(), - topics: Vec::new(), - }); - } - - let prompt = self.build_prompt(&body, effective_budget); - - log::debug!( - "[tree_source::summariser::llm] chat provider={} model={} tree_id={} level={} \ - inputs={} budget={}", - self.provider.name(), - self.cfg.model, - ctx.tree_id, - ctx.target_level, - inputs.len(), - ctx.token_budget - ); - - let raw = match self.provider.chat_for_text(&prompt).await { - Ok(v) => v, - Err(e) => { - log::warn!( - "[tree_source::summariser::llm] chat provider={} failed: {e:#} — \ - falling back to inert summariser for tree_id={} level={}", - self.provider.name(), - ctx.tree_id, - ctx.target_level - ); - return self.fallback.summarise(inputs, ctx).await; - } - }; - - // When structured_facet_extraction is enabled, attempt to split the response - // into a prose summary and an optional JSON block. On parse failure, the - // prose is used as-is and zero facets are emitted (fail-soft). - let summary_text: &str; - - if self.cfg.structured_facet_extraction { - let (prose, maybe_structured) = split_structured_response(raw.trim()); - summary_text = prose; - match maybe_structured { - Some(Ok(parsed)) => { - tracing::debug!( - "[learning::extract::summary] source_id={} facets_emitted={}", - ctx.tree_id, - parsed.facets.len() - ); - summary_facets::route_facets_to_buffer(&parsed, ctx.tree_id); - } - Some(Err(e)) => { - log::warn!( - "[tree_source::summariser::llm] structured facet parse failed \ - tree_id={} level={}: {e:#} — using raw prose, emitting 0 facets", - ctx.tree_id, - ctx.target_level - ); - } - None => { - // No JSON block present — normal for content with no clear signals. - tracing::debug!( - "[tree_source::summariser::llm] no structured JSON block in response \ - tree_id={} level={}", - ctx.tree_id, - ctx.target_level - ); - } - } - } else { - summary_text = raw.trim(); - } - - let (content, token_count) = clamp_to_budget(summary_text, effective_budget); - log::debug!( - "[tree_source::summariser::llm] sealed tree_id={} level={} inputs={} tokens={}", - ctx.tree_id, - ctx.target_level, - inputs.len(), - token_count - ); - - Ok(SummaryOutput { - content, - token_count, - entities: Vec::new(), - topics: Vec::new(), - }) - } -} - -/// Build the user-message body that precedes the model call. Each -/// contribution is prefixed with a short id header and separated by a -/// blank line — matches the layout the model is instructed to -/// summarise. Each input's content is clamped to -/// `per_input_cap_tokens` so the joined body fits inside `num_ctx` even -/// at upper-level seals where many large summaries fold together. A -/// `0` cap means "don't include any content" (used when there are no -/// inputs); pass `u32::MAX` to disable clamping. -fn build_user_prompt(inputs: &[SummaryInput], per_input_cap_tokens: u32) -> String { - let mut out = String::new(); - for inp in inputs { - let trimmed = inp.content.trim(); - if trimmed.is_empty() { - continue; - } - let (clamped, _) = clamp_to_budget(trimmed, per_input_cap_tokens); - if !out.is_empty() { - out.push_str("\n\n"); - } - out.push_str(&format!("[{}]\n{clamped}", inp.id)); - } - out -} - -/// System prompt. -/// -/// When `structured_facets` is `true`, appends instructions for the model to -/// emit a fenced `json` block after the prose summary containing any clearly -/// evidenced facets. -/// -/// Length isn't templated in — empirically, telling instruction-tuned models -/// "stay under N tokens" makes them produce curt, generic output even when the -/// input has plenty of substance. Output is clamped post-generation by -/// [`clamp_to_budget`] in the caller. -fn system_prompt(_budget: u32, structured_facets: bool, output_language: Option<&str>) -> String { - let base = "You are a precise summariser. Summarise the user-provided contributions into a \ - single cohesive passage that preserves concrete facts, decisions, \ - and temporal ordering. Do not invent facts.\n\ - \n\ - Return the summary text first."; - let language_directive = crate::openhuman::config::output_language_directive(output_language) - .map(|directive| format!("\n\n{directive}")) - .unwrap_or_default(); - - if !structured_facets { - return format!( - "{base}{language_directive} No commentary, no preamble, no headings, \ - no markdown wrappers, no JSON — just the prose summary." - ); - } - - format!( - "{base}{language_directive}\n\ - \n\ - After the summary, output a JSON object as the second part of your response, \ - fenced in a ```json block:\n\ - \n\ - ```json\n\ - {{\n\ - \"summary\": \"\",\n\ - \"facets\": [\n\ - {{\n\ - \"class\": \"style|identity|tooling|veto|goal\",\n\ - \"key\": \"\",\n\ - \"value\": \"\",\n\ - \"evidence_chunks\": [\"\", \"...\"],\n\ - \"confidence\": 0.0,\n\ - \"cue_family\": \"explicit|structural|behavioral\"\n\ - }}\n\ - ]\n\ - }}\n\ - ```\n\ - \n\ - Rules:\n\ - - Only include facets that are clearly evidenced in the content above.\n\ - - Each facet must cite at least one chunk_id from this batch (the id in brackets \ - before each contribution, e.g. [chunk-abc]).\n\ - - Use canonical keys: verbosity, format, name, timezone, role, package_manager, \ - lang, framework, runtime, etc.\n\ - - Cap the facets array at 8 items per call. Skip the array entirely (emit \ - facets: []) if no clear evidence.\n\ - - No commentary outside the prose summary and the JSON block." - ) -} - -/// Split a raw LLM response that may contain a trailing fenced `json` block. -/// -/// Returns `(prose, Option)` where: -/// - `prose` is the text before the ` ```json ` fence (trimmed), or the full -/// raw text when no fence is present. -/// - The second element is `None` when no fence was found, or -/// `Some(Ok(StructuredSummary))` / `Some(Err(…))` on parse success/failure. -fn split_structured_response(raw: &str) -> (&str, Option>) { - // Look for the opening ` ```json ` fence. - const OPEN_FENCE: &str = "```json"; - const CLOSE_FENCE: &str = "```"; - - let Some(fence_start) = raw.find(OPEN_FENCE) else { - return (raw, None); - }; - - let prose = raw[..fence_start].trim(); - let after_open = &raw[fence_start + OPEN_FENCE.len()..]; - - // Find the closing fence. - let json_str = if let Some(close_pos) = after_open.find(CLOSE_FENCE) { - after_open[..close_pos].trim() - } else { - // No closing fence — treat everything after the open as JSON. - after_open.trim() - }; - - let result = serde_json::from_str::(json_str) - .map_err(|e| anyhow::anyhow!("structured summary JSON parse error: {e}")); - - (prose, Some(result)) -} - -/// Truncate to the caller's token budget using the same ~4 chars/token -/// heuristic as [`InertSummariser`]. -fn clamp_to_budget(text: &str, budget: u32) -> (String, u32) { - let initial = approx_token_count(text); - if initial <= budget { - return (text.to_string(), initial); - } - let char_ceiling = (budget as usize).saturating_mul(4); - let truncated: String = text.chars().take(char_ceiling).collect(); - let tokens = approx_token_count(&truncated); - (truncated, tokens) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::openhuman::memory_tree::tree_source::types::TreeKind; - use chrono::Utc; - - fn sample_input(id: &str, content: &str) -> SummaryInput { - let ts = Utc::now(); - SummaryInput { - id: id.to_string(), - content: content.to_string(), - token_count: approx_token_count(content), - entities: Vec::new(), - topics: Vec::new(), - time_range_start: ts, - time_range_end: ts, - score: 0.5, - } - } - - fn test_ctx() -> SummaryContext<'static> { - SummaryContext { - tree_id: "tree-1", - tree_kind: TreeKind::Source, - target_level: 1, - token_budget: 10_000, - } - } - - #[test] - fn build_user_prompt_includes_ids_and_content() { - let inputs = vec![ - sample_input("a", "hello world"), - sample_input("b", "second contribution"), - ]; - let out = build_user_prompt(&inputs, u32::MAX); - assert!(out.contains("[a]")); - assert!(out.contains("hello world")); - assert!(out.contains("[b]")); - assert!(out.contains("second contribution")); - } - - #[test] - fn build_user_prompt_skips_blank_contributions() { - let inputs = vec![sample_input("a", " "), sample_input("b", "kept")]; - let out = build_user_prompt(&inputs, u32::MAX); - assert!(!out.contains("[a]")); - assert!(out.contains("[b]")); - assert!(out.contains("kept")); - } - - #[test] - fn build_user_prompt_clamps_each_input_to_per_input_cap() { - // Regression guard for upper-level context overflow: at L2 with - // SUMMARY_FANOUT=4 and large child summaries, the joined body - // would otherwise blow past NUM_CTX_TOKENS. The clamp keeps - // each contribution under per_input_cap_tokens regardless of - // how big the original content is. - let long = "x".repeat(2_000); // ~500 approx-tokens - let inputs = vec![ - sample_input("a", &long), - sample_input("b", &long), - sample_input("c", &long), - sample_input("d", &long), - ]; - let cap_tokens: u32 = 50; // ~200 chars per input - let out = build_user_prompt(&inputs, cap_tokens); - - // Each input contributes at most cap_tokens*4 chars of content, - // plus a small id header. Total stays well under the unclamped - // 4 * 2_000 = 8_000 chars baseline. - let unclamped_baseline = 4 * 2_000; - assert!( - out.len() < unclamped_baseline / 2, - "expected clamp to halve the body or better, got {} chars", - out.len() - ); - assert!(out.contains("[a]")); - assert!(out.contains("[d]")); - } - - #[test] - fn system_prompt_describes_plain_text_output() { - // When structured_facets is disabled, the prompt asks for plain prose. - let p = system_prompt(4096, false, None); - assert!(!p.contains("4096")); - assert!(!p.contains("Stay well under")); - assert!(!p.contains("\"summary\"")); - assert!(p.to_lowercase().contains("no commentary")); - assert!(p.to_lowercase().contains("no json")); - } - - #[test] - fn extends_prompt_when_flag_enabled() { - let p = system_prompt(4096, true, None); - // When structured_facets is true, the prompt should contain the JSON fence instruction. - assert!( - p.contains("```json"), - "should contain JSON fence instruction" - ); - assert!(p.contains("\"facets\""), "should mention the facets array"); - assert!( - p.contains("evidence_chunks"), - "should mention evidence_chunks" - ); - assert!( - p.contains("canonical keys"), - "should specify canonical keys" - ); - } - - #[test] - fn system_prompt_includes_output_language_directive() { - let p = system_prompt(4096, true, Some("zh-CN")); - assert!(p.contains("Simplified Chinese")); - assert!(p.contains("Keep JSON keys")); - assert!(p.contains("\"summary\"")); - } - - #[test] - fn parses_well_formed_response() { - let raw = "The user prefers pnpm.\n\n\ - ```json\n\ - {\"summary\": \"The user prefers pnpm.\", \"facets\": [\ - {\"class\": \"tooling\", \"key\": \"package_manager\", \ - \"value\": \"pnpm\", \"evidence_chunks\": [\"c1\"], \ - \"confidence\": 0.9, \"cue_family\": \"explicit\"}\ - ]}\n\ - ```"; - let (prose, maybe) = split_structured_response(raw); - assert!( - prose.contains("prefers pnpm"), - "prose should precede the JSON block" - ); - let parsed = maybe - .expect("should find JSON block") - .expect("should parse"); - assert_eq!(parsed.facets.len(), 1); - assert_eq!(parsed.facets[0].key, "package_manager"); - } - - #[test] - fn gracefully_falls_back_on_invalid_json() { - let raw = "Summary text.\n\n```json\nnot valid json\n```"; - let (prose, maybe) = split_structured_response(raw); - assert!(prose.contains("Summary"), "prose should be extracted"); - let result = maybe.expect("fence found"); - assert!(result.is_err(), "invalid JSON should produce Err"); - } - - #[test] - fn respects_disabled_flag() { - let p = system_prompt(4096, false, None); - assert!( - !p.contains("```json"), - "disabled flag must omit JSON instruction" - ); - } - - #[test] - fn clamp_to_budget_no_op_when_under() { - let (out, t) = clamp_to_budget("short", 1000); - assert_eq!(out, "short"); - assert_eq!(t, approx_token_count("short")); - } - - #[test] - fn clamp_to_budget_truncates_when_over() { - let long = "a".repeat(1000); - let (out, t) = clamp_to_budget(&long, 5); - assert!(out.len() < long.len()); - assert!(t <= 6); - } - - /// Mock chat provider that lets us assert prompt shape and stub responses - /// in summariser unit tests without hitting the network. - struct StubProvider { - response: anyhow::Result, - calls: std::sync::atomic::AtomicUsize, - } - - impl StubProvider { - fn ok(text: impl Into) -> Self { - Self { - response: Ok(text.into()), - calls: std::sync::atomic::AtomicUsize::new(0), - } - } - fn err(msg: &'static str) -> Self { - Self { - response: Err(anyhow::anyhow!(msg)), - calls: std::sync::atomic::AtomicUsize::new(0), - } - } - } - - #[async_trait] - impl ChatProvider for StubProvider { - fn name(&self) -> &str { - "test:stub" - } - async fn chat_for_json(&self, _p: &ChatPrompt) -> anyhow::Result { - self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - self.response - .as_ref() - .map(|s| s.clone()) - .map_err(|e| anyhow::anyhow!("{e}")) - } - } - - /// Helper config with structured facet extraction disabled for legacy tests. - fn no_facets_cfg() -> LlmSummariserConfig { - LlmSummariserConfig { - model: "qwen2.5:0.5b".into(), - structured_facet_extraction: false, - output_language: None, - } - } - - #[tokio::test] - async fn empty_inputs_yield_empty_summary_without_provider_call() { - // All inputs are blank → prompt body is empty → the summariser - // short-circuits and returns an empty output without invoking the - // chat provider. - let provider = std::sync::Arc::new(StubProvider::ok("never returned")); - let s = LlmSummariser::new(no_facets_cfg(), provider.clone()); - let inputs = vec![sample_input("a", " "), sample_input("b", "")]; - let out = s.summarise(&inputs, &test_ctx()).await.unwrap(); - assert!(out.content.is_empty()); - assert_eq!(out.token_count, 0); - assert_eq!( - provider.calls.load(std::sync::atomic::Ordering::SeqCst), - 0, - "blank inputs must not call the chat provider" - ); - } - - #[tokio::test] - async fn provider_failure_falls_back_to_inert() { - // Provider errors → must NOT return Err; must fall through to - // InertSummariser's concatenate+truncate behaviour (content - // present, entities empty). - let provider = std::sync::Arc::new(StubProvider::err("simulated")); - let s = LlmSummariser::new(no_facets_cfg(), provider); - let inputs = vec![sample_input("a", "alice decided to ship friday")]; - let out = s.summarise(&inputs, &test_ctx()).await.unwrap(); - assert!(out.content.contains("alice decided to ship")); - assert!(out.entities.is_empty()); - assert!(out.topics.is_empty()); - } - - #[tokio::test] - async fn provider_summary_response_is_used_and_clamped() { - // Provider returns plain text; summariser uses it verbatim - // (after trim) and clamps to the budget. - let provider = std::sync::Arc::new(StubProvider::ok("alice decided to ship friday\n")); - let s = LlmSummariser::new(no_facets_cfg(), provider.clone()); - let inputs = vec![sample_input("a", "alice ships friday")]; - let out = s.summarise(&inputs, &test_ctx()).await.unwrap(); - assert_eq!(out.content, "alice decided to ship friday"); - assert!(out.token_count > 0); - assert_eq!(provider.calls.load(std::sync::atomic::Ordering::SeqCst), 1); - } - - #[test] - fn build_prompt_carries_body_and_kind_tag() { - let provider = std::sync::Arc::new(StubProvider::ok("hi")); - // With structured_facet_extraction disabled, expect plain-text prompt. - let s = LlmSummariser::new( - LlmSummariserConfig { - model: "llama3.1:8b".into(), - structured_facet_extraction: false, - output_language: Some("zh-CN".into()), - }, - provider, - ); - let prompt = s.build_prompt("body", 2048); - assert!(prompt.system.to_lowercase().contains("no commentary")); - assert!(prompt.system.contains("Simplified Chinese")); - assert!(!prompt.system.contains("\"summary\"")); - assert_eq!(prompt.user, "body"); - assert_eq!(prompt.temperature, 0.0); - assert_eq!(prompt.kind, "memory_tree::summarise"); - } -} diff --git a/src/openhuman/memory_tree/tree_source/summariser/mod.rs b/src/openhuman/memory_tree/tree_source/summariser/mod.rs deleted file mode 100644 index c324dffb1..000000000 --- a/src/openhuman/memory_tree/tree_source/summariser/mod.rs +++ /dev/null @@ -1,154 +0,0 @@ -//! Summariser trait + fallback (#709). -//! -//! A summariser folds N buffered items into one sealed summary. Phase 3a -//! ships an `InertSummariser` that concatenates the contributions and -//! truncates to the token budget — enough to make the tree mechanics -//! observable end-to-end without requiring an LLM. Real summarisation -//! (Ollama, etc.) can slot in by implementing the trait. - -use anyhow::Result; -use async_trait::async_trait; -use chrono::{DateTime, Utc}; - -use std::sync::Arc; - -use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::tree_source::types::TreeKind; - -pub mod inert; -pub mod llm; - -/// One contribution being folded — either a raw leaf (chunk) at L0→L1, or -/// a lower-level summary at L_n→L_{n+1}. -#[derive(Clone, Debug)] -pub struct SummaryInput { - /// Primary key of the contribution (chunk id or summary id). - pub id: String, - pub content: String, - pub token_count: u32, - pub entities: Vec, - pub topics: Vec, - pub time_range_start: DateTime, - pub time_range_end: DateTime, - /// Score signal from scoring (for leaves) or parent seal (for summaries). - pub score: f32, -} - -/// Opaque context passed to the summariser — lets implementations log / -/// identify which tree is being sealed without threading config globally. -#[derive(Clone, Debug)] -pub struct SummaryContext<'a> { - pub tree_id: &'a str, - pub tree_kind: TreeKind, - pub target_level: u32, - pub token_budget: u32, -} - -/// Output of a summariser invocation. -#[derive(Clone, Debug)] -pub struct SummaryOutput { - pub content: String, - pub token_count: u32, - pub entities: Vec, - pub topics: Vec, -} - -#[async_trait] -pub trait Summariser: Send + Sync { - /// Fold the inputs into a single summary. `ctx.token_budget` is an - /// upper bound on the produced `token_count`; implementations SHOULD - /// stay well under it so parents have room to include this summary. - async fn summarise( - &self, - inputs: &[SummaryInput], - ctx: &SummaryContext<'_>, - ) -> Result; -} - -/// Build the summariser implementation driven by the workspace's -/// [`Config`]. The cloud-default refactor changed the resolution rules: -/// -/// - `llm_backend = "cloud"` (default): always returns the LLM summariser -/// routed through the OpenHuman backend's `cloud_llm_model` -/// (defaulting to `summarization-v1`). -/// - `llm_backend = "local"`: returns the LLM summariser only when both -/// `llm_summariser_endpoint` AND `llm_summariser_model` are set; -/// otherwise returns the [`inert::InertSummariser`] fallback. -/// -/// In all cases the LLM summariser itself soft-falls-back to inert per -/// seal on transport failure, so seal cascades never abort. -/// -/// Returned as `Arc` so the ingest pipeline can pass it -/// by reference to `append_leaf` and `route_leaf_to_topic_trees` -/// without threading a generic type parameter through every caller. -pub fn build_summariser(config: &Config) -> Arc { - use crate::openhuman::config::DEFAULT_CLOUD_LLM_MODEL; - use crate::openhuman::memory_tree::chat::{build_chat_provider, ChatConsumer}; - - // Resolve the model identifier to log alongside the provider name. - // When memory_provider is local (ollama:*), prefer the legacy - // llm_summariser_endpoint/_model pair when both are set (for back-compat), - // then fall back to the unified workload_local_model so that users who - // configure memory_provider=ollama: without the legacy fields still get - // an LLM summariser instead of the inert fallback. - let model: Option = if config.workload_uses_local("memory") { - let endpoint = config - .memory_tree - .llm_summariser_endpoint - .as_deref() - .map(str::trim) - .filter(|s| !s.is_empty()); - let legacy_model = config - .memory_tree - .llm_summariser_model - .as_deref() - .map(str::trim) - .filter(|s| !s.is_empty()); - match (endpoint, legacy_model) { - (Some(_), Some(m)) => Some(m.to_string()), - _ => config.workload_local_model("memory"), - } - } else { - Some( - config - .memory_tree - .cloud_llm_model - .clone() - .unwrap_or_else(|| DEFAULT_CLOUD_LLM_MODEL.to_string()), - ) - }; - - let Some(model) = model else { - log::debug!( - "[tree_source::summariser] llm_summariser not configured for memory_provider={:?} \ - — using InertSummariser", - config.memory_provider.as_deref().unwrap_or("cloud") - ); - return Arc::new(inert::InertSummariser::new()); - }; - - let provider = match build_chat_provider(config, ChatConsumer::Summarise) { - Ok(p) => p, - Err(err) => { - log::warn!( - "[tree_source::summariser] build_chat_provider failed: {err:#} — \ - falling back to InertSummariser" - ); - return Arc::new(inert::InertSummariser::new()); - } - }; - - log::info!( - "[tree_source::summariser] using LlmSummariser provider={} model={}", - provider.name(), - model - ); - Arc::new(llm::LlmSummariser::new( - llm::LlmSummariserConfig { - model, - structured_facet_extraction: true, - output_language: config.output_language.clone(), - }, - provider, - )) -} diff --git a/src/openhuman/memory_tree/tree_topic/registry.rs b/src/openhuman/memory_tree/tree_topic/registry.rs deleted file mode 100644 index 23aa30c15..000000000 --- a/src/openhuman/memory_tree/tree_topic/registry.rs +++ /dev/null @@ -1,300 +0,0 @@ -//! Topic tree registry — get-or-create / archive (#709 Phase 3c). -//! -//! Topic trees share the same `mem_tree_trees` schema as source trees; the -//! only difference is `kind = 'topic'` and `scope = `. -//! Callers should NOT reach into this module to create topic trees -//! eagerly — use the curator ([`super::curator::maybe_spawn_topic_tree`]) -//! so creation is gated on hotness. Admin flows (future RPC) that want to -//! bypass the gate can call [`force_create_topic_tree`] directly. - -use anyhow::{Context, Result}; -use chrono::Utc; -use uuid::Uuid; - -use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::tree_source::store; -use crate::openhuman::memory_tree::tree_source::types::{Tree, TreeKind, TreeStatus}; - -/// Look up the topic tree for `entity_id`, or create a new one. -/// -/// The `entity_id` is a canonical id from the entity resolver (e.g. -/// `"email:alice@example.com"` or `"hashtag:launch"`). Scope uses the -/// canonical id verbatim so re-lookups are stable. -pub fn get_or_create_topic_tree(config: &Config, entity_id: &str) -> Result { - let entity_kind = entity_id - .split_once(':') - .map(|(k, _)| k) - .unwrap_or("unknown"); - if let Some(existing) = store::get_tree_by_scope(config, TreeKind::Topic, entity_id)? { - log::debug!( - "[tree_topic::registry] found tree id={} entity_kind={}", - existing.id, - entity_kind - ); - return Ok(existing); - } - create_new(config, entity_id) -} - -/// Public alias used by the admin "force materialise" path — semantically -/// identical to [`get_or_create_topic_tree`] but named to make intent at -/// the call site obvious. -pub fn force_create_topic_tree(config: &Config, entity_id: &str) -> Result { - get_or_create_topic_tree(config, entity_id) -} - -/// List all topic trees (both active and archived). Ordered by creation time -/// ascending for stable output. -pub fn list_topic_trees(config: &Config) -> Result> { - use rusqlite::params; - crate::openhuman::memory_tree::store::with_connection(config, |conn| { - let mut stmt = conn.prepare( - "SELECT id, kind, scope, root_id, max_level, status, - created_at_ms, last_sealed_at_ms - FROM mem_tree_trees - WHERE kind = ?1 - ORDER BY created_at_ms ASC", - )?; - let rows = stmt - .query_map(params![TreeKind::Topic.as_str()], row_to_tree_loose)? - .collect::>>() - .context("failed to list topic trees")?; - Ok(rows) - }) -} - -/// Flip a topic tree's status to `archived`. Existing rows remain queryable; -/// new leaves will NOT be routed to this tree until it's manually unarchived -/// (unarchive is not a Phase 3c primitive — Phase 3c just stops routing). -pub fn archive_topic_tree(config: &Config, tree_id: &str) -> Result<()> { - use rusqlite::params; - crate::openhuman::memory_tree::store::with_connection(config, |conn| { - let n = conn - .execute( - "UPDATE mem_tree_trees - SET status = ?1 - WHERE id = ?2 AND kind = ?3", - params![ - TreeStatus::Archived.as_str(), - tree_id, - TreeKind::Topic.as_str() - ], - ) - .with_context(|| format!("failed to archive topic tree {tree_id}"))?; - if n == 0 { - log::warn!( - "[tree_topic::registry] archive_topic_tree: no topic tree with id={tree_id}" - ); - } else { - log::info!("[tree_topic::registry] archived topic tree id={tree_id}"); - } - Ok(()) - }) -} - -fn create_new(config: &Config, entity_id: &str) -> Result { - let tree = Tree { - id: new_topic_tree_id(), - kind: TreeKind::Topic, - scope: entity_id.to_string(), - root_id: None, - max_level: 0, - status: TreeStatus::Active, - created_at: Utc::now(), - last_sealed_at: None, - }; - let entity_kind = entity_id - .split_once(':') - .map(|(k, _)| k) - .unwrap_or("unknown"); - match store::insert_tree(config, &tree) { - Ok(()) => { - log::info!( - "[tree_topic::registry] created topic tree id={} entity_kind={}", - tree.id, - entity_kind - ); - Ok(tree) - } - Err(err) if is_unique_violation(&err) => { - log::debug!( - "[tree_topic::registry] UNIQUE race for entity_kind={} — re-querying", - entity_kind - ); - // Re-query is keyed on the full entity_id; only the *log* line - // has been redacted. This still surfaces enough context to - // diagnose without leaking the recoverable id. - store::get_tree_by_scope(config, TreeKind::Topic, entity_id)?.ok_or_else(|| { - anyhow::anyhow!( - "UNIQUE violation on insert but no row found on re-query (entity_kind={entity_kind})" - ) - }) - } - Err(err) => Err(err), - } -} - -fn is_unique_violation(err: &anyhow::Error) -> bool { - if let Some(rusqlite_err) = err.downcast_ref::() { - if let rusqlite::Error::SqliteFailure(sqlite_err, _) = rusqlite_err { - return sqlite_err.code == rusqlite::ErrorCode::ConstraintViolation; - } - } - let msg = format!("{err:#}"); - msg.contains("UNIQUE constraint failed") -} - -fn new_topic_tree_id() -> String { - format!("{}:{}", TreeKind::Topic.as_str(), Uuid::new_v4()) -} - -/// Row mapper — duplicated from `tree_source::store::row_to_tree` because -/// that one is private. Kept intentionally loose: topic-tree listing is -/// not a hot path so the string parsing cost is immaterial. -fn row_to_tree_loose(row: &rusqlite::Row<'_>) -> rusqlite::Result { - use chrono::TimeZone; - let id: String = row.get(0)?; - let kind_s: String = row.get(1)?; - let scope: String = row.get(2)?; - let root_id: Option = row.get(3)?; - let max_level: i64 = row.get(4)?; - let status_s: String = row.get(5)?; - let created_ms: i64 = row.get(6)?; - let last_sealed_ms: Option = row.get(7)?; - - let kind = TreeKind::parse(&kind_s).map_err(|e| { - rusqlite::Error::FromSqlConversionFailure(1, rusqlite::types::Type::Text, e.into()) - })?; - let status = TreeStatus::parse(&status_s).map_err(|e| { - rusqlite::Error::FromSqlConversionFailure(5, rusqlite::types::Type::Text, e.into()) - })?; - let created_at = Utc - .timestamp_millis_opt(created_ms) - .single() - .ok_or_else(|| { - rusqlite::Error::FromSqlConversionFailure( - 6, - rusqlite::types::Type::Integer, - format!("invalid created_at_ms {created_ms}").into(), - ) - })?; - let last_sealed_at = last_sealed_ms - .map(|ms| { - Utc.timestamp_millis_opt(ms).single().ok_or_else(|| { - rusqlite::Error::FromSqlConversionFailure( - 7, - rusqlite::types::Type::Integer, - format!("invalid last_sealed_at_ms {ms}").into(), - ) - }) - }) - .transpose()?; - - Ok(Tree { - id, - kind, - scope, - root_id, - max_level: max_level.max(0) as u32, - status, - created_at, - last_sealed_at, - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::TempDir; - - fn test_config() -> (TempDir, Config) { - let tmp = TempDir::new().unwrap(); - let mut cfg = Config::default(); - cfg.workspace_dir = tmp.path().to_path_buf(); - (tmp, cfg) - } - - #[test] - fn get_or_create_is_idempotent_on_entity_id() { - let (_tmp, cfg) = test_config(); - let first = get_or_create_topic_tree(&cfg, "email:alice@example.com").unwrap(); - let second = get_or_create_topic_tree(&cfg, "email:alice@example.com").unwrap(); - assert_eq!(first.id, second.id); - assert_eq!(first.kind, TreeKind::Topic); - assert_eq!(first.status, TreeStatus::Active); - assert_eq!(first.scope, "email:alice@example.com"); - } - - #[test] - fn different_entities_yield_different_trees() { - let (_tmp, cfg) = test_config(); - let a = get_or_create_topic_tree(&cfg, "email:alice@example.com").unwrap(); - let b = get_or_create_topic_tree(&cfg, "email:bob@example.com").unwrap(); - assert_ne!(a.id, b.id); - assert_ne!(a.scope, b.scope); - } - - #[test] - fn topic_tree_and_source_tree_share_scope_space_cleanly() { - // A source tree and a topic tree can have the same *logical* - // scope string (e.g. an entity id that looks like a source id) — - // the UNIQUE constraint is on (kind, scope), not scope alone. - let (_tmp, cfg) = test_config(); - let source = - crate::openhuman::memory_tree::tree_source::registry::get_or_create_source_tree( - &cfg, - "shared:slack:#eng", - ) - .unwrap(); - let topic = get_or_create_topic_tree(&cfg, "shared:slack:#eng").unwrap(); - assert_ne!(source.id, topic.id); - assert_eq!(source.kind, TreeKind::Source); - assert_eq!(topic.kind, TreeKind::Topic); - } - - #[test] - fn topic_tree_id_has_expected_prefix() { - let id = new_topic_tree_id(); - assert!(id.starts_with("topic:")); - } - - #[test] - fn archive_flips_status_and_keeps_rows_readable() { - let (_tmp, cfg) = test_config(); - let t = get_or_create_topic_tree(&cfg, "email:alice@example.com").unwrap(); - archive_topic_tree(&cfg, &t.id).unwrap(); - let refetched = store::get_tree(&cfg, &t.id).unwrap().unwrap(); - assert_eq!(refetched.status, TreeStatus::Archived); - // get_or_create should still return the same (archived) row rather - // than creating a new one — archiving is NOT deletion. - let again = get_or_create_topic_tree(&cfg, "email:alice@example.com").unwrap(); - assert_eq!(again.id, t.id); - assert_eq!(again.status, TreeStatus::Archived); - } - - #[test] - fn archive_is_noop_on_nonexistent() { - let (_tmp, cfg) = test_config(); - // Shouldn't error — just log a warning. - archive_topic_tree(&cfg, "topic:does-not-exist").unwrap(); - } - - #[test] - fn list_topic_trees_returns_only_topics() { - let (_tmp, cfg) = test_config(); - // Mix of source + topic trees. - crate::openhuman::memory_tree::tree_source::registry::get_or_create_source_tree( - &cfg, - "slack:#eng", - ) - .unwrap(); - get_or_create_topic_tree(&cfg, "email:alice@example.com").unwrap(); - get_or_create_topic_tree(&cfg, "email:bob@example.com").unwrap(); - - let topics = list_topic_trees(&cfg).unwrap(); - assert_eq!(topics.len(), 2); - for t in &topics { - assert_eq!(t.kind, TreeKind::Topic); - } - } -} diff --git a/src/openhuman/memory_tree/tree_topic/types.rs b/src/openhuman/memory_tree/tree_topic/types.rs deleted file mode 100644 index 734c96840..000000000 --- a/src/openhuman/memory_tree/tree_topic/types.rs +++ /dev/null @@ -1,150 +0,0 @@ -//! Core types for Phase 3c — lazy topic-tree materialisation (#709). -//! -//! A *topic tree* is a per-entity summary tree whose leaves are all chunks -//! mentioning that entity, regardless of the source they came from. They -//! are materialised lazily, driven by a cheap arithmetic *hotness* score -//! over pre-existing signals. Tree mechanics (buffer / seal / cascade) -//! reuse [`source_tree`] end-to-end — topic trees only differ by the -//! `TreeKind::Topic` discriminator and the per-entity `scope`. -//! -//! This file defines: -//! - [`EntityIndexStats`] — input record for the hotness calculation -//! - [`HotnessCounters`] — the persisted row in `mem_tree_entity_hotness` -//! - threshold / cadence constants ([`TOPIC_CREATION_THRESHOLD`], -//! [`TOPIC_ARCHIVE_THRESHOLD`], [`TOPIC_RECHECK_EVERY`]) -//! -//! Persistence helpers for these types live in [`super::store`]. - -use serde::{Deserialize, Serialize}; - -/// Hotness threshold above which a topic tree is materialised for an -/// entity. Tuned (per design) to roughly "several mentions across a few -/// sources" — see [`super::hotness::hotness`] for the formula. -pub const TOPIC_CREATION_THRESHOLD: f32 = 10.0; - -/// Hotness threshold below which a topic tree becomes an archive candidate. -/// Archiving is a primitive in Phase 3c — the scheduled sweep is deferred. -pub const TOPIC_ARCHIVE_THRESHOLD: f32 = 2.0; - -/// How often (in ingests touching the entity) to recompute hotness from -/// the full [`EntityIndexStats`]. Between recomputes we only bump -/// `mention_count_30d` and `last_seen_ms` — cheap integer arithmetic. -pub const TOPIC_RECHECK_EVERY: u32 = 100; - -/// Input record fed to [`super::hotness::hotness`]. -/// -/// Every field is a signal that already exists somewhere in the memory -/// tree (scoring rows, entity index, potential future graph metrics); the -/// struct is an explicit contract so the hotness math can be unit-tested -/// in isolation without touching SQLite. -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct EntityIndexStats { - /// Total mentions in the last 30 days. Phase 3c currently bumps this - /// forever — the 30d window is a TODO once we have a billable clock. - pub mention_count_30d: u32, - /// Number of distinct source trees this entity has appeared in. A - /// cross-source signal — an entity spoken about in one chat channel - /// but nowhere else is less interesting than one that appears in - /// Slack + email + docs. - pub distinct_sources: u32, - /// Epoch-millis of the last ingest that referenced this entity. - pub last_seen_ms: Option, - /// Reserved for Phase 4 retrieval: bump whenever a user query returns - /// this entity. Phase 3c stores the column but never increments it. - pub query_hits_30d: u32, - /// Reserved for a later phase: graph centrality from the entity graph. - /// `None` means "unknown" — not "zero". See [`super::hotness::hotness`]. - pub graph_centrality: Option, -} - -/// Row persisted in `mem_tree_entity_hotness`. Callers interact with this -/// through [`super::store`]; [`EntityIndexStats`] is the hotness-compute -/// view derived from it. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct HotnessCounters { - pub entity_id: String, - pub mention_count_30d: u32, - pub distinct_sources: u32, - pub last_seen_ms: Option, - pub query_hits_30d: u32, - pub graph_centrality: Option, - /// Counts ingests **touching this entity** since the last full hotness - /// recompute. When `>= TOPIC_RECHECK_EVERY` the curator refreshes - /// `distinct_sources` / `last_hotness` and resets this to 0. - pub ingests_since_check: u32, - pub last_hotness: Option, - pub last_updated_ms: i64, -} - -impl HotnessCounters { - /// Fresh row for an entity seen for the first time. - pub fn fresh(entity_id: &str, now_ms: i64) -> Self { - Self { - entity_id: entity_id.to_string(), - mention_count_30d: 0, - distinct_sources: 0, - last_seen_ms: None, - query_hits_30d: 0, - graph_centrality: None, - ingests_since_check: 0, - last_hotness: None, - last_updated_ms: now_ms, - } - } - - /// Project the persisted row into an [`EntityIndexStats`] ready for - /// [`super::hotness::hotness`]. - pub fn stats(&self) -> EntityIndexStats { - EntityIndexStats { - mention_count_30d: self.mention_count_30d, - distinct_sources: self.distinct_sources, - last_seen_ms: self.last_seen_ms, - query_hits_30d: self.query_hits_30d, - graph_centrality: self.graph_centrality, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn fresh_counters_are_zero() { - let c = HotnessCounters::fresh("email:alice@example.com", 1_700_000_000_000); - assert_eq!(c.entity_id, "email:alice@example.com"); - assert_eq!(c.mention_count_30d, 0); - assert_eq!(c.distinct_sources, 0); - assert_eq!(c.ingests_since_check, 0); - assert!(c.last_hotness.is_none()); - assert!(c.last_seen_ms.is_none()); - assert_eq!(c.last_updated_ms, 1_700_000_000_000); - } - - #[test] - fn stats_projection_mirrors_row() { - let c = HotnessCounters { - entity_id: "e".into(), - mention_count_30d: 5, - distinct_sources: 2, - last_seen_ms: Some(42), - query_hits_30d: 1, - graph_centrality: Some(0.3), - ingests_since_check: 4, - last_hotness: Some(9.9), - last_updated_ms: 100, - }; - let s = c.stats(); - assert_eq!(s.mention_count_30d, 5); - assert_eq!(s.distinct_sources, 2); - assert_eq!(s.last_seen_ms, Some(42)); - assert_eq!(s.query_hits_30d, 1); - assert_eq!(s.graph_centrality, Some(0.3)); - } - - #[test] - fn thresholds_make_creation_strictly_above_archive() { - assert!(TOPIC_CREATION_THRESHOLD > TOPIC_ARCHIVE_THRESHOLD); - assert!(TOPIC_RECHECK_EVERY > 0); - } -} diff --git a/src/openhuman/mod.rs b/src/openhuman/mod.rs index 4a29ac5cf..503b3e630 100644 --- a/src/openhuman/mod.rs +++ b/src/openhuman/mod.rs @@ -53,6 +53,14 @@ pub mod mcp_server; pub mod meet; pub mod meet_agent; pub mod memory; +pub mod memory_archivist; +pub mod memory_conversations; +pub mod memory_entities; +pub mod memory_graph; +pub mod memory_queue; +pub mod memory_store; +pub mod memory_sync; +pub mod memory_tools; pub mod memory_tree; pub mod migration; pub mod migrations; diff --git a/src/openhuman/scheduler_gate/gate.rs b/src/openhuman/scheduler_gate/gate.rs index b8add05cf..96fd7a846 100644 --- a/src/openhuman/scheduler_gate/gate.rs +++ b/src/openhuman/scheduler_gate/gate.rs @@ -24,7 +24,7 @@ use crate::openhuman::scheduler_gate::signals::Signals; /// simultaneous Ollama requests have crashed the user's laptop twice. /// /// Cloud-backend LLM calls bypass this semaphore at the worker layer -/// (see `memory_tree::jobs::worker::run_once`) because they're +/// (see `memory::jobs::worker::run_once`) because they're /// bandwidth-bound, not RAM-bound, and the worker pool itself bounds /// concurrency upstream. Keeping this at 1 preserves the laptop-RAM /// contract regardless of backend. diff --git a/src/openhuman/screen_intelligence/tests.rs b/src/openhuman/screen_intelligence/tests.rs index 9d71c0bfe..a19c6317e 100644 --- a/src/openhuman/screen_intelligence/tests.rs +++ b/src/openhuman/screen_intelligence/tests.rs @@ -16,7 +16,7 @@ use super::types::{CaptureFrame, InputActionParams, StartSessionParams}; use crate::openhuman::accessibility::{parse_foreground_output, AppContext}; use crate::openhuman::config::{Config, ScreenIntelligenceConfig}; use crate::openhuman::embeddings::NoopEmbedding; -use crate::openhuman::memory::store::UnifiedMemory; +use crate::openhuman::memory_store::UnifiedMemory; struct EnvVarGuard { key: &'static str, diff --git a/src/openhuman/subconscious/engine.rs b/src/openhuman/subconscious/engine.rs index 49c11156f..1a31638b8 100644 --- a/src/openhuman/subconscious/engine.rs +++ b/src/openhuman/subconscious/engine.rs @@ -21,10 +21,8 @@ use super::types::{ }; use crate::openhuman::config::Config; use crate::openhuman::credentials::{AuthService, APP_SESSION_PROVIDER}; +use crate::openhuman::memory::chat::{build_chat_provider, ChatPrompt, ChatProvider}; use crate::openhuman::memory::MemoryClientRef; -use crate::openhuman::memory_tree::chat::{ - build_chat_provider, ChatConsumer, ChatPrompt, ChatProvider, -}; use anyhow::Result; use executor::ExecutionOutcome; use std::collections::HashMap; @@ -533,7 +531,8 @@ impl SubconsciousEngine { } /// Run the per-tick LLM call. Routes via `subconscious_provider` - /// while reusing the memory-tree chat provider plumbing (#623). On failure returns + /// while reusing the memory LLM adapter over unified inference routing. + /// On failure returns /// `(empty_evaluations, empty_drafts)` so `last_tick_at` is NOT /// advanced — the next tick re-fetches from the same point. async fn evaluate_tasks_and_reflections( @@ -790,23 +789,14 @@ impl SubconsciousEngine { #[derive(Clone, Debug, Eq, PartialEq)] enum SubconsciousProviderRoute { - LocalOllama { endpoint_set: bool, model: String }, + LocalOllama { model: String }, OpenHumanCloud, Other(String), } pub(crate) fn subconscious_provider_unavailable_reason(config: &Config) -> Option { match resolve_subconscious_route(config) { - SubconsciousProviderRoute::LocalOllama { - endpoint_set: true, .. - } => None, - SubconsciousProviderRoute::LocalOllama { - endpoint_set: false, - .. - } => Some( - "Configure the Ollama summarizer endpoint for Subconscious in Settings > AI." - .to_string(), - ), + SubconsciousProviderRoute::LocalOllama { .. } => None, SubconsciousProviderRoute::OpenHumanCloud => { if crate::openhuman::scheduler_gate::is_signed_out() { return Some( @@ -835,16 +825,7 @@ pub(crate) fn subconscious_provider_unavailable_reason(config: &Config) -> Optio fn resolve_subconscious_route(config: &Config) -> SubconsciousProviderRoute { if let Some(model) = config.workload_local_model("subconscious") { - let endpoint_set = config - .memory_tree - .llm_summariser_endpoint - .as_deref() - .map(str::trim) - .is_some_and(|s| !s.is_empty()); - return SubconsciousProviderRoute::LocalOllama { - endpoint_set, - model, - }; + return SubconsciousProviderRoute::LocalOllama { model }; } let raw = config @@ -866,11 +847,11 @@ fn resolve_subconscious_route(config: &Config) -> SubconsciousProviderRoute { fn build_subconscious_chat_provider(config: &Config) -> Result> { let mut routed = config.clone(); routed.memory_provider = match resolve_subconscious_route(config) { - SubconsciousProviderRoute::LocalOllama { model, .. } => Some(format!("ollama:{model}")), + SubconsciousProviderRoute::LocalOllama { model } => Some(format!("ollama:{model}")), SubconsciousProviderRoute::OpenHumanCloud => Some("cloud".to_string()), SubconsciousProviderRoute::Other(provider) => Some(provider), }; - build_chat_provider(&routed, ChatConsumer::Summarise) + build_chat_provider(&routed) } /// Parse the per-tick LLM response into evaluations + reflection drafts. diff --git a/src/openhuman/subconscious/engine_tests.rs b/src/openhuman/subconscious/engine_tests.rs index 94c32fd1e..a7e123c0a 100644 --- a/src/openhuman/subconscious/engine_tests.rs +++ b/src/openhuman/subconscious/engine_tests.rs @@ -93,13 +93,12 @@ async fn tick_skips_unavailable_provider_without_activity_log_spam() { } #[test] -fn local_subconscious_provider_with_endpoint_is_available() { +fn local_subconscious_provider_is_available() { let tmp = tempfile::tempdir().expect("tempdir"); let mut config = Config::default(); config.config_path = tmp.path().join("config.toml"); config.workspace_dir = tmp.path().join("workspace"); config.subconscious_provider = Some("ollama:qwen2.5:0.5b".into()); - config.memory_tree.llm_summariser_endpoint = Some("http://localhost:11434".into()); assert!(subconscious_provider_unavailable_reason(&config).is_none()); } @@ -108,26 +107,22 @@ fn local_subconscious_provider_with_endpoint_is_available() { fn local_subconscious_route_preserves_ollama_model() { let mut config = Config::default(); config.subconscious_provider = Some("ollama:qwen2.5:0.5b".into()); - config.memory_tree.llm_summariser_endpoint = Some("http://localhost:11434".into()); assert_eq!( resolve_subconscious_route(&config), SubconsciousProviderRoute::LocalOllama { - endpoint_set: true, model: "qwen2.5:0.5b".into(), } ); } #[test] -fn local_subconscious_provider_without_endpoint_is_unavailable() { +fn local_subconscious_provider_does_not_require_legacy_endpoint() { let mut config = Config::default(); config.subconscious_provider = Some("ollama:qwen2.5:0.5b".into()); config.memory_tree.llm_summariser_endpoint = None; - let reason = subconscious_provider_unavailable_reason(&config).expect("unavailable reason"); - - assert!(reason.contains("Ollama summarizer endpoint")); + assert!(subconscious_provider_unavailable_reason(&config).is_none()); } #[test] diff --git a/src/openhuman/subconscious/situation_report/digest.rs b/src/openhuman/subconscious/situation_report/digest.rs index 0e0ce94cd..93b742328 100644 --- a/src/openhuman/subconscious/situation_report/digest.rs +++ b/src/openhuman/subconscious/situation_report/digest.rs @@ -12,7 +12,7 @@ //! exactly what was happening before this section was gated. use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::tree_source::types::TreeKind; +use crate::openhuman::memory_store::trees::types::TreeKind; /// Truncate point for the digest body in the situation report. const DIGEST_BODY_PREVIEW: usize = 1200; @@ -63,7 +63,7 @@ struct DigestRow { } fn read_latest_global_l0(config: &Config, cutoff_ms: i64) -> anyhow::Result> { - crate::openhuman::memory_tree::store::with_connection(config, |conn| { + crate::openhuman::memory_store::chunks::store::with_connection(config, |conn| { let row = conn .query_row( "SELECT s.id, s.content, s.sealed_at_ms diff --git a/src/openhuman/subconscious/situation_report/hotness.rs b/src/openhuman/subconscious/situation_report/hotness.rs index 6999a1655..76a37d7ec 100644 --- a/src/openhuman/subconscious/situation_report/hotness.rs +++ b/src/openhuman/subconscious/situation_report/hotness.rs @@ -145,7 +145,7 @@ struct HotnessDelta { /// subquery over `mem_tree_entity_index` (#1365): true iff any indexed /// row for this entity has `is_user = 1`. fn read_current_hotness(config: &Config) -> anyhow::Result> { - crate::openhuman::memory_tree::store::with_connection(config, |conn| { + crate::openhuman::memory_store::chunks::store::with_connection(config, |conn| { let mut stmt = conn.prepare( "SELECT h.entity_id, h.last_hotness, diff --git a/src/openhuman/subconscious/situation_report/query_window.rs b/src/openhuman/subconscious/situation_report/query_window.rs index d167239a9..8c39c20f2 100644 --- a/src/openhuman/subconscious/situation_report/query_window.rs +++ b/src/openhuman/subconscious/situation_report/query_window.rs @@ -11,7 +11,7 @@ use std::fmt::Write; use crate::openhuman::config::Config; -use crate::openhuman::memory_tree::retrieval::global::query_global; +use crate::openhuman::memory::retrieval::global::query_global; /// Cold-start fallback window when `last_tick_at` is unset. const COLD_START_DAYS: u32 = 7; diff --git a/src/openhuman/subconscious/situation_report/summaries.rs b/src/openhuman/subconscious/situation_report/summaries.rs index 253e79262..58919e605 100644 --- a/src/openhuman/subconscious/situation_report/summaries.rs +++ b/src/openhuman/subconscious/situation_report/summaries.rs @@ -65,7 +65,7 @@ struct SummaryRow { } fn read_recent_summaries(config: &Config, cutoff_ms: i64) -> anyhow::Result> { - crate::openhuman::memory_tree::store::with_connection(config, |conn| { + crate::openhuman::memory_store::chunks::store::with_connection(config, |conn| { let mut stmt = conn.prepare( "SELECT s.id, s.level, s.content, t.scope FROM mem_tree_summaries s diff --git a/src/openhuman/subconscious/source_chunk.rs b/src/openhuman/subconscious/source_chunk.rs index f9870e6c1..4ec709977 100644 --- a/src/openhuman/subconscious/source_chunk.rs +++ b/src/openhuman/subconscious/source_chunk.rs @@ -150,7 +150,7 @@ fn resolve_summary(config: &crate::openhuman::config::Config, raw: &str) -> Sour // `L:` token, which left no row matching anything in the // table. let lookup: anyhow::Result> = - crate::openhuman::memory_tree::store::with_connection(config, |conn| { + crate::openhuman::memory_store::chunks::store::with_connection(config, |conn| { let mut stmt = conn.prepare( "SELECT s.content, s.level, t.scope FROM mem_tree_summaries s @@ -219,7 +219,7 @@ fn resolve_entity(config: &crate::openhuman::config::Config, raw: &str) -> Sourc let original_kind = parse_ref(raw).0.to_string(); type EntityLookup = anyhow::Result)>>; let lookup: EntityLookup = - crate::openhuman::memory_tree::store::with_connection(config, |conn| { + crate::openhuman::memory_store::chunks::store::with_connection(config, |conn| { // Top-scoring surface form for this entity. let mut stmt = conn.prepare( "SELECT entity_kind, surface, score diff --git a/src/openhuman/test_support/rpc.rs b/src/openhuman/test_support/rpc.rs index 25ed25610..6396d0015 100644 --- a/src/openhuman/test_support/rpc.rs +++ b/src/openhuman/test_support/rpc.rs @@ -15,7 +15,7 @@ use serde_json::json; use crate::openhuman::config::Config; use crate::openhuman::config::{clear_active_user, default_root_openhuman_dir}; use crate::openhuman::cron; -use crate::openhuman::memory_tree::read_rpc; +use crate::openhuman::memory::read_rpc; use crate::rpc::RpcOutcome; const E2E_MODE_ENV_VAR: &str = "OPENHUMAN_E2E_MODE"; diff --git a/src/openhuman/tools/impl/agent/save_preference.rs b/src/openhuman/tools/impl/agent/save_preference.rs index 922ed41c1..3550ed217 100644 --- a/src/openhuman/tools/impl/agent/save_preference.rs +++ b/src/openhuman/tools/impl/agent/save_preference.rs @@ -23,7 +23,8 @@ use std::sync::Arc; use async_trait::async_trait; use serde_json::json; -use crate::openhuman::memory::{safety, Memory, MemoryCategory}; +use crate::openhuman::memory::{Memory, MemoryCategory}; +use crate::openhuman::memory_store::safety; use crate::openhuman::security::policy::ToolOperation; use crate::openhuman::security::SecurityPolicy; use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolResult}; diff --git a/src/openhuman/tools/impl/memory/store.rs b/src/openhuman/tools/impl/memory/store.rs index a00808e38..2c2add0b2 100644 --- a/src/openhuman/tools/impl/memory/store.rs +++ b/src/openhuman/tools/impl/memory/store.rs @@ -1,5 +1,5 @@ -use crate::openhuman::memory::safety; use crate::openhuman::memory::{Memory, MemoryCategory}; +use crate::openhuman::memory_store::safety; use crate::openhuman::security::policy::ToolOperation; use crate::openhuman::security::SecurityPolicy; use crate::openhuman::tools::traits::{Tool, ToolResult}; diff --git a/src/openhuman/whatsapp_data/sqlite_retry.rs b/src/openhuman/whatsapp_data/sqlite_retry.rs index f4066c11f..92e64586d 100644 --- a/src/openhuman/whatsapp_data/sqlite_retry.rs +++ b/src/openhuman/whatsapp_data/sqlite_retry.rs @@ -1,6 +1,6 @@ //! SQLite busy/locked detection and retry-with-backoff for WhatsApp data writes. //! -//! Modelled on [`crate::openhuman::memory_tree::jobs::worker::is_sqlite_busy`] — +//! Modelled on [`crate::openhuman::memory::jobs::worker::is_sqlite_busy`] — //! the configured `busy_timeout` absorbs short waits inside rusqlite; this layer //! catches residual `SQLITE_BUSY` / `SQLITE_LOCKED` after that window. diff --git a/tests/agent_retrieval_e2e.rs b/tests/agent_retrieval_e2e.rs index 1efc7fdba..e5ebfa3be 100644 --- a/tests/agent_retrieval_e2e.rs +++ b/tests/agent_retrieval_e2e.rs @@ -21,10 +21,10 @@ use chrono::{TimeZone, Utc}; use openhuman_core::openhuman::config::Config; -use openhuman_core::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage}; -use openhuman_core::openhuman::memory_tree::canonicalize::email::{EmailMessage, EmailThread}; -use openhuman_core::openhuman::memory_tree::ingest::{ingest_chat, ingest_email}; -use openhuman_core::openhuman::memory_tree::jobs::drain_until_idle; +use openhuman_core::openhuman::memory::ingest_pipeline::{ingest_chat, ingest_email}; +use openhuman_core::openhuman::memory::jobs::drain_until_idle; +use openhuman_core::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage}; +use openhuman_core::openhuman::memory_sync::canonicalize::email::{EmailMessage, EmailThread}; use openhuman_core::openhuman::tools::{ MemoryTreeFetchLeavesTool, MemoryTreeQueryTopicTool, MemoryTreeSearchEntitiesTool, Tool, }; diff --git a/tests/agentmemory_backend.rs b/tests/agentmemory_backend.rs deleted file mode 100644 index 1cc011097..000000000 --- a/tests/agentmemory_backend.rs +++ /dev/null @@ -1,544 +0,0 @@ -//! Integration tests for the agentmemory `Memory` backend. -//! -//! Spins up a small axum mock server that mimics the agentmemory REST -//! contract (matches the v0.9.12 wire shapes) and exercises every trait -//! method end-to-end. Tests are kept in `tests/` so they share the public -//! crate surface — they do not reach into private modules. - -use std::net::SocketAddr; -use std::sync::{Arc, Mutex}; - -use axum::{ - extract::{Query, State}, - http::StatusCode, - routing::{get, post}, - Json, Router, -}; -use openhuman_core::openhuman::config::MemoryConfig; -use openhuman_core::openhuman::memory::store::AgentMemoryBackend; -use openhuman_core::openhuman::memory::traits::{Memory, MemoryCategory, RecallOpts}; -use serde::{Deserialize, Serialize}; - -#[derive(Default, Clone)] -struct MockState { - memories: Arc>>, - next_id: Arc>, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -struct MockMemory { - id: String, - project: Option, - title: Option, - content: Option, - #[serde(rename = "type")] - kind: Option, - #[serde(default)] - concepts: Vec, - #[serde(default, rename = "sessionIds")] - session_ids: Vec, - #[serde(rename = "updatedAt")] - updated_at: Option, - #[serde(skip_serializing_if = "Option::is_none")] - score: Option, -} - -#[derive(Debug, Deserialize)] -struct RememberRequest { - project: String, - title: String, - content: String, - #[serde(rename = "type")] - kind: String, - #[serde(default)] - concepts: Vec, - #[serde(default, rename = "sessionIds")] - session_ids: Vec, -} - -#[derive(Debug, Deserialize)] -struct SmartSearchRequest { - query: String, - limit: usize, - #[serde(default)] - project: Option, -} - -#[derive(Debug, Deserialize)] -struct ForgetRequest { - id: String, -} - -#[derive(Debug, Deserialize)] -struct MemoriesQuery { - #[serde(default)] - project: Option, - #[serde(default)] - #[allow(dead_code)] - latest: Option, -} - -async fn start_mock_server() -> (SocketAddr, MockState) { - let state = MockState::default(); - let app = Router::new() - .route( - "/agentmemory/livez", - get(|| async { Json(serde_json::json!({"service":"agentmemory","status":"ok"})) }), - ) - .route( - "/agentmemory/health", - get(handle_health).with_state(state.clone()), - ) - .route( - "/agentmemory/remember", - post(handle_remember).with_state(state.clone()), - ) - .route( - "/agentmemory/smart-search", - post(handle_smart_search).with_state(state.clone()), - ) - .route( - "/agentmemory/forget", - post(handle_forget).with_state(state.clone()), - ) - .route( - "/agentmemory/memories", - get(handle_memories).with_state(state.clone()), - ) - .route( - "/agentmemory/projects", - get(handle_projects).with_state(state.clone()), - ); - - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve(listener, app).await.unwrap(); - }); - (addr, state) -} - -async fn handle_health(State(state): State) -> Json { - let n = state.memories.lock().unwrap().len(); - Json(serde_json::json!({"memories": n, "status": "ok"})) -} - -async fn handle_remember( - State(state): State, - Json(req): Json, -) -> (StatusCode, Json) { - let mut next_id = state.next_id.lock().unwrap(); - *next_id += 1; - let id = format!("mem_{}", *next_id); - state.memories.lock().unwrap().push(MockMemory { - id: id.clone(), - project: Some(req.project), - title: Some(req.title), - content: Some(req.content), - kind: Some(req.kind), - concepts: req.concepts, - session_ids: req.session_ids, - updated_at: Some("2026-05-14T00:00:00Z".to_string()), - score: None, - }); - (StatusCode::CREATED, Json(serde_json::json!({ "id": id }))) -} - -async fn handle_smart_search( - State(state): State, - Json(req): Json, -) -> Json { - let memories = state.memories.lock().unwrap(); - let q = req.query.to_lowercase(); - let project = req.project.as_deref(); - let hits: Vec = memories - .iter() - .filter(|m| project.is_none_or(|p| m.project.as_deref() == Some(p))) - .filter(|m| { - m.title - .as_deref() - .map(|t| t.to_lowercase().contains(&q)) - .unwrap_or(false) - || m.content - .as_deref() - .map(|c| c.to_lowercase().contains(&q)) - .unwrap_or(false) - || m.concepts.iter().any(|c| c.to_lowercase().contains(&q)) - }) - .take(req.limit) - .cloned() - .map(|mut m| { - m.score = Some(0.9); - m - }) - .collect(); - Json(serde_json::json!({ "mode": "compact", "results": hits })) -} - -async fn handle_forget( - State(state): State, - Json(req): Json, -) -> Json { - let mut memories = state.memories.lock().unwrap(); - let before = memories.len(); - memories.retain(|m| m.id != req.id); - let forgotten = memories.len() < before; - Json(serde_json::json!({ "forgotten": forgotten })) -} - -async fn handle_memories( - State(state): State, - Query(q): Query, -) -> Json { - let memories = state.memories.lock().unwrap(); - let filtered: Vec = memories - .iter() - .filter(|m| { - q.project - .as_deref() - .is_none_or(|p| m.project.as_deref() == Some(p)) - }) - .cloned() - .collect(); - Json(serde_json::json!({ "memories": filtered })) -} - -async fn handle_projects(State(state): State) -> Json { - use std::collections::BTreeMap; - let memories = state.memories.lock().unwrap(); - let mut by_project: BTreeMap)> = BTreeMap::new(); - for m in memories.iter() { - let ns = m.project.clone().unwrap_or_else(|| "default".to_string()); - let entry = by_project.entry(ns).or_insert((0, None)); - entry.0 += 1; - if entry.1.is_none() { - entry.1 = m.updated_at.clone(); - } - } - let projects: Vec = by_project - .into_iter() - .map(|(name, (count, last_updated))| { - serde_json::json!({ - "name": name, - "count": count, - "lastUpdated": last_updated, - }) - }) - .collect(); - Json(serde_json::json!({ "projects": projects })) -} - -fn make_config(addr: SocketAddr) -> MemoryConfig { - MemoryConfig { - backend: "agentmemory".to_string(), - agentmemory_url: Some(format!("http://{addr}")), - agentmemory_timeout_ms: Some(2_000), - ..MemoryConfig::default() - } -} - -#[tokio::test] -async fn health_check_passes_against_running_daemon() { - let (addr, _state) = start_mock_server().await; - let backend = AgentMemoryBackend::from_config(&make_config(addr)).unwrap(); - assert!(backend.health_check().await); -} - -#[tokio::test] -async fn health_check_fails_when_daemon_is_unreachable() { - let cfg = MemoryConfig { - backend: "agentmemory".to_string(), - agentmemory_url: Some("http://127.0.0.1:1".to_string()), - agentmemory_timeout_ms: Some(500), - ..MemoryConfig::default() - }; - let backend = AgentMemoryBackend::from_config(&cfg).unwrap(); - assert!(!backend.health_check().await); -} - -#[tokio::test] -async fn store_then_get_round_trip() { - let (addr, _state) = start_mock_server().await; - let backend = AgentMemoryBackend::from_config(&make_config(addr)).unwrap(); - - backend - .store( - "demo", - "auth-stack", - "Service uses HMAC bearer tokens; refresh every 24h.", - MemoryCategory::Core, - None, - ) - .await - .unwrap(); - - let entry = backend - .get("demo", "auth-stack") - .await - .unwrap() - .expect("expected to recall the stored memory"); - assert_eq!(entry.key, "auth-stack"); - assert_eq!(entry.namespace.as_deref(), Some("demo")); - assert!(entry.content.contains("HMAC")); - assert_eq!(entry.category, MemoryCategory::Core); -} - -#[tokio::test] -async fn store_then_recall_finds_matching_memory() { - let (addr, _state) = start_mock_server().await; - let backend = AgentMemoryBackend::from_config(&make_config(addr)).unwrap(); - - backend - .store( - "demo", - "k1", - "hmac bearer auth refresh", - MemoryCategory::Core, - None, - ) - .await - .unwrap(); - backend - .store( - "demo", - "k2", - "stripe webhook handling", - MemoryCategory::Core, - None, - ) - .await - .unwrap(); - - let opts = RecallOpts { - namespace: Some("demo"), - ..RecallOpts::default() - }; - let hits = backend.recall("hmac", 10, opts).await.unwrap(); - assert_eq!(hits.len(), 1); - assert_eq!(hits[0].key, "k1"); - assert!(hits[0].score.unwrap() > 0.5); -} - -#[tokio::test] -async fn recall_filters_by_session_id_client_side() { - let (addr, _state) = start_mock_server().await; - let backend = AgentMemoryBackend::from_config(&make_config(addr)).unwrap(); - - backend - .store( - "demo", - "k1", - "hmac one", - MemoryCategory::Core, - Some("ses-1"), - ) - .await - .unwrap(); - backend - .store( - "demo", - "k2", - "hmac two", - MemoryCategory::Core, - Some("ses-2"), - ) - .await - .unwrap(); - - let opts = RecallOpts { - namespace: Some("demo"), - session_id: Some("ses-1"), - ..RecallOpts::default() - }; - let hits = backend.recall("hmac", 10, opts).await.unwrap(); - assert_eq!(hits.len(), 1); - assert_eq!(hits[0].session_id.as_deref(), Some("ses-1")); -} - -#[tokio::test] -async fn recall_min_score_drops_scoreless_and_below_threshold_hits() { - let (addr, _state) = start_mock_server().await; - let backend = AgentMemoryBackend::from_config(&make_config(addr)).unwrap(); - - backend - .store("demo", "k", "hmac auth refresh", MemoryCategory::Core, None) - .await - .unwrap(); - - // Mock always returns score=0.9; a threshold above that should - // drop the hit. Scoreless rows are not relevant on this path - // (smart-search hits always carry a score in the mock). - let opts = RecallOpts { - namespace: Some("demo"), - min_score: Some(0.95), - ..RecallOpts::default() - }; - let hits = backend.recall("hmac", 10, opts).await.unwrap(); - assert!( - hits.is_empty(), - "min_score = 0.95 should drop the 0.9 hit, got {hits:?}" - ); - - let opts_loose = RecallOpts { - namespace: Some("demo"), - min_score: Some(0.5), - ..RecallOpts::default() - }; - let hits_loose = backend.recall("hmac", 10, opts_loose).await.unwrap(); - assert_eq!(hits_loose.len(), 1); -} - -#[tokio::test] -async fn list_with_no_namespace_returns_every_project() { - let (addr, _state) = start_mock_server().await; - let backend = AgentMemoryBackend::from_config(&make_config(addr)).unwrap(); - - backend - .store("alpha", "a1", "x", MemoryCategory::Core, None) - .await - .unwrap(); - backend - .store("beta", "b1", "y", MemoryCategory::Core, None) - .await - .unwrap(); - - let all = backend.list(None, None, None).await.unwrap(); - assert_eq!(all.len(), 2); - let mut ns: Vec<_> = all - .iter() - .map(|e| e.namespace.clone().unwrap_or_default()) - .collect(); - ns.sort(); - assert_eq!(ns, vec!["alpha", "beta"]); -} - -#[tokio::test] -async fn list_filters_by_namespace_and_category() { - let (addr, _state) = start_mock_server().await; - let backend = AgentMemoryBackend::from_config(&make_config(addr)).unwrap(); - - backend - .store("alpha", "a1", "fact", MemoryCategory::Core, None) - .await - .unwrap(); - backend - .store("alpha", "a2", "convo", MemoryCategory::Conversation, None) - .await - .unwrap(); - backend - .store("beta", "b1", "fact", MemoryCategory::Core, None) - .await - .unwrap(); - - let all_alpha = backend.list(Some("alpha"), None, None).await.unwrap(); - assert_eq!(all_alpha.len(), 2); - - let only_facts = backend - .list(Some("alpha"), Some(&MemoryCategory::Core), None) - .await - .unwrap(); - assert_eq!(only_facts.len(), 1); - assert_eq!(only_facts[0].key, "a1"); -} - -#[tokio::test] -async fn forget_removes_existing_memory_and_reports_missing() { - let (addr, _state) = start_mock_server().await; - let backend = AgentMemoryBackend::from_config(&make_config(addr)).unwrap(); - - backend - .store("demo", "doomed", "delete me", MemoryCategory::Core, None) - .await - .unwrap(); - - assert!(backend.forget("demo", "doomed").await.unwrap()); - // Second time around the key is gone. - assert!(!backend.forget("demo", "doomed").await.unwrap()); - // Unknown key reports missing without an error. - assert!(!backend.forget("demo", "never-existed").await.unwrap()); -} - -#[tokio::test] -async fn namespace_summaries_aggregate_per_project() { - let (addr, _state) = start_mock_server().await; - let backend = AgentMemoryBackend::from_config(&make_config(addr)).unwrap(); - - backend - .store("alpha", "a1", "x", MemoryCategory::Core, None) - .await - .unwrap(); - backend - .store("alpha", "a2", "y", MemoryCategory::Core, None) - .await - .unwrap(); - backend - .store("beta", "b1", "z", MemoryCategory::Core, None) - .await - .unwrap(); - - let mut summaries = backend.namespace_summaries().await.unwrap(); - summaries.sort_by(|a, b| a.namespace.cmp(&b.namespace)); - assert_eq!(summaries.len(), 2); - assert_eq!(summaries[0].namespace, "alpha"); - assert_eq!(summaries[0].count, 2); - assert_eq!(summaries[1].namespace, "beta"); - assert_eq!(summaries[1].count, 1); -} - -#[tokio::test] -async fn count_reads_total_from_health_endpoint() { - let (addr, _state) = start_mock_server().await; - let backend = AgentMemoryBackend::from_config(&make_config(addr)).unwrap(); - - backend - .store("demo", "k1", "x", MemoryCategory::Core, None) - .await - .unwrap(); - backend - .store("demo", "k2", "y", MemoryCategory::Core, None) - .await - .unwrap(); - - assert_eq!(backend.count().await.unwrap(), 2); -} - -#[tokio::test] -async fn name_returns_agentmemory_string() { - let (addr, _state) = start_mock_server().await; - let backend = AgentMemoryBackend::from_config(&make_config(addr)).unwrap(); - assert_eq!(backend.name(), "agentmemory"); -} - -#[test] -fn from_config_rejects_empty_url() { - let cfg = MemoryConfig { - backend: "agentmemory".to_string(), - agentmemory_url: Some(" ".to_string()), - ..MemoryConfig::default() - }; - // `AgentMemoryBackend` does not derive `Debug` (its inner `reqwest::Client` - // is opaque), so use a `match` instead of `.unwrap_err()`. - match AgentMemoryBackend::from_config(&cfg) { - Ok(_) => panic!("expected error for empty url"), - Err(err) => assert!( - err.to_string().contains("cannot be empty"), - "unexpected error: {err}" - ), - } -} - -#[test] -fn from_config_rejects_invalid_url() { - let cfg = MemoryConfig { - backend: "agentmemory".to_string(), - agentmemory_url: Some("not a url".to_string()), - ..MemoryConfig::default() - }; - match AgentMemoryBackend::from_config(&cfg) { - Ok(_) => panic!("expected error for invalid url"), - Err(err) => assert!( - err.to_string().contains("not a valid URL"), - "expected URL error, got: {err}" - ), - } -} diff --git a/tests/learning_phase4_integration_test.rs b/tests/learning_phase4_integration_test.rs index 2cb02620f..687cf52b1 100644 --- a/tests/learning_phase4_integration_test.rs +++ b/tests/learning_phase4_integration_test.rs @@ -22,7 +22,7 @@ use openhuman_core::openhuman::learning::candidate::{ }; use openhuman_core::openhuman::learning::profile_md_renderer::ProfileMdRenderer; use openhuman_core::openhuman::learning::stability_detector::StabilityDetector; -use openhuman_core::openhuman::memory::store::profile::{ +use openhuman_core::openhuman::memory_store::profile::{ FacetState, FacetType, ProfileFacet, UserState, PROFILE_INIT_SQL, }; use parking_lot::Mutex; diff --git a/tests/memory_tree_summarizer_e2e.rs b/tests/memory_tree_summarizer_e2e.rs index ac387a19a..1a7e9176c 100644 --- a/tests/memory_tree_summarizer_e2e.rs +++ b/tests/memory_tree_summarizer_e2e.rs @@ -32,7 +32,7 @@ use tempfile::tempdir; use openhuman_core::openhuman::config::Config; use openhuman_core::openhuman::inference::provider::traits::Provider; -use openhuman_core::openhuman::memory_tree::summarizer::{engine, store}; +use openhuman_core::openhuman::memory_tree::tree_runtime::{engine, store}; // ── Env isolation ───────────────────────────────────────────────────────── diff --git a/tests/memory_tree_walk_e2e.rs b/tests/memory_tree_walk_e2e.rs index f3b15ac01..2f042fd9e 100644 --- a/tests/memory_tree_walk_e2e.rs +++ b/tests/memory_tree_walk_e2e.rs @@ -29,11 +29,11 @@ use openhuman_core::openhuman::config::Config; use openhuman_core::openhuman::inference::provider::compatible::{ AuthStyle, OpenAiCompatibleProvider, }; -use openhuman_core::openhuman::memory_tree::summarizer::store::write_node; -use openhuman_core::openhuman::memory_tree::summarizer::types::{ +use openhuman_core::openhuman::memory_tree::tools::walk::{run_walk, WalkOptions, WalkStopReason}; +use openhuman_core::openhuman::memory_tree::tree_runtime::store::write_node; +use openhuman_core::openhuman::memory_tree::tree_runtime::types::{ derive_parent_id, estimate_tokens, level_from_node_id, TreeNode, }; -use openhuman_core::openhuman::memory_tree::tools::walk::{run_walk, WalkOptions, WalkStopReason}; // ── Environment serialisation lock ────────────────────────────────────────── // diff --git a/tests/ollama_embeddings_fallback_e2e.rs b/tests/ollama_embeddings_fallback_e2e.rs index 01951016b..eae79266d 100644 --- a/tests/ollama_embeddings_fallback_e2e.rs +++ b/tests/ollama_embeddings_fallback_e2e.rs @@ -28,9 +28,8 @@ use openhuman_core::openhuman::embeddings::{ DEFAULT_CLOUD_EMBEDDING_DIMENSIONS, DEFAULT_CLOUD_EMBEDDING_MODEL, DEFAULT_OLLAMA_DIMENSIONS, DEFAULT_OLLAMA_MODEL, }; -use openhuman_core::openhuman::memory::{ - effective_embedding_settings, effective_embedding_settings_probed, -}; +use openhuman_core::openhuman::memory::effective_embedding_settings; +use openhuman_core::openhuman::memory_store::factories::effective_embedding_settings_probed; // ── Env isolation ───────────────────────────────────────────────────────────── diff --git a/tests/screen_intelligence_vision_e2e.rs b/tests/screen_intelligence_vision_e2e.rs index e22921815..f0dc17de9 100644 --- a/tests/screen_intelligence_vision_e2e.rs +++ b/tests/screen_intelligence_vision_e2e.rs @@ -36,8 +36,8 @@ use image::{ImageBuffer, Rgb, RgbImage}; use tempfile::tempdir; use openhuman_core::openhuman::embeddings::NoopEmbedding; -use openhuman_core::openhuman::memory::store::types::NamespaceDocumentInput; -use openhuman_core::openhuman::memory::store::UnifiedMemory; +use openhuman_core::openhuman::memory_store::types::NamespaceDocumentInput; +use openhuman_core::openhuman::memory_store::UnifiedMemory; use openhuman_core::openhuman::screen_intelligence::CaptureFrame; use openhuman_core::openhuman::screen_intelligence::{ global_engine, AccessibilityEngine, VisionSummary,