test(memory): extend memory coverage across retrieval and tooling (#2574)

This commit is contained in:
Steven Enamakel
2026-05-24 13:21:12 -07:00
committed by GitHub
parent 59c1687dd4
commit a47740d4c9
313 changed files with 14757 additions and 6643 deletions
+6
View File
@@ -14,6 +14,7 @@ import { waitForApp, waitForAppReady } from '../helpers/app-helpers';
import { hasAppChrome } from '../helpers/element-helpers';
import { resetApp } from '../helpers/reset-app';
import { waitForHomePage } from '../helpers/shared-flows';
import { startMockServer, stopMockServer } from '../mock-server';
const USER_ID = 'e2e-smoke';
@@ -21,10 +22,15 @@ describe('Smoke', function () {
this.timeout(120_000);
before(async () => {
await startMockServer();
await waitForApp();
await resetApp(USER_ID);
});
after(async () => {
await stopMockServer();
});
it('has a live WebDriver session', async () => {
const sessionId = browser.sessionId;
expect(sessionId).toBeDefined();
+5 -5
View File
@@ -41,13 +41,13 @@ use openhuman_core::openhuman::composio::providers::registry::{
get_provider, init_default_providers,
};
use openhuman_core::openhuman::config::Config;
use openhuman_core::openhuman::memory_tree::content_store::read::{
verify_chunk_file, verify_summary_file, VerifyResult,
};
use openhuman_core::openhuman::memory_tree::jobs::drain_until_idle;
use openhuman_core::openhuman::memory_tree::store::{
use openhuman_core::openhuman::memory::jobs::drain_until_idle;
use openhuman_core::openhuman::memory_store::chunks::store::{
get_chunk_content_pointers, list_chunks, list_summaries_with_content_path, ListChunksQuery,
};
use openhuman_core::openhuman::memory_store::content::read::{
verify_chunk_file, verify_summary_file, VerifyResult,
};
#[derive(Parser, Debug)]
#[command(
+1 -1
View File
@@ -30,7 +30,7 @@ use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use openhuman_core::openhuman::config::Config;
use openhuman_core::openhuman::memory_tree::store::with_connection;
use openhuman_core::openhuman::memory_store::chunks::store::with_connection;
fn main() -> ExitCode {
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info"))
+2 -2
View File
@@ -211,8 +211,8 @@ async fn main() -> Result<()> {
if cli.seal_probe {
use chrono::{Duration, Utc};
use openhuman_core::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage};
use openhuman_core::openhuman::memory_tree::ingest::ingest_chat;
use openhuman_core::openhuman::memory::ingest_pipeline::ingest_chat;
use openhuman_core::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage};
let connection_id = cli.connection_id.clone().ok_or_else(|| {
anyhow::anyhow!(
+3 -1
View File
@@ -68,7 +68,9 @@ pub fn run_from_cli_args(args: &[String]) -> Result<()> {
}
"text-input" => crate::openhuman::text_input::cli::run_text_input_command(&args[1..]),
"tree-summarizer" => {
crate::openhuman::memory_tree::summarizer::cli::run_tree_summarizer_command(&args[1..])
crate::openhuman::memory_tree::tree_runtime::cli::run_tree_summarizer_command(
&args[1..],
)
}
"memory" => crate::core::memory_cli::run_memory_command(&args[1..]),
"agent" => {
+1 -1
View File
@@ -1558,7 +1558,7 @@ fn register_domain_subscribers(
);
}
crate::openhuman::memory_tree::jobs::start(config.clone());
crate::openhuman::memory::jobs::start(config.clone());
// Restart requests go through a subscriber so every trigger path shares
// the same respawn logic.
+162 -65
View File
@@ -18,23 +18,18 @@
use crate::openhuman::agent::hooks::{PostTurnHook, TurnContext};
use crate::openhuman::config::Config;
use crate::openhuman::memory::store::events::{self, EventRecord, EventType};
use crate::openhuman::memory::store::fts5::{self, EpisodicEntry};
use crate::openhuman::memory::store::profile::{self, FacetType};
use crate::openhuman::memory::store::segments::{
use crate::openhuman::memory::chat::ChatProvider;
use crate::openhuman::memory::ingest_pipeline;
use crate::openhuman::memory::score::embed::{build_embedder_from_config, Embedder};
use crate::openhuman::memory_store::events::{self, EventRecord, EventType};
use crate::openhuman::memory_store::fts5::{self, EpisodicEntry};
use crate::openhuman::memory_store::profile::{self, FacetType};
use crate::openhuman::memory_store::segments::{
self, BoundaryConfig, BoundaryDecision, ConversationSegment,
};
use crate::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage};
use crate::openhuman::memory_tree::chat::{ChatConsumer, ChatProvider};
use crate::openhuman::memory_tree::ingest;
use crate::openhuman::memory_tree::score::embed::{build_embedder_from_config, Embedder};
use crate::openhuman::memory_tree::tree_source::summariser::llm::{
LlmSummariser, LlmSummariserConfig,
};
use crate::openhuman::memory_tree::tree_source::summariser::{
Summariser, SummaryContext, SummaryInput,
};
use crate::openhuman::memory_tree::tree_source::types::TreeKind;
use crate::openhuman::memory_store::trees::types::TreeKind;
use crate::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage};
use crate::openhuman::memory_tree::summarise::{summarise, SummaryContext, SummaryInput};
use async_trait::async_trait;
use parking_lot::Mutex;
use rusqlite::Connection;
@@ -98,10 +93,7 @@ impl ArchivistHook {
pub fn with_config(mut self, config: Config) -> Self {
// Build the LLM chat provider for segment recap.
let chat_provider: Option<Arc<dyn ChatProvider>> =
match crate::openhuman::memory_tree::chat::build_chat_provider(
&config,
ChatConsumer::Summarise,
) {
match crate::openhuman::memory::chat::build_chat_provider(&config) {
Ok(p) => {
tracing::debug!("[archivist] segment recap provider={} registered", p.name());
Some(p)
@@ -207,6 +199,7 @@ impl ArchivistHook {
timestamp: f64,
user_message: &str,
current_episodic_id: i64,
current_seq: Option<u32>,
) -> Option<ConversationSegment> {
let now = Self::now_timestamp();
@@ -241,6 +234,7 @@ impl ArchivistHook {
conn,
&segment.segment_id,
current_episodic_id,
current_seq,
timestamp,
now,
) {
@@ -269,6 +263,7 @@ impl ArchivistHook {
session_id,
"global",
current_episodic_id,
current_seq,
timestamp,
now,
) {
@@ -293,6 +288,7 @@ impl ArchivistHook {
session_id,
"global",
current_episodic_id,
current_seq,
timestamp,
now,
) {
@@ -318,8 +314,10 @@ impl ArchivistHook {
session_id: &str,
now: f64,
) {
// Gather the conversation text for this segment from episodic entries.
let entries = fts5::episodic_session_entries(conn, session_id).unwrap_or_default();
// Gather the conversation text for this segment. Prefer the
// md-backed memory_archivist read when config is available; fall
// back to FTS5 in test paths or when config isn't wired.
let entries = self.read_session_entries(conn, session_id);
// Filter entries that fall within the segment's time window.
// Use <= for end_timestamp (entries at the boundary are part of this
@@ -583,6 +581,56 @@ impl PostTurnHook for ArchivistHook {
tracing::debug!("[archivist] episodic rows written: session={session_id}");
// Dual-write into memory_archivist::store (md-backed) so we can
// validate the FTS5 → md migration before flipping the read side.
// Best-effort: a write failure here must not break the turn. The
// user turn's assigned seq is captured into `current_seq` so the
// segment ops can store it alongside the FTS5 episodic id.
let mut current_seq: Option<u32> = None;
if let Some(cfg) = self.config.as_ref() {
let ts_ms = (timestamp * 1000.0) as i64;
let user_turn = crate::openhuman::memory_archivist::ArchivedTurn {
session_id: session_id.to_string(),
seq: 0, // assigned by record_turn
timestamp_ms: ts_ms,
role: "user".to_string(),
content: ctx.user_message.clone(),
lesson: None,
tool_calls_json: None,
cost_microdollars: 0,
};
match crate::openhuman::memory_archivist::store::record_turn(cfg, user_turn) {
Ok(stored) => current_seq = Some(stored.seq),
Err(e) => {
tracing::warn!("[archivist] memory_archivist user dual-write failed: {e}");
}
}
// Assistant turn carries the tool_calls_json + lesson the FTS5
// insert just wrote. Re-derive locally so we don't depend on
// FTS5 having returned.
let assistant_lesson = extract_lesson_from_tools(&ctx.tool_calls);
let assistant_tool_calls = if ctx.tool_calls.is_empty() {
None
} else {
Some(serde_json::to_string(&ctx.tool_calls).unwrap_or_default())
};
let assistant_turn = crate::openhuman::memory_archivist::ArchivedTurn {
session_id: session_id.to_string(),
seq: 0,
timestamp_ms: ts_ms + 1,
role: "assistant".to_string(),
content: ctx.assistant_response.clone(),
lesson: assistant_lesson,
tool_calls_json: assistant_tool_calls,
cost_microdollars: 0,
};
if let Err(e) =
crate::openhuman::memory_archivist::store::record_turn(cfg, assistant_turn)
{
tracing::warn!("[archivist] memory_archivist assistant dual-write failed: {e}");
}
}
// Manage conversation segmentation (sync boundary detection + SQLite
// operations). Returns the just-closed segment when a boundary fired.
let closed_segment = self.manage_segment_sync(
@@ -591,6 +639,7 @@ impl PostTurnHook for ArchivistHook {
timestamp,
&ctx.user_message,
current_episodic_id,
current_seq,
);
// Run async recap + embed + segment-tree ingest on the closed segment
@@ -607,6 +656,47 @@ impl PostTurnHook for ArchivistHook {
}
impl ArchivistHook {
/// Read every entry recorded for `session_id`, preferring the
/// md-backed `memory_archivist::store` when `self.config` is set and
/// falling back to the legacy FTS5 episodic table otherwise.
///
/// Returns `EpisodicEntry` so the existing call sites (segment
/// gathering, recap rendering, tree push) keep their shape unchanged
/// during the FTS5 retirement migration.
fn read_session_entries(
&self,
conn: &Arc<Mutex<Connection>>,
session_id: &str,
) -> Vec<EpisodicEntry> {
if let Some(cfg) = self.config.as_ref() {
match crate::openhuman::memory_archivist::store::session_entries(cfg, session_id) {
Ok(turns) => {
return turns
.into_iter()
.map(|t| EpisodicEntry {
id: None,
session_id: t.session_id,
// ArchivedTurn stores epoch-ms; EpisodicEntry
// takes epoch-seconds as f64.
timestamp: (t.timestamp_ms as f64) / 1000.0,
role: t.role,
content: t.content,
lesson: t.lesson,
tool_calls_json: t.tool_calls_json,
cost_microdollars: t.cost_microdollars,
})
.collect();
}
Err(e) => {
tracing::warn!(
"[archivist] memory_archivist read failed (falling back to FTS5): {e}"
);
}
}
}
fts5::episodic_session_entries(conn, session_id).unwrap_or_default()
}
/// Shared summarize helper — the **single LLM summarizer** used by both
/// the finalize path (`on_segment_closed`) and the rolling-recap path
/// (`rolling_segment_recap`).
@@ -650,7 +740,7 @@ impl ArchivistHook {
.iter()
.filter(|e| !e.content.trim().is_empty())
.map(|e| {
use crate::openhuman::memory_tree::types::approx_token_count;
use crate::openhuman::memory_store::chunks::types::approx_token_count;
let content = e.content.clone();
let token_count = approx_token_count(&content);
let ts = chrono::DateTime::from_timestamp(e.timestamp as i64, 0)
@@ -678,53 +768,60 @@ impl ArchivistHook {
let first = entries.first().map(|e| e.content.as_str()).unwrap_or("");
let last = entries.last().map(|e| e.content.as_str()).unwrap_or(first);
if let Some(ref provider) = self.chat_provider {
let cfg = LlmSummariserConfig {
model: provider.name().to_string(),
structured_facet_extraction: false,
output_language: self
.config
.as_ref()
.and_then(|cfg| cfg.output_language.clone()),
};
let summariser = LlmSummariser::new(cfg, Arc::clone(provider));
tracing::debug!(
"[archivist] summarize_entries: LLM recap segment={segment_id} \
provider={} entries={}",
provider.name(),
entries.len()
);
match summariser.summarise(&corpus_inputs, &summary_ctx).await {
Ok(output) if !output.content.is_empty() => {
tracing::debug!(
"[archivist] summarize_entries: LLM recap ok segment={segment_id} \
chars={}",
output.content.len()
);
(output.content, true)
}
Ok(_) => {
tracing::debug!(
"[archivist] summarize_entries: LLM returned empty — \
heuristic fallback segment={segment_id}"
);
(segments::fallback_summary(first, last, turn_count), false)
}
Err(e) => {
tracing::warn!(
"[archivist] summarize_entries: LLM recap failed (non-fatal) \
segment={segment_id}: {e} — heuristic fallback"
);
(segments::fallback_summary(first, last, turn_count), false)
if self.chat_provider.is_some() {
if let Some(ref config) = self.config {
tracing::debug!(
"[archivist] summarize_entries: LLM recap segment={segment_id} entries={}",
entries.len()
);
#[cfg(test)]
let summary_result = if let Some(provider) = self.chat_provider.as_ref() {
crate::openhuman::memory::chat::test_override::with_provider(
Arc::clone(provider),
summarise(config, &corpus_inputs, &summary_ctx),
)
.await
} else {
summarise(config, &corpus_inputs, &summary_ctx).await
};
#[cfg(not(test))]
let summary_result = summarise(config, &corpus_inputs, &summary_ctx).await;
match summary_result {
Ok(output) if !output.content.is_empty() => {
tracing::debug!(
"[archivist] summarize_entries: LLM recap ok segment={segment_id} \
chars={}",
output.content.len()
);
return (output.content, true);
}
Ok(_) => {
tracing::debug!(
"[archivist] summarize_entries: LLM returned empty — \
heuristic fallback segment={segment_id}"
);
}
Err(e) => {
tracing::warn!(
"[archivist] summarize_entries: LLM recap failed (non-fatal) \
segment={segment_id}: {e} — heuristic fallback"
);
}
}
} else {
tracing::debug!(
"[archivist] summarize_entries: no config — \
heuristic fallback segment={segment_id}"
);
}
} else {
tracing::debug!(
"[archivist] summarize_entries: no chat provider — \
heuristic fallback segment={segment_id}"
);
(segments::fallback_summary(first, last, turn_count), false)
}
(segments::fallback_summary(first, last, turn_count), false)
}
/// Produce a rolling recap of the **currently-open** segment for
@@ -782,7 +879,7 @@ impl ArchivistHook {
};
// Gather the episodic entries for this session so far.
let all_entries = fts5::episodic_session_entries(conn, session_id).unwrap_or_default();
let all_entries = self.read_session_entries(conn, session_id);
// Keep only entries within the open segment's time window (start →
// now, inclusive). An open segment has `end_timestamp = None`.
@@ -867,7 +964,7 @@ impl ArchivistHook {
async fn pipe_segment_to_tree(
&self,
config: &Config,
segment: &crate::openhuman::memory::store::segments::ConversationSegment,
segment: &crate::openhuman::memory_store::segments::ConversationSegment,
session_id: &str,
entries: &[&fts5::EpisodicEntry],
) {
@@ -945,7 +1042,7 @@ impl ArchivistHook {
segment={segment_id} ep_span={start_ep}-{end_ep} provenance={provenance}"
);
match ingest::ingest_chat(config, source_id, owner, tags, batch).await {
match ingest_pipeline::ingest_chat(config, source_id, owner, tags, batch).await {
Ok(result) => {
tracing::debug!(
"[archivist] tree ingest ok: source_id={source_id} \
@@ -1118,7 +1215,7 @@ impl ArchivistHook {
conn: Some(conn),
enabled: true,
boundary_config: BoundaryConfig::default(),
config: None,
config: Some(Config::default()),
chat_provider: Some(chat_provider),
embedder: Some(embedder),
}
@@ -1,7 +1,7 @@
use super::*;
use crate::openhuman::agent::hooks::{ToolCallRecord, TurnContext};
use crate::openhuman::memory::store::{events as ev, fts5, segments as seg};
use crate::openhuman::memory_tree::chat::ChatPrompt;
use crate::openhuman::memory::chat::ChatPrompt;
use crate::openhuman::memory_store::{events as ev, fts5, segments as seg};
fn setup_conn() -> Arc<Mutex<Connection>> {
let conn = Connection::open_in_memory().unwrap();
@@ -343,7 +343,7 @@ async fn phase0_episodic_rows_and_segment_without_learning_enabled() {
struct StubChatProvider;
#[async_trait::async_trait]
impl crate::openhuman::memory_tree::chat::ChatProvider for StubChatProvider {
impl crate::openhuman::memory::chat::ChatProvider for StubChatProvider {
fn name(&self) -> &str {
"stub:test"
}
@@ -361,7 +361,7 @@ impl crate::openhuman::memory_tree::chat::ChatProvider for StubChatProvider {
struct StubEmbedder;
#[async_trait::async_trait]
impl crate::openhuman::memory_tree::score::embed::Embedder for StubEmbedder {
impl crate::openhuman::memory::score::embed::Embedder for StubEmbedder {
fn name(&self) -> &'static str {
"stub-embedder-v1"
}
@@ -546,7 +546,7 @@ async fn phase1_flush_open_segment_finalizes_trailing_segment() {
// g) flush_open_segment also triggers tree ingest.
use crate::openhuman::config::Config;
use crate::openhuman::memory_tree::store::{count_chunks, list_chunks, ListChunksQuery};
use crate::openhuman::memory_store::chunks::store::{count_chunks, list_chunks, ListChunksQuery};
use tempfile::TempDir;
/// Build a Config that points at a temp workspace, suitable for tree-ingest tests.
@@ -736,7 +736,7 @@ async fn phase2_provenance_stamped_on_leaf_and_source_id_is_constant() {
.iter()
.find(|s| {
s.session_id == session
&& s.status != crate::openhuman::memory::store::segments::SegmentStatus::Open
&& s.status != crate::openhuman::memory_store::segments::SegmentStatus::Open
})
.expect("Expected a closed segment after flush");
@@ -836,7 +836,9 @@ async fn phase2_ingested_content_is_raw_prose_not_recap() {
}
// The raw prose text MUST appear in at least one chunk.
let has_user_prose = chunks.iter().any(|c| c.content.contains("lifetimes"));
let has_user_prose = chunks
.iter()
.any(|c| c.content.to_ascii_lowercase().contains("lifetimes"));
assert!(
has_user_prose,
"Expected at least one chunk body to contain raw prose from the turn \
@@ -309,7 +309,7 @@ impl PayloadSummarizer for SubagentPayloadSummarizer {
}
/// Rough token estimate: ~4 characters per token. Mirrors
/// [`crate::openhuman::memory_tree::summarizer::types::estimate_tokens`] but
/// [`crate::openhuman::memory_tree::tree_runtime::types::estimate_tokens`] but
/// returns `usize` (not `u32`) and lives here to avoid a cross-module
/// dependency from the agent harness on the tree summarizer.
fn estimate_tokens(text: &str) -> usize {
+2 -2
View File
@@ -2150,7 +2150,7 @@ impl Agent {
}
/// Wrapper around
/// [`crate::openhuman::memory_tree::summarizer::store::collect_root_summaries_with_caps`]
/// [`crate::openhuman::memory_tree::tree_runtime::store::collect_root_summaries_with_caps`]
/// that takes user-resolved per-namespace and total caps. The actual
/// limits are derived from the active
/// [`crate::openhuman::config::schema::agent::MemoryContextWindow`]
@@ -2160,7 +2160,7 @@ fn collect_tree_root_summaries(
per_namespace_cap: usize,
total_cap: usize,
) -> Vec<(String, String)> {
crate::openhuman::memory_tree::summarizer::store::collect_root_summaries_with_caps(
crate::openhuman::memory_tree::tree_runtime::store::collect_root_summaries_with_caps(
workspace_dir,
per_namespace_cap,
total_cap,
@@ -29,7 +29,7 @@ use std::sync::Mutex as StdMutex;
/// cache instead of being pushed into history raw. Token count is
/// estimated at ~4 chars/token (mirrors
/// `crate::openhuman::agent::harness::payload_summarizer` and
/// `crate::openhuman::memory_tree::summarizer::types::estimate_tokens`).
/// `crate::openhuman::memory_tree::tree_runtime::types::estimate_tokens`).
///
/// Set at `50_000` so the clean Gmail / Notion envelopes emitted by provider
/// post-processing fit through unchanged for normal workloads — only
+1 -1
View File
@@ -21,7 +21,7 @@
//! concatenate without branching.
use crate::openhuman::config::Config;
use crate::openhuman::memory_tree::retrieval::query_global;
use crate::openhuman::memory::retrieval::query_global;
/// Default lookback window for the eager digest. Mirrors the language in
/// the orchestrator prompt ("7-day digest pre-loaded into session context").
+1 -1
View File
@@ -25,7 +25,7 @@ use chrono::{DateTime, Utc};
use rusqlite::{params, types::Type, Connection};
use crate::openhuman::config::Config;
use crate::openhuman::memory::safety::sanitize_text;
use crate::openhuman::memory_store::safety::sanitize_text;
use super::types::{ApprovalAuditEntry, ApprovalDecision, ExecutionOutcome, PendingApproval};
+1 -1
View File
@@ -587,7 +587,7 @@ pub async fn start_channels(config: Config) -> Result<()> {
};
// Register the tree summarizer event subscriber for observability logging.
let _tree_summarizer_handle = bus.subscribe(Arc::new(
crate::openhuman::memory_tree::summarizer::bus::TreeSummarizerEventSubscriber::new(),
crate::openhuman::memory_tree::tree_runtime::bus::TreeSummarizerEventSubscriber::new(),
));
let max_in_flight_messages = compute_max_in_flight_messages(channels.len());
+2 -2
View File
@@ -576,8 +576,8 @@ async fn composio_execute_via_mock_propagates_backend_error() {
#[tokio::test]
async fn composio_sync_gmail_via_mock_archives_raw_email_and_updates_outcome() {
use crate::openhuman::config::TEST_ENV_LOCK;
use crate::openhuman::memory_tree::content_store::raw::{raw_rel_path, RawKind};
use crate::openhuman::memory_tree::rpc::{list_chunks_rpc, ListChunksRequest};
use crate::openhuman::memory::tree_rpc::{list_chunks_rpc, ListChunksRequest};
use crate::openhuman::memory_store::content::raw::{raw_rel_path, RawKind};
let _cache_guard = CACHE_TEST_GUARD.lock().unwrap_or_else(|e| e.into_inner());
let _env_guard = TEST_ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
@@ -21,14 +21,14 @@ use anyhow::Result;
use serde_json::Value;
use crate::openhuman::config::Config;
use crate::openhuman::memory_tree::canonicalize::email::{EmailMessage, EmailThread};
use crate::openhuman::memory_tree::canonicalize::email_clean::{extract_email, parse_message_date};
use crate::openhuman::memory_tree::content_store::raw::{
use crate::openhuman::memory::ingest_pipeline::{ingest_email, IngestResult};
use crate::openhuman::memory::util::redact::redact;
use crate::openhuman::memory_store::chunks::store::{set_chunk_raw_refs, RawRef};
use crate::openhuman::memory_store::content::raw::{
self as raw_store, raw_rel_path, slug_account_email, RawItem, RawKind,
};
use crate::openhuman::memory_tree::ingest::{ingest_email, IngestResult};
use crate::openhuman::memory_tree::store::{set_chunk_raw_refs, RawRef};
use crate::openhuman::memory_tree::util::redact::redact;
use crate::openhuman::memory_sync::canonicalize::email::{EmailMessage, EmailThread};
use crate::openhuman::memory_sync::canonicalize::email_clean::{extract_email, parse_message_date};
/// Provider name embedded in the canonical email-thread header. Matches
/// the value `memory::tree::retrieval::source::PLATFORM_KINDS` expects.
+3 -3
View File
@@ -21,7 +21,7 @@ use super::ProviderUserProfile;
use crate::openhuman::learning::candidate::{
self as learning_candidate, CueFamily, EvidenceRef, FacetClass, LearningCandidate,
};
use crate::openhuman::memory::store::profile::{self, FacetType};
use crate::openhuman::memory_store::profile::{self, FacetType};
use rusqlite::params;
use serde_json::Value;
use std::collections::BTreeMap;
@@ -32,7 +32,7 @@ use std::collections::BTreeMap;
/// Shape of an identifier persisted against a connection. Mirrors the
/// matching dimensions of the memory tree's
/// `crate::openhuman::memory_tree::score::extract::EntityKind` so the
/// `crate::openhuman::memory::score::extract::EntityKind` so the
/// self-check is a direct `(toolkit, kind, value)` lookup.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum IdentityKind {
@@ -534,7 +534,7 @@ fn now_secs() -> f64 {
#[cfg(test)]
mod tests {
use super::*;
use crate::openhuman::memory::store::profile::{profile_load_all, PROFILE_INIT_SQL};
use crate::openhuman::memory_store::profile::{profile_load_all, PROFILE_INIT_SQL};
use parking_lot::Mutex;
use rusqlite::Connection;
use serde_json::json;
@@ -2,7 +2,7 @@
//!
//! Owns the conversion from a page of [`SlackMessage`]s (post-processed
//! and enriched by [`super::sync`]) into per-channel [`ChatBatch`]es and
//! drives [`memory_tree::ingest::ingest_chat`] per message.
//! drives [`memory::ingest_pipeline::ingest_chat`] per message.
//!
//! ## Source-id scope
//!
@@ -30,16 +30,16 @@ use anyhow::Result;
use super::types::SlackMessage;
use crate::openhuman::config::Config;
use crate::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage};
use crate::openhuman::memory_tree::content_store::raw::{
use crate::openhuman::memory::ingest_pipeline::ingest_chat;
use crate::openhuman::memory::util::redact::redact;
use crate::openhuman::memory_store::chunks::store::{set_chunk_raw_refs, RawRef};
use crate::openhuman::memory_store::content::raw::{
self as raw_store, raw_rel_path, RawItem, RawKind,
};
use crate::openhuman::memory_tree::ingest::ingest_chat;
use crate::openhuman::memory_tree::store::{set_chunk_raw_refs, RawRef};
use crate::openhuman::memory_tree::util::redact::redact;
use crate::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage};
/// Platform identifier embedded in the canonical chat transcript header.
/// Matches the value `memory_tree::retrieval::source::PLATFORM_KINDS` expects.
/// Matches the value `memory::retrieval::source::PLATFORM_KINDS` expects.
pub const SLACK_PLATFORM: &str = "slack";
/// Tags attached to every Slack-ingested chunk. Stable list — retrieval
+2 -2
View File
@@ -588,7 +588,7 @@ pub async fn apply_model_settings(
// so a UI embedder switch recovers prior memory under the new
// signature. Coverage-gated + non-fatal: if the active signature did
// not actually change, this enqueues nothing.
crate::openhuman::memory_tree::jobs::ensure_reembed_backfill(config);
crate::openhuman::memory::jobs::ensure_reembed_backfill(config);
let snapshot = snapshot_config_json(config)?;
Ok(RpcOutcome::new(
snapshot,
@@ -638,7 +638,7 @@ pub async fn apply_memory_settings(
// dark. Idempotent + non-fatal (covered space enqueues nothing; errors
// are logged, never fail the settings save). §7's migration is
// one-shot so it does not cover a later switch — this does.
crate::openhuman::memory_tree::jobs::ensure_reembed_backfill(config);
crate::openhuman::memory::jobs::ensure_reembed_backfill(config);
let snapshot = snapshot_config_json(config)?;
Ok(RpcOutcome::new(
snapshot,
@@ -15,8 +15,8 @@ use crate::openhuman::agent::harness::archivist::ArchivistHook;
use crate::openhuman::agent::hooks::{PostTurnHook as _, TurnContext};
use crate::openhuman::context::summarizer::{Summarizer, SummaryStats};
use crate::openhuman::inference::provider::{ChatMessage, ConversationMessage};
use crate::openhuman::memory::store::{fts5, segments as seg};
use crate::openhuman::memory_tree::chat::ChatPrompt;
use crate::openhuman::memory::chat::ChatPrompt;
use crate::openhuman::memory_store::{fts5, segments as seg};
use anyhow::Result;
use async_trait::async_trait;
use parking_lot::Mutex;
@@ -29,9 +29,9 @@ fn setup_conn() -> Arc<Mutex<Connection>> {
let conn = Connection::open_in_memory().unwrap();
conn.execute_batch(fts5::EPISODIC_INIT_SQL).unwrap();
conn.execute_batch(seg::SEGMENTS_INIT_SQL).unwrap();
conn.execute_batch(crate::openhuman::memory::store::events::EVENTS_INIT_SQL)
conn.execute_batch(crate::openhuman::memory_store::events::EVENTS_INIT_SQL)
.unwrap();
conn.execute_batch(crate::openhuman::memory::store::profile::PROFILE_INIT_SQL)
conn.execute_batch(crate::openhuman::memory_store::profile::PROFILE_INIT_SQL)
.unwrap();
Arc::new(Mutex::new(conn))
}
@@ -40,7 +40,7 @@ fn setup_conn() -> Arc<Mutex<Connection>> {
struct StubChatProvider;
#[async_trait]
impl crate::openhuman::memory_tree::chat::ChatProvider for StubChatProvider {
impl crate::openhuman::memory::chat::ChatProvider for StubChatProvider {
fn name(&self) -> &str {
"stub:test"
}
@@ -56,7 +56,7 @@ impl crate::openhuman::memory_tree::chat::ChatProvider for StubChatProvider {
struct FailingChatProvider;
#[async_trait]
impl crate::openhuman::memory_tree::chat::ChatProvider for FailingChatProvider {
impl crate::openhuman::memory::chat::ChatProvider for FailingChatProvider {
fn name(&self) -> &str {
"stub:failing"
}
@@ -72,7 +72,7 @@ impl crate::openhuman::memory_tree::chat::ChatProvider for FailingChatProvider {
struct StubEmbedder;
#[async_trait]
impl crate::openhuman::memory_tree::score::embed::Embedder for StubEmbedder {
impl crate::openhuman::memory::score::embed::Embedder for StubEmbedder {
fn name(&self) -> &'static str {
"stub-embedder-v1"
}
@@ -439,13 +439,13 @@ async fn failing_provider_yields_inert_clipped_recap_used_as_compaction() {
.await
.unwrap();
// Provider present but failing → LlmSummariser inert fallback → real
// clipped content (not the bookend stub) → Some, treated as usable.
// Provider present but failing → summarize_entries returns a non-LLM
// fallback, which rolling_segment_recap must treat as unavailable for
// live compaction.
let recap = hook.rolling_segment_recap(session).await;
assert!(
recap.is_some(),
"Inert clipped-content recap (real text) is acceptable compaction \
text — must be Some, not None"
recap.is_none(),
"Non-LLM fallback recap text must not be used as live compaction input"
);
let inner = RecordingSummarizer::new();
@@ -464,9 +464,9 @@ async fn failing_provider_yields_inert_clipped_recap_used_as_compaction() {
assert_eq!(
inner.call_count(),
0,
"Inner summarizer must NOT run when an inert clipped-content recap \
is available (real content, better than no compaction)"
1,
"Inner summarizer must run when rolling recap is unavailable after \
provider failure"
);
}
+1 -1
View File
@@ -817,7 +817,7 @@ fn check_memory_tree_db(config: &Config, items: &mut Vec<DiagnosticItem>) {
}
// ── Probe connection ─────────────────────────────────────────────
match crate::openhuman::memory_tree::store::with_connection(config, |conn| {
match crate::openhuman::memory_store::chunks::store::with_connection(config, |conn| {
let n: i64 = conn.query_row("SELECT COUNT(*) FROM mem_tree_chunks", [], |r| r.get(0))?;
Ok(n)
}) {
+1 -1
View File
@@ -89,7 +89,7 @@ fn check_memory_tree_db_ok_when_accessible() {
let cfg = test_config_in(&tmp);
// Trigger DB creation.
crate::openhuman::memory_tree::store::with_connection(&cfg, |_conn| Ok(()))
crate::openhuman::memory_store::chunks::store::with_connection(&cfg, |_conn| Ok(()))
.expect("DB init must succeed");
let mut items = vec![];
+6 -2
View File
@@ -18,8 +18,13 @@ pub mod ollama;
pub mod openai;
mod provider_trait;
pub mod rate_limit;
pub mod store;
// VectorStore has moved to memory_store::vectors; re-exported for callers.
pub use crate::openhuman::memory_store::vectors::store;
pub use crate::openhuman::memory_store::vectors::{
bytes_to_vec, cosine_similarity, vec_to_bytes, SearchResult, VectorStore,
};
pub use cloud::{
OpenHumanCloudEmbedding, DEFAULT_CLOUD_EMBEDDING_DIMENSIONS, DEFAULT_CLOUD_EMBEDDING_MODEL,
};
@@ -30,7 +35,6 @@ pub use noop::NoopEmbedding;
pub use ollama::{OllamaEmbedding, DEFAULT_OLLAMA_DIMENSIONS, DEFAULT_OLLAMA_MODEL};
pub use openai::OpenAiEmbedding;
pub use provider_trait::{format_embedding_signature, EmbeddingProvider};
pub use store::{bytes_to_vec, cosine_similarity, vec_to_bytes, SearchResult, VectorStore};
#[cfg(test)]
mod tests {
@@ -2,7 +2,7 @@
//!
//! The memory tree's embedder (`bge-m3`) is requested with
//! `num_ctx = 8192` (see
//! [`crate::openhuman::memory_tree::score::embed::ollama::EMBED_NUM_CTX`])
//! [`crate::openhuman::memory::score::embed::ollama::EMBED_NUM_CTX`])
//! and the summariser hard-caps its output to fit that 8192-token embed
//! ceiling. A local model whose native context window is below this floor
//! silently truncates chunks/summaries and corrupts recall, so we refuse
@@ -21,7 +21,7 @@ use serde::Serialize;
/// time. Changing the embedder's context request automatically moves the
/// acceptance floor with it.
pub const MIN_CONTEXT_TOKENS: u64 =
crate::openhuman::memory_tree::score::embed::ollama::EMBED_NUM_CTX as u64;
crate::openhuman::memory::score::embed::ollama::EMBED_NUM_CTX as u64;
/// Verdict for a single model's context window against
/// [`MIN_CONTEXT_TOKENS`]. Serialized into the diagnostics payload so the
@@ -79,7 +79,7 @@ mod tests {
// requests; this guards against the two drifting apart.
assert_eq!(
MIN_CONTEXT_TOKENS,
crate::openhuman::memory_tree::score::embed::ollama::EMBED_NUM_CTX as u64
crate::openhuman::memory::score::embed::ollama::EMBED_NUM_CTX as u64
);
assert_eq!(MIN_CONTEXT_TOKENS, 8_192);
}
+2 -2
View File
@@ -9,7 +9,7 @@ use rusqlite::Connection;
use std::sync::Arc;
use crate::openhuman::learning::candidate::FacetClass;
use crate::openhuman::memory::store::profile::{self, ProfileFacet, UserState};
use crate::openhuman::memory_store::profile::{self, ProfileFacet, UserState};
/// Thin wrapper around the `user_profile` table.
///
@@ -115,7 +115,7 @@ pub fn class_prefix(class: FacetClass) -> &'static str {
// ── Facet state enum re-export (convenience for callers of this module) ───────
pub use crate::openhuman::memory::store::profile::{
pub use crate::openhuman::memory_store::profile::{
FacetState as CacheFacetState, UserState as CacheUserState,
};
+1 -1
View File
@@ -6,7 +6,7 @@ use std::sync::Arc;
use super::*;
use crate::openhuman::learning::candidate::{EvidenceRef, FacetClass};
use crate::openhuman::memory::store::profile::{
use crate::openhuman::memory_store::profile::{
FacetState, FacetType, ProfileFacet, UserState, PROFILE_INIT_SQL,
};
@@ -680,15 +680,15 @@ pub async fn scrape_linkedin_profile(
}
/// Build a local memory client for profile persistence.
fn build_memory_client() -> anyhow::Result<crate::openhuman::memory::store::MemoryClient> {
crate::openhuman::memory::store::MemoryClient::new_local()
fn build_memory_client() -> anyhow::Result<crate::openhuman::memory_store::MemoryClient> {
crate::openhuman::memory_store::MemoryClient::new_local()
.map_err(|e| anyhow::anyhow!("memory client unavailable: {e}"))
}
/// Persist the full scraped LinkedIn profile to the user-profile memory
/// namespace so the agent has rich context about the user.
async fn persist_linkedin_profile(
memory: &crate::openhuman::memory::store::MemoryClient,
memory: &crate::openhuman::memory_store::MemoryClient,
url: &str,
data: &serde_json::Value,
) -> anyhow::Result<()> {
@@ -720,7 +720,7 @@ async fn persist_linkedin_profile(
/// Fallback: persist just the LinkedIn URL when the full scrape fails.
async fn persist_linkedin_url_only(
memory: &crate::openhuman::memory::store::MemoryClient,
memory: &crate::openhuman::memory_store::MemoryClient,
url: &str,
) -> anyhow::Result<()> {
memory
@@ -42,7 +42,7 @@ use async_trait::async_trait;
use crate::core::event_bus::{subscribe_global, DomainEvent, EventHandler, SubscriptionHandle};
use crate::openhuman::composio::providers::profile_md::replace_managed_block;
use crate::openhuman::learning::cache::FacetCache;
use crate::openhuman::memory::store::profile::UserState;
use crate::openhuman::memory_store::profile::UserState;
// ── Class → block metadata ────────────────────────────────────────────────────
@@ -215,7 +215,7 @@ impl EventHandler for RendererSubscriber {
mod tests {
use super::*;
use crate::openhuman::composio::providers::profile_md::{block_end, block_start};
use crate::openhuman::memory::store::profile::{
use crate::openhuman::memory_store::profile::{
FacetState, FacetType, ProfileFacet, UserState, PROFILE_INIT_SQL,
};
use parking_lot::Mutex;
+4 -4
View File
@@ -163,7 +163,7 @@ pub fn load_learned_from_cache(
// Group by class prefix (portion before the first '/'), then sort within
// each class by stability descending, then by key alphabetically.
use crate::openhuman::memory::store::profile::ProfileFacet;
use crate::openhuman::memory_store::profile::ProfileFacet;
use std::collections::BTreeMap;
let mut by_class: BTreeMap<String, Vec<usize>> = BTreeMap::new();
@@ -197,7 +197,7 @@ pub fn load_learned_from_cache(
// agent can parse the source. Goal class keeps value-only (full
// sentence, no key prefix). Pinned entries get a trailing suffix.
let pinned =
if f.user_state == crate::openhuman::memory::store::profile::UserState::Pinned {
if f.user_state == crate::openhuman::memory_store::profile::UserState::Pinned {
" *(pinned)*"
} else {
""
@@ -376,7 +376,7 @@ mod tests {
#[test]
fn load_learned_from_cache_formats_active_facets() {
use crate::openhuman::learning::cache::FacetCache;
use crate::openhuman::memory::store::profile::{
use crate::openhuman::memory_store::profile::{
FacetState, FacetType, ProfileFacet, UserState, PROFILE_INIT_SQL,
};
use parking_lot::Mutex;
@@ -456,7 +456,7 @@ mod tests {
#[test]
fn load_learned_from_cache_empty_when_no_active_facets() {
use crate::openhuman::learning::cache::FacetCache;
use crate::openhuman::memory::store::profile::PROFILE_INIT_SQL;
use crate::openhuman::memory_store::profile::PROFILE_INIT_SQL;
use parking_lot::Mutex;
use rusqlite::Connection;
@@ -8,7 +8,7 @@ use std::sync::Arc;
use super::load_learned_from_cache;
use crate::openhuman::learning::cache::FacetCache;
use crate::openhuman::memory::store::profile::{
use crate::openhuman::memory_store::profile::{
FacetState, FacetType, ProfileFacet, UserState, PROFILE_INIT_SQL,
};
+8 -8
View File
@@ -644,7 +644,7 @@ fn handle_rebuild_cache(_params: Map<String, Value>) -> ControllerFuture {
fn handle_cache_stats(_params: Map<String, Value>) -> ControllerFuture {
Box::pin(async move {
use crate::openhuman::learning::cache::FacetCache;
use crate::openhuman::memory::store::profile::FacetState;
use crate::openhuman::memory_store::profile::FacetState;
tracing::debug!("[learning.cache_stats] cache stats requested via RPC");
@@ -723,7 +723,7 @@ fn full_key(class_str: &str, key_suffix: &str) -> String {
}
/// Serialize a [`ProfileFacet`] to a serde_json [`Value`] for RPC output.
fn facet_to_json(f: &crate::openhuman::memory::store::profile::ProfileFacet) -> serde_json::Value {
fn facet_to_json(f: &crate::openhuman::memory_store::profile::ProfileFacet) -> serde_json::Value {
serde_json::json!({
"key": f.key,
"value": f.value,
@@ -742,7 +742,7 @@ fn facet_to_json(f: &crate::openhuman::memory::store::profile::ProfileFacet) ->
fn handle_list_facets(params: Map<String, Value>) -> ControllerFuture {
Box::pin(async move {
use crate::openhuman::memory::store::profile::FacetState;
use crate::openhuman::memory_store::profile::FacetState;
tracing::debug!("[learning.list_facets] called");
@@ -822,7 +822,7 @@ fn handle_get_facet(params: Map<String, Value>) -> ControllerFuture {
fn handle_update_facet(params: Map<String, Value>) -> ControllerFuture {
Box::pin(async move {
use crate::openhuman::memory::store::profile::UserState;
use crate::openhuman::memory_store::profile::UserState;
let class_str = params
.get("class")
@@ -875,7 +875,7 @@ fn handle_update_facet(params: Map<String, Value>) -> ControllerFuture {
fn handle_pin_facet(params: Map<String, Value>) -> ControllerFuture {
Box::pin(async move {
use crate::openhuman::memory::store::profile::UserState;
use crate::openhuman::memory_store::profile::UserState;
let class_str = params
.get("class")
@@ -915,7 +915,7 @@ fn handle_pin_facet(params: Map<String, Value>) -> ControllerFuture {
fn handle_unpin_facet(params: Map<String, Value>) -> ControllerFuture {
Box::pin(async move {
use crate::openhuman::memory::store::profile::UserState;
use crate::openhuman::memory_store::profile::UserState;
let class_str = params
.get("class")
@@ -955,7 +955,7 @@ fn handle_unpin_facet(params: Map<String, Value>) -> ControllerFuture {
fn handle_forget_facet(params: Map<String, Value>) -> ControllerFuture {
Box::pin(async move {
use crate::openhuman::memory::store::profile::{FacetState, UserState};
use crate::openhuman::memory_store::profile::{FacetState, UserState};
let class_str = params
.get("class")
@@ -1003,7 +1003,7 @@ fn handle_forget_facet(params: Map<String, Value>) -> ControllerFuture {
fn handle_reset_cache(_params: Map<String, Value>) -> ControllerFuture {
Box::pin(async move {
use crate::openhuman::memory::store::profile::UserState;
use crate::openhuman::memory_store::profile::UserState;
tracing::debug!("[learning.reset_cache] called");
+3 -3
View File
@@ -37,7 +37,7 @@ use crate::core::event_bus;
use crate::core::event_bus::DomainEvent;
use crate::openhuman::learning::cache::FacetCache;
use crate::openhuman::learning::candidate::{self, CueFamily, FacetClass, LearningCandidate};
use crate::openhuman::memory::store::profile::{FacetState, FacetType, ProfileFacet, UserState};
use crate::openhuman::memory_store::profile::{FacetState, FacetType, ProfileFacet, UserState};
// ── Thresholds ────────────────────────────────────────────────────────────────
@@ -554,7 +554,7 @@ mod tests {
use crate::openhuman::learning::candidate::{
Buffer, EvidenceRef, FacetClass, LearningCandidate,
};
use crate::openhuman::memory::store::profile::PROFILE_INIT_SQL;
use crate::openhuman::memory_store::profile::PROFILE_INIT_SQL;
use parking_lot::Mutex;
use rusqlite::Connection;
use std::sync::Arc;
@@ -788,7 +788,7 @@ mod tests {
let now = 1_000_000.0;
// Manually insert a Pinned row.
use crate::openhuman::memory::store::profile::{FacetState, FacetType, UserState};
use crate::openhuman::memory_store::profile::{FacetState, FacetType, UserState};
let pinned = ProfileFacet {
facet_id: "f-pinned".into(),
facet_type: FacetType::Preference,
+65 -72
View File
@@ -1,82 +1,75 @@
# Memory
# memory
Persistent knowledge layer. Owns the unified store (SQLite + FTS5 + vector embeddings + graph relations), document ingestion pipelines, namespace + KV operations, conversation history, and retrieval scoring. Does NOT own raw provider embedding APIs (`local_ai/`), agent prompt assembly (`agent/memory_loader.rs`), or per-channel ingestion adapters beyond the bundled Slack importer.
Orchestration layer over the memory stack. Owns:
## Architecture
- **Ingest pipeline** — orchestrates source → canonicalise → chunk →
score → persist → enqueue extract jobs.
- **Job handlers** — background workers that drain the queue (admit,
extract, seal, digest, topic-curate).
- **Scoring** — fast scorers, signal aggregation, score persistence.
- **Tree policy**`tree_global` and `tree_topic` building rules.
- **RPC surface**`read_rpc`, `tree_rpc`, controller schemas for the
memory_* RPC namespace.
- **Recall**`stm_recall`, `retrieval` ranking, query orchestration.
The module is organised in concentric layers — the contract on the
inside, the persistent backend around it, the ingestion + retrieval
pipelines on top, and the per-domain glue at the edge:
Does **not** own any storage primitives — those live in
[`memory_store`](../memory_store/). See that module for raw md, chunks,
entities, trees, vectors, kv, and contacts.
```text
┌──────────────────────────────────────┐
│ conversations/ slack_ingestion/ │ per-domain plumbing
├──────────────────────────────────────┤
│ tree/ (bucket-seal LLD pipeline) │ new retrieval architecture
├──────────────────────────────────────┤
│ ingestion/ (extract chunks) │ document ingestion
├──────────────────────────────────────┤
│ store/ (UnifiedMemory backend) │ SQLite + FTS5 + vectors
├──────────────────────────────────────┤
│ traits.rs (Memory trait) │ contract
└──────────────────────────────────────┘
```
## Sibling memory_* modules
- **`traits.rs`** — `Memory`, `MemoryEntry`, `MemoryCategory`,
`RecallOpts`. The backend-agnostic contract every store implements.
- **`store/`** — `UnifiedMemory` is the production backend (SQLite
with FTS5 for keyword search, vector tables for embeddings, and
graph tables for entity/relation triples) plus the `MemoryClient`
handle used by the rest of the process.
- **`ingestion/`** — chunking + extraction pipeline (entities,
relations, embeddings) and the background `IngestionQueue` worker.
- **`tree/`** — the new bucket-seal retrieval architecture from
`docs/MEMORY_ARCHITECTURE_LLD.md`: `canonicalize` (normalise
inputs), `chunker` and `content_store` (durable chunks),
`score`/`retrieval` (ranking surface),
`tree_source`/`tree_topic`/`tree_global` (the three concentric
trees the LLD calls for), and `jobs` (background seals/summaries).
- **`conversations/`** — workspace-backed JSONL chat thread/message
history. See `conversations/README.md`.
- **`slack_ingestion/`** — Slack provider plumbing (bucketer +
ingest wrapper + RPC). See `slack_ingestion/README.md`.
The memory stack is split across several top-level modules so each has
one job. memory orchestrates and routes between them.
The legacy memory store (`store/` + `ingestion/`) and the new
`tree/` pipeline coexist for now — `tree/` is replacing the older
retrieval surface incrementally and both must remain wired into RPC
until the migration completes.
| Module | Role |
| --- | --- |
| [`memory_store`](../memory_store/) | Storage primitives: raw / chunks / entities / trees / vectors / kv / contacts. SQLite + on-disk md. |
| [`memory_tree`](../memory_tree/) | Generic tree mechanics: bucket-seal, flush, summarise, walk. Kind-agnostic. |
| [`memory_archivist`](../memory_archivist/) | Chat conversation → clip tool-calls → push to tree. |
| [`memory_entities`](../memory_entities/) | Md-backed entity registry (people + orgs + topics + …). Replacing `people/`. |
| [`memory_graph`](../memory_graph/) | Derived co-occurrence edges over the entity index. |
| [`memory_tools`](../memory_tools/) | Tool-scoped rules + agent read/write tools. |
| [`memory_sync`](../memory_sync/) | Composio + workspace + MCP sync pipelines. |
## Public surface
## What lives here
- `pub trait Memory` / `pub struct MemoryEntry` / `pub enum MemoryCategory` / `pub struct RecallOpts``traits.rs:11-100` — backend contract for any memory store.
- `pub struct UnifiedMemory``store/unified/` (re-exported `store/mod.rs:40`) — primary SQLite + FTS5 + vector implementation.
- `pub struct MemoryClient` / `pub struct MemoryClientRef` / `pub enum MemoryState``store/client.rs` — async client handle used by RPC handlers.
- `pub fn create_memory` / `pub fn create_memory_with_storage` / `pub fn create_memory_with_storage_and_routes` / `pub fn create_memory_for_migration``store/factories.rs` — bootstrap a memory instance.
- `pub struct MemoryIngestionRequest` / `pub struct MemoryIngestionResult` / `pub struct MemoryIngestionConfig` / `pub enum ExtractionMode` / `pub struct ExtractedEntity` / `pub struct ExtractedRelation` / `const DEFAULT_MEMORY_EXTRACTION_MODEL``ingestion.rs` (re-exported `mod.rs:22`).
- `pub struct IngestionQueue` / `pub struct IngestionJob``ingestion_queue.rs` — async background ingestion worker.
- `pub struct NamespaceDocumentInput` / `pub struct NamespaceMemoryHit` / `pub struct NamespaceQueryResult` / `pub struct NamespaceRetrievalContext` / `pub struct RetrievalScoreBreakdown` / `pub enum MemoryItemKind``store/types.rs`.
- RPC `memory.{init, list_documents, list_namespaces, delete_document, query_namespace, recall_context, recall_memories, list_files, read_file, write_file, namespace_list, doc_put, doc_ingest, doc_list, doc_delete, context_query, context_recall, kv_set, kv_get, kv_delete, kv_list_namespace, graph_upsert, graph_query, clear_namespace}``schemas.rs:29-55`.
- RPC tree `memory.tree.*` and retrieval — `tree/` (re-exported via `all_memory_tree_*` / `all_retrieval_*`).
- RPC slack ingestion — `slack_ingestion/` (re-exported via `all_slack_ingestion_*`).
| Path | Role |
| --- | --- |
| [`mod.rs`](mod.rs) | Module root + re-exports. |
| [`ingest_pipeline.rs`](ingest_pipeline.rs) | Source-agnostic ingest orchestration. Called by sync pipelines and tree_rpc. |
| [`jobs/`](../memory_queue/) | Background workers: extract, admit, seal, digest, topic curate. Re-exported here from `memory_queue`. |
| [`score/`](score/) | Fast scorer, signals, embeddings, entity extraction, entity-index persistence. `store.rs` will eventually split — entity-index pieces move to `memory_store::entities`. |
| [`retrieval/`](retrieval/) | Drill-down, fetch, query_source, query_global, query_topic, search; scoring + ranking on top of memory_store. |
| [`tree_global/`](../memory_tree/global/) | Global digest tree building policy: seal, digest, recap. Implemented in `memory_tree/global` and routed from here. |
| [`tree_topic/`](../memory_tree/topic/) | Topic tree building policy: hotness gating, routing, curator, backfill. Implemented in `memory_tree/topic` and routed from here. |
| [`summarizer/`](../memory_tree/summarise.rs) | LLM summarisation pipeline for sealed buckets. Implemented in `memory_tree/summarise.rs`. |
| [`stm_recall/`](stm_recall/) | Short-term recall: cross-session FTS5 lookup + ranking. |
| [`ingestion/`](ingestion/) | Document ingestion queue + extraction (entities, relations, embeddings) — feeds UnifiedMemory documents. |
| [`canonicalize/`](../memory_sync/canonicalize/) | Source → canonical markdown (chat / email / document). Implemented in `memory_sync/canonicalize` and used at ingest time. |
| [`chat/`](chat.rs) | Chat-source canonicalisation helpers. |
| [`conversations/`](../memory_conversations/) | Workspace-backed JSONL chat thread/message history. Re-exported here from `memory_conversations`. |
| [`read_rpc.rs`](read_rpc.rs) | RPC handlers for memory reads. |
| [`tree_rpc.rs`](tree_rpc.rs) | RPC handlers for tree ingest + introspection. |
| [`schemas/`](schemas/) + [`schema.rs`](schema.rs) | Controller schema definitions for the memory + memory_tree RPC namespaces. |
| [`sync_status/`](../memory_sync/sync_status/) | Sync freshness tracking + RPC. Re-exported here from `memory_sync::sync_status`. |
| [`ops/`](ops/) | RPC operation handlers + the shared `active_memory_client` helper. |
| [`preferences.rs`](preferences.rs) | User preference read/write helpers. |
| [`rpc_models.rs`](rpc_models.rs) | Shared RPC request/response shapes. |
| [`traits.rs`](traits.rs) | `Memory`, `MemoryEntry`, `MemoryCategory`, `NamespaceSummary`, `RecallOpts`. The backend-agnostic contract every store implements. |
| [`util/`](util/) | Small helpers (redact for log PII). |
| [`global.rs`](global.rs) | Global-namespace helpers. |
## Calls into
## Layer rules
- `src/openhuman/local_ai/` — embedding model, sentiment scoring, extraction LLM.
- `src/openhuman/embeddings/` — vector backend selection.
- `src/openhuman/config/` — memory backend choice + filesystem paths.
- `src/openhuman/encryption/` — at-rest secrets for KV namespaces.
- `src/core/event_bus/` — emits `DomainEvent::Memory(*)` on ingestion / mutation.
## Called by
- `src/openhuman/agent/` (`memory_loader.rs`, `harness/memory_context.rs`, `harness/archivist*.rs`, `harness/fork_context.rs`)context injection and episodic indexing.
- `src/openhuman/learning/{reflection,tool_tracker,user_profile,prompt_sections}.rs` — long-term insight storage.
- `src/openhuman/screen_intelligence/{helpers,tests}.rs` — recall surfaces for visual context.
- `src/openhuman/autocomplete/history.rs` — query-history recall.
- `src/openhuman/tools/ops.rs` and `tools/impl/system/tool_stats.rs` — memory-backed tool stats.
- `src/core/all.rs` — registers `all_memory_*` controllers.
## Tests
- Unit: `ops_tests.rs`, `schemas_tests.rs`, `rpc_models_tests.rs`, `ingestion_tests.rs`, plus `*_tests.rs` files inside `store/`, `tree/`, `conversations/`, `slack_ingestion/`.
- Integration: `tests/autocomplete_memory_e2e.rs`, `tests/memory_graph_sync_e2e.rs`.
- **No storage in this module.** All persistence goes through
`memory_store::*`. If you're tempted to open a SQLite connection
here, the connection helper belongs one layer down.
- **No upward dependencies.** memory may import from memory_store /
memory_tree / memory_entities / memory_archivist / memory_graph /
memory_tools, but the inverse is a layer violation. (The two
documented exceptions today — `memory_store::retrieval::tree_walk`
calling `memory::retrieval::drill_down`, and `memory_store::trees::registry`
pulling `GLOBAL_SCOPE` from `memory::tree_global`are tracked in
their respective READMEs.)
- **Surface high-level tool calls** that route to the right submodule;
don't expose internals at the call site.
+257
View File
@@ -0,0 +1,257 @@
//! Memory LLM adapter backed by the unified inference provider stack.
//!
//! Memory callers still want a tiny prompt surface: one system message, one
//! user message, and a string response. This module keeps that narrow contract
//! for the rest of the memory layer, but routes every production call through
//! `openhuman::inference::provider` so memory uses the same workload routing as
//! the rest of the app.
use std::sync::Arc;
use anyhow::Result;
use async_trait::async_trait;
use crate::openhuman::config::{Config, DEFAULT_CLOUD_LLM_MODEL};
use crate::openhuman::inference::provider::{
create_chat_provider, provider_for_role, ChatMessage, Provider,
};
/// One pair of prompt messages handed to the memory LLM backend.
#[derive(Debug, Clone)]
pub struct ChatPrompt {
pub system: String,
pub user: String,
pub temperature: f64,
pub kind: &'static str,
}
/// Pluggable LLM surface used by the memory layer.
#[async_trait]
pub trait ChatProvider: Send + Sync {
fn name(&self) -> &str;
async fn chat_for_json(&self, prompt: &ChatPrompt) -> Result<String>;
async fn chat_for_text(&self, prompt: &ChatPrompt) -> Result<String> {
self.chat_for_json(prompt).await
}
}
struct InferenceChatProvider {
inner: Box<dyn Provider>,
model: String,
display: String,
}
impl InferenceChatProvider {
fn new(inner: Box<dyn Provider>, model: String) -> Self {
let display = format!("inference:{model}");
Self {
inner,
model,
display,
}
}
async fn run(&self, prompt: &ChatPrompt) -> Result<String> {
log::debug!(
"[memory::chat] provider={} kind={} model={} sys_chars={} user_chars={}",
self.display,
prompt.kind,
self.model,
prompt.system.len(),
prompt.user.len()
);
let messages = vec![
ChatMessage::system(prompt.system.clone()),
ChatMessage::user(prompt.user.clone()),
];
let text = self
.inner
.chat_with_history(&messages, &self.model, prompt.temperature)
.await?;
log::debug!(
"[memory::chat] provider={} kind={} response_chars={}",
self.display,
prompt.kind,
text.len()
);
Ok(text)
}
}
#[async_trait]
impl ChatProvider for InferenceChatProvider {
fn name(&self) -> &str {
&self.display
}
async fn chat_for_json(&self, prompt: &ChatPrompt) -> Result<String> {
self.run(prompt).await
}
async fn chat_for_text(&self, prompt: &ChatPrompt) -> Result<String> {
self.run(prompt).await
}
}
fn routed_memory_config(config: &Config) -> Config {
let mut routed = config.clone();
if !config.workload_uses_local("memory") {
routed.default_model = Some(
config
.memory_tree
.cloud_llm_model
.clone()
.unwrap_or_else(|| DEFAULT_CLOUD_LLM_MODEL.to_string()),
);
}
routed
}
#[cfg(test)]
fn test_override_runtime() -> Option<(Arc<dyn ChatProvider>, String)> {
test_override::current().map(|provider| (provider, "test:override".to_string()))
}
#[cfg(not(test))]
fn test_override_runtime() -> Option<(Arc<dyn ChatProvider>, String)> {
None
}
/// Build the memory LLM provider and return the resolved model id.
pub fn build_chat_runtime(config: &Config) -> Result<(Arc<dyn ChatProvider>, String)> {
if let Some(runtime) = test_override_runtime() {
return Ok(runtime);
}
let routed = routed_memory_config(config);
let resolved_provider = provider_for_role("summarization", &routed);
let (provider, model) = create_chat_provider("summarization", &routed)?;
log::debug!(
"[memory::chat] built provider route={} model={}",
resolved_provider,
model
);
Ok((
Arc::new(InferenceChatProvider::new(provider, model.clone())),
model,
))
}
/// Build the memory LLM provider dictated by the inference workload routing.
pub fn build_chat_provider(config: &Config) -> Result<Arc<dyn ChatProvider>> {
Ok(build_chat_runtime(config)?.0)
}
#[cfg(test)]
pub struct StaticChatProvider {
pub response: String,
pub calls: std::sync::atomic::AtomicUsize,
}
#[cfg(test)]
impl StaticChatProvider {
pub fn new(response: impl Into<String>) -> Self {
Self {
response: response.into(),
calls: std::sync::atomic::AtomicUsize::new(0),
}
}
}
#[cfg(test)]
#[async_trait]
impl ChatProvider for StaticChatProvider {
fn name(&self) -> &str {
"test:static"
}
async fn chat_for_json(&self, _prompt: &ChatPrompt) -> Result<String> {
self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Ok(self.response.clone())
}
}
#[cfg(test)]
pub mod test_override {
use super::ChatProvider;
use std::sync::Arc;
tokio::task_local! {
static OVERRIDE: Arc<dyn ChatProvider>;
}
pub fn current() -> Option<Arc<dyn ChatProvider>> {
OVERRIDE.try_with(Arc::clone).ok()
}
pub async fn with_provider<F, T>(provider: Arc<dyn ChatProvider>, fut: F) -> T
where
F: std::future::Future<Output = T>,
{
OVERRIDE.scope(provider, fut).await
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn build_provider_returns_inference_wrapper_when_default() {
let cfg = Config::default();
let provider = build_chat_provider(&cfg).unwrap();
assert!(provider.name().contains("inference:"));
}
#[test]
fn build_chat_runtime_defaults_to_openhuman_resolved_model() {
let cfg = Config::default();
let (_provider, model) = build_chat_runtime(&cfg).unwrap();
assert_eq!(model, "reasoning-v1");
}
#[test]
fn build_chat_runtime_still_builds_when_cloud_memory_model_is_overridden() {
let mut cfg = Config::default();
cfg.memory_tree.cloud_llm_model = Some("custom-summary-model".into());
let (_provider, model) = build_chat_runtime(&cfg).unwrap();
assert_eq!(model, "reasoning-v1");
}
#[test]
fn build_provider_returns_inference_wrapper_when_local_memory_is_configured() {
let mut cfg = Config::default();
cfg.memory_provider = Some("ollama:qwen2.5:0.5b".into());
let provider = build_chat_provider(&cfg).unwrap();
assert!(provider.name().contains("qwen2.5:0.5b"));
}
#[test]
fn build_chat_runtime_preserves_local_memory_model() {
let mut cfg = Config::default();
cfg.memory_provider = Some("ollama:qwen2.5:0.5b".into());
let (_provider, model) = build_chat_runtime(&cfg).unwrap();
assert_eq!(model, "qwen2.5:0.5b");
}
#[tokio::test]
async fn static_chat_provider_returns_response_and_counts() {
let p = StaticChatProvider::new("hello");
let prompt = ChatPrompt {
system: "sys".into(),
user: "u".into(),
temperature: 0.0,
kind: "test",
};
assert_eq!(p.chat_for_json(&prompt).await.unwrap(), "hello");
assert_eq!(p.calls.load(std::sync::atomic::Ordering::SeqCst), 1);
}
}
@@ -1,72 +0,0 @@
//! Wire/storage types for the workspace-backed conversation store: threads,
//! messages, create requests, and partial-update patches.
use serde::{Deserialize, Serialize};
use serde_json::Value;
/// A persisted conversation thread, mirroring one entry in `threads.jsonl`.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct ConversationThread {
pub id: String,
pub title: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub chat_id: Option<i64>,
pub is_active: bool,
pub message_count: usize,
pub last_message_at: String,
pub created_at: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub parent_thread_id: Option<String>,
#[serde(default)]
pub labels: Vec<String>,
}
/// A single message appended to a thread's JSONL log.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct ConversationMessage {
pub id: String,
pub content: String,
#[serde(rename = "type")]
pub message_type: String,
#[serde(default)]
pub extra_metadata: Value,
pub sender: String,
pub created_at: String,
}
/// Input payload to create-or-update a thread via [`super::ensure_thread`].
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CreateConversationThread {
pub id: String,
pub title: String,
pub created_at: String,
#[serde(default)]
pub parent_thread_id: Option<String>,
#[serde(default)]
pub labels: Option<Vec<String>>,
}
/// Partial update to apply to a stored message (e.g. rewriting `extraMetadata`).
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct ConversationMessagePatch {
#[serde(default)]
pub extra_metadata: Option<Value>,
}
/// A single match returned by
/// [`super::store::ConversationStore::search_cross_thread_messages`]. Carries
/// the source `thread_id` so the caller can render provenance into the
/// `[Cross-chat context]` block (issue #1505).
#[derive(Debug, Clone, PartialEq)]
pub struct CrossThreadHit {
pub thread_id: String,
pub message_id: String,
pub role: String,
pub content: String,
pub created_at: String,
pub score: f64,
}
@@ -11,19 +11,19 @@ use serde::{Deserialize, Serialize};
use crate::core::event_bus::{publish_global, DomainEvent};
use crate::openhuman::config::Config;
use crate::openhuman::memory_tree::canonicalize::{
use crate::openhuman::memory::jobs::{self, ExtractChunkPayload, NewJob};
use crate::openhuman::memory::score::{self, ScoreResult, ScoringConfig};
use crate::openhuman::memory::util::redact::redact;
use crate::openhuman::memory_store::chunks::store as chunk_store;
use crate::openhuman::memory_store::chunks::types::SourceKind;
use crate::openhuman::memory_store::chunks::{chunk_markdown, ChunkerInput, ChunkerOptions};
use crate::openhuman::memory_store::content as content_store;
use crate::openhuman::memory_sync::canonicalize::{
chat::{self, ChatBatch},
document::{self, DocumentInput},
email::{self, EmailThread},
CanonicalisedSource,
};
use crate::openhuman::memory_tree::chunker::{chunk_markdown, ChunkerInput, ChunkerOptions};
use crate::openhuman::memory_tree::content_store;
use crate::openhuman::memory_tree::jobs::{self, ExtractChunkPayload, NewJob};
use crate::openhuman::memory_tree::score::{self, ScoreResult, ScoringConfig};
use crate::openhuman::memory_tree::store;
use crate::openhuman::memory_tree::types::SourceKind;
use crate::openhuman::memory_tree::util::redact::redact;
use std::time::{SystemTime, UNIX_EPOCH};
const BODY_PREVIEW_MAX_BYTES: usize = 2048;
@@ -124,7 +124,7 @@ pub async fn ingest_document(
) -> Result<IngestResult> {
if already_ingested(config, SourceKind::Document, source_id).await? {
log::debug!(
"[memory_tree::ingest] skip ingest_document — source_id_hash={} already ingested",
"[memory::ingest_pipeline] skip ingest_document — source_id_hash={} already ingested",
redact(source_id)
);
return Ok(IngestResult::already_ingested(source_id));
@@ -147,7 +147,7 @@ async fn already_ingested(
) -> Result<bool> {
let cfg = config.clone();
let sid = source_id.to_string();
tokio::task::spawn_blocking(move || store::is_source_ingested(&cfg, source_kind, &sid))
tokio::task::spawn_blocking(move || chunk_store::is_source_ingested(&cfg, source_kind, &sid))
.await
.map_err(|e| anyhow::anyhow!("already_ingested join error: {e}"))?
}
@@ -175,8 +175,8 @@ async fn persist(
Ok(preview) => Some(preview),
Err(_) => {
log::error!(
"[memory_tree::ingest] markdown_body_preview panicked for source_id_hash={}; falling back to no preview",
crate::openhuman::memory_tree::util::redact::redact(source_id)
"[memory::ingest_pipeline] markdown_body_preview panicked for source_id_hash={}; falling back to no preview",
crate::openhuman::memory::util::redact::redact(source_id)
);
None
}
@@ -201,13 +201,13 @@ async fn persist(
// spawn_blocking so errors surface before the DB transaction opens.
let content_root = config.memory_tree_content_root();
let staged = content_store::stage_chunks(&content_root, &chunks)
.map_err(|e| anyhow::anyhow!("[memory_tree::ingest] stage_chunks failed: {e}"))?;
.map_err(|e| anyhow::anyhow!("[memory::ingest_pipeline] stage_chunks failed: {e}"))?;
let scoring_cfg = ScoringConfig::from_config(config);
let scores = score::score_chunks_fast(&chunks, &scoring_cfg).await?;
if scores.len() != chunks.len() {
anyhow::bail!(
"[memory_tree::ingest] scorer length mismatch: chunks={} scores={}",
"[memory::ingest_pipeline] scorer length mismatch: chunks={} scores={}",
chunks.len(),
scores.len()
);
@@ -226,7 +226,7 @@ async fn persist(
let source_id_for_store = source_id.to_string();
let written = tokio::task::spawn_blocking(move || -> Result<Option<usize>> {
use std::collections::{HashMap, HashSet};
store::with_connection(&config_owned, |conn| {
chunk_store::with_connection(&config_owned, |conn| {
// IMMEDIATE, not the default DEFERRED: this transaction reads
// (get_chunk_lifecycle_status_tx) before it writes
// (upsert_staged_chunks_tx). A DEFERRED tx takes only a read
@@ -262,7 +262,7 @@ async fn persist(
// genuine replays without blocking legitimate appends.
if source_kind_for_store == SourceKind::Document {
let now_ms = chrono::Utc::now().timestamp_millis();
let claimed = store::claim_source_ingest_tx(
let claimed = chunk_store::claim_source_ingest_tx(
&tx,
source_kind_for_store,
&source_id_for_store,
@@ -270,7 +270,7 @@ async fn persist(
)?;
if !claimed {
log::debug!(
"[memory_tree::ingest] persist gate: document already ingested source_id_hash={}",
"[memory::ingest_pipeline] persist gate: document already ingested source_id_hash={}",
redact(&source_id_for_store)
);
// Drop the (empty) transaction implicitly; nothing to commit.
@@ -287,11 +287,11 @@ async fn persist(
// "already-admitted-from-prior-ingest".
let mut prior: HashMap<String, Option<String>> = HashMap::new();
for s in &staged_for_store {
let status = store::get_chunk_lifecycle_status_tx(&tx, &s.chunk.id)?;
let status = chunk_store::get_chunk_lifecycle_status_tx(&tx, &s.chunk.id)?;
prior.insert(s.chunk.id.clone(), status);
}
let n = store::upsert_staged_chunks_tx(&tx, &staged_for_store)?;
let n = chunk_store::upsert_staged_chunks_tx(&tx, &staged_for_store)?;
// Re-ingest of identical content (same chunk_id) must NOT
// downgrade chunks that have already progressed through the
@@ -312,13 +312,13 @@ async fn persist(
let pre = prior.get(&s.chunk.id).cloned().flatten();
let needs_processing = matches!(
pre.as_deref(),
None | Some(store::CHUNK_STATUS_PENDING_EXTRACTION),
None | Some(chunk_store::CHUNK_STATUS_PENDING_EXTRACTION),
);
if needs_processing {
store::set_chunk_lifecycle_status_tx(
chunk_store::set_chunk_lifecycle_status_tx(
&tx,
&s.chunk.id,
store::CHUNK_STATUS_PENDING_EXTRACTION,
chunk_store::CHUNK_STATUS_PENDING_EXTRACTION,
)?;
to_schedule.insert(s.chunk.id.clone());
}
@@ -407,7 +407,7 @@ fn markdown_body_preview(md: &str) -> String {
// continuation byte; fall back to the full string rather than panicking.
if start > len || !md.is_char_boundary(start) {
log::error!(
"[memory_tree::ingest] ceil_char_boundary returned invalid boundary start={start} len={len}; returning full markdown"
"[memory::ingest_pipeline] ceil_char_boundary returned invalid boundary start={start} len={len}; returning full markdown"
);
md.to_string()
} else {
@@ -419,14 +419,14 @@ fn markdown_body_preview(md: &str) -> String {
#[cfg(test)]
mod tests {
use super::*;
use crate::openhuman::memory_tree::canonicalize::chat::ChatMessage;
use crate::openhuman::memory_tree::jobs::drain_until_idle;
use crate::openhuman::memory_tree::score::store::{count_scores, lookup_entity};
use crate::openhuman::memory_tree::store::{
use crate::openhuman::memory::jobs::drain_until_idle;
use crate::openhuman::memory::score::store::{count_scores, lookup_entity};
use crate::openhuman::memory_store::chunks::store::{
count_chunks, count_chunks_by_lifecycle_status, get_chunk_embedding, list_chunks,
ListChunksQuery, CHUNK_STATUS_BUFFERED, CHUNK_STATUS_DROPPED,
};
use crate::openhuman::memory_tree::types::SourceKind;
use crate::openhuman::memory_store::chunks::types::SourceKind;
use crate::openhuman::memory_sync::canonicalize::chat::ChatMessage;
use chrono::{TimeZone, Utc};
use tempfile::TempDir;
+1 -1
View File
@@ -30,8 +30,8 @@ use parse::{enrich_document_metadata, parse_document};
use serde_json::json;
use types::ParsedIngestion;
use crate::openhuman::memory::store::types::NamespaceDocumentInput;
use crate::openhuman::memory::UnifiedMemory;
use crate::openhuman::memory_store::types::NamespaceDocumentInput;
impl UnifiedMemory {
/// Run the full ingestion pipeline for a document: parse + chunk + extract
+141 -1
View File
@@ -14,8 +14,8 @@ use super::types::{
ExtractedEntity, ExtractedRelation, ExtractionAccumulator, ExtractionMode, ExtractionUnit,
MemoryIngestionConfig, ParsedIngestion, RawEntity, RawRelation, DEFAULT_CHUNK_TOKENS,
};
use crate::openhuman::memory::store::types::NamespaceDocumentInput;
use crate::openhuman::memory::UnifiedMemory;
use crate::openhuman::memory_store::types::NamespaceDocumentInput;
// ── Chunking helpers ──────────────────────────────────────────────────────────
@@ -933,3 +933,143 @@ pub(super) async fn parse_document(
decision_count: accumulator.decisions.len(),
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::openhuman::memory_store::types::NamespaceDocumentInput;
use serde_json::json;
fn sample_input() -> NamespaceDocumentInput {
NamespaceDocumentInput {
namespace: "global".into(),
key: "doc-1".into(),
title: "OpenHuman roadmap".into(),
content: "Alice owns roadmap".into(),
source_type: "manual".into(),
priority: "normal".into(),
tags: vec!["existing".into()],
metadata: json!({"seed": true}),
category: "core".into(),
session_id: Some("session-1".into()),
document_id: Some("doc-id-1".into()),
}
}
#[test]
fn split_sentences_breaks_on_punctuation_and_merges_tiny_fragments() {
let parts = split_sentences("Hello world. Ok.\nNext line?");
assert_eq!(parts.len(), 2);
assert_eq!(parts[0], "Hello world Ok");
assert_eq!(parts[1], "Next line?");
}
#[test]
fn build_units_respects_extraction_mode() {
let chunks = vec!["One. Two.".to_string(), "Three".to_string()];
let sentence_units = build_units(&chunks, ExtractionMode::Sentence);
let chunk_units = build_units(&chunks, ExtractionMode::Chunk);
assert_eq!(sentence_units.len(), 2);
assert_eq!(sentence_units[0].chunk_index, 0);
assert_eq!(sentence_units[0].text, "One Two");
assert_eq!(sentence_units[1].chunk_index, 1);
assert_eq!(sentence_units[1].text, "Three");
assert_eq!(chunk_units.len(), 2);
assert_eq!(chunk_units[0].text, "One. Two");
assert_eq!(chunk_units[1].text, "Three");
}
#[test]
fn find_chunk_index_prefers_hint_then_wraps() {
let chunks = vec![
"alpha content".to_string(),
"beta needle".to_string(),
"gamma trailing".to_string(),
];
assert_eq!(find_chunk_index(&chunks, "needle", 1), 1);
assert_eq!(find_chunk_index(&chunks, "alpha", 2), 0);
assert_eq!(find_chunk_index(&chunks, "missing", 2), 2);
}
#[test]
fn alias_map_builds_reverse_lookup() {
let mut entities = HashMap::new();
entities.insert(
"ALICE".into(),
RawEntity {
name: "ALICE".into(),
entity_type: "PERSON".into(),
confidence: 0.8,
},
);
entities.insert(
"ALICE SMITH".into(),
RawEntity {
name: "ALICE SMITH".into(),
entity_type: "PERSON".into(),
confidence: 0.9,
},
);
let aliases = build_alias_map(&entities);
assert_eq!(
aliases.get("ALICE").map(String::as_str),
Some("ALICE SMITH")
);
assert_eq!(resolve_alias("ALICE", &aliases), "ALICE SMITH");
let reverse = reverse_aliases(&aliases);
assert_eq!(reverse.get("ALICE SMITH"), Some(&vec!["ALICE".to_string()]));
}
#[test]
fn enrich_document_metadata_merges_tags_and_ingestion_details() {
let input = sample_input();
let parsed = ParsedIngestion {
tags: vec!["decision".into(), "existing".into()],
metadata: json!({"kind": "profile", "extra": 1}),
entities: vec![],
relations: vec![],
chunk_count: 3,
preference_count: 1,
decision_count: 2,
};
let config = MemoryIngestionConfig::default();
let (enriched, tags) = enrich_document_metadata(&input, &parsed, &config);
assert_eq!(tags, vec!["decision".to_string(), "existing".to_string()]);
assert_eq!(enriched.tags, tags);
assert_eq!(enriched.metadata["seed"], json!(true));
assert_eq!(enriched.metadata["extra"], json!(1));
assert_eq!(
enriched.metadata["ingestion"]["model_name"],
config.model_name
);
assert_eq!(enriched.metadata["ingestion"]["chunk_count"], json!(3));
}
#[test]
fn extract_people_from_header_collects_named_people() {
let mut acc = ExtractionAccumulator::default();
let people = extract_people_from_header(
"Alice Smith <alice@example.com>, Bob Jones <bob@example.com>",
&mut acc,
);
assert_eq!(
people,
vec!["ALICE SMITH".to_string(), "BOB JONES".to_string()]
);
assert!(acc.entities.contains_key("ALICE SMITH"));
assert!(acc.entities.contains_key("BOB JONES"));
}
#[test]
fn detect_primary_subject_only_matches_openhuman() {
assert_eq!(
detect_primary_subject("OpenHuman desktop roadmap"),
Some("OPENHUMAN".to_string())
);
assert_eq!(detect_primary_subject("General roadmap"), None);
}
}
+1 -1
View File
@@ -19,7 +19,7 @@ use tokio::sync::mpsc;
use super::state::IngestionState;
use super::MemoryIngestionConfig;
use crate::core::event_bus::{publish_global, DomainEvent};
use crate::openhuman::memory::store::{NamespaceDocumentInput, UnifiedMemory};
use crate::openhuman::memory_store::{NamespaceDocumentInput, UnifiedMemory};
/// Default capacity of the ingestion job channel.
///
+30
View File
@@ -187,3 +187,33 @@ pub(super) fn classify_entity(name: &str, known_people: &HashMap<String, String>
}
"TOPIC"
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn sanitize_entity_name_trims_punctuation_and_uppercases() {
assert_eq!(sanitize_entity_name(" Alice Smith. "), "ALICE SMITH");
assert_eq!(sanitize_entity_name("\"openhuman\""), "OPENHUMAN");
assert_eq!(sanitize_entity_name(""), "");
}
#[test]
fn sanitize_fact_text_collapses_whitespace_and_strips_edges() {
assert_eq!(sanitize_fact_text(" - Hello world. "), "Hello world");
assert_eq!(sanitize_fact_text(":: spaced\ttext ;;"), "spaced text");
}
#[test]
fn classify_entity_detects_dates_people_and_products() {
let mut known_people = HashMap::new();
known_people.insert("ALICE SMITH".to_string(), "ALICE SMITH".to_string());
assert_eq!(classify_entity("Jan 5, 2026", &known_people), "DATE");
assert_eq!(classify_entity("Alice Smith", &known_people), "PERSON");
assert_eq!(classify_entity("OpenHuman", &known_people), "PRODUCT");
assert_eq!(classify_entity("Kitchen", &known_people), "ROOM");
assert_eq!(classify_entity("phoenix-project", &known_people), "PROJECT");
}
}
+103
View File
@@ -220,3 +220,106 @@ impl ExtractionAccumulator {
});
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn relation_rule_normalizes_supported_predicates() {
let owns = relation_rule("works_on").unwrap();
assert_eq!(owns.canonical, "OWNS");
assert_eq!(owns.allowed_head, PERSON_TYPES);
let prefers = relation_rule("prefers").unwrap();
assert_eq!(prefers.canonical, "PREFERS");
let deadline = relation_rule("due_on").unwrap();
assert_eq!(deadline.canonical, "HAS_DEADLINE");
}
#[test]
fn type_allowed_honors_allowlist_and_empty_list() {
assert!(type_allowed("PERSON", PERSON_TYPES));
assert!(!type_allowed("PROJECT", PERSON_TYPES));
assert!(type_allowed("ANYTHING", &[]));
}
#[test]
fn resolve_person_alias_uses_known_people_map() {
let mut known = std::collections::HashMap::new();
known.insert("ALICE".to_string(), "ALICE SMITH".to_string());
assert_eq!(resolve_person_alias("ALICE", &known), "ALICE SMITH");
assert_eq!(resolve_person_alias("BOB", &known), "BOB");
}
#[test]
fn add_entity_tracks_highest_confidence_and_person_aliases() {
let mut acc = ExtractionAccumulator::default();
let first = acc.add_entity("Alice Smith", "PERSON", 0.6).unwrap();
let second = acc.add_entity("Alice Smith", "PERSON", 0.9).unwrap();
assert_eq!(first, "ALICE SMITH");
assert_eq!(second, "ALICE SMITH");
assert_eq!(acc.entities["ALICE SMITH"].confidence, 0.9);
assert_eq!(
acc.known_people.get("ALICE").map(String::as_str),
Some("ALICE SMITH")
);
}
#[test]
fn add_relation_rejects_invalid_or_self_relations() {
let mut acc = ExtractionAccumulator::default();
acc.add_relation(
"Alice",
"PERSON",
"owns",
"Alice",
"PERSON",
0.8,
0,
0,
Map::new(),
);
assert!(acc.relations.is_empty(), "self relation should be dropped");
acc.add_relation(
"Alice",
"PERSON",
"unknown_predicate",
"Project X",
"PROJECT",
0.8,
0,
0,
Map::new(),
);
assert!(
acc.relations.is_empty(),
"unknown predicate should be ignored"
);
}
#[test]
fn add_relation_canonicalizes_predicate_and_collects_chunk_index() {
let mut acc = ExtractionAccumulator::default();
acc.add_relation(
"Alice",
"PERSON",
"works_on",
"Phoenix",
"PROJECT",
0.8,
3,
11,
Map::new(),
);
assert_eq!(acc.relations.len(), 1);
let relation = &acc.relations[0];
assert_eq!(relation.predicate, "OWNS");
assert_eq!(relation.subject, "ALICE");
assert_eq!(relation.object, "PHOENIX");
assert!(relation.chunk_indexes.contains(&3));
assert_eq!(relation.order_index, 11);
}
}
+43 -1
View File
@@ -5,7 +5,7 @@ use std::collections::{BTreeSet, HashMap};
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use crate::openhuman::memory::store::types::NamespaceDocumentInput;
use crate::openhuman::memory_store::types::NamespaceDocumentInput;
/// Default extraction backend label reported in ingestion metadata.
pub const DEFAULT_MEMORY_EXTRACTION_MODEL: &str = "heuristic-only";
@@ -243,3 +243,45 @@ pub(super) struct ParsedIngestion {
pub(super) preference_count: usize,
pub(super) decision_count: usize,
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn extraction_mode_default_is_sentence() {
assert_eq!(ExtractionMode::default(), ExtractionMode::Sentence);
assert_eq!(ExtractionMode::Sentence.as_str(), "sentence");
assert_eq!(ExtractionMode::Chunk.as_str(), "chunk");
}
#[test]
fn memory_ingestion_config_default_matches_expected_thresholds() {
let cfg = MemoryIngestionConfig::default();
assert_eq!(cfg.model_name, DEFAULT_MEMORY_EXTRACTION_MODEL);
assert_eq!(cfg.extraction_mode, ExtractionMode::Sentence);
assert_eq!(cfg.entity_threshold, 0.45);
assert_eq!(cfg.relation_threshold, 0.30);
assert_eq!(cfg.adjacency_threshold, 0.50);
assert_eq!(cfg.batch_size, 16);
}
#[test]
fn memory_ingestion_request_defaults_config_when_absent() {
let request: MemoryIngestionRequest = serde_json::from_value(json!({
"document": {
"namespace": "global",
"key": "doc-1",
"title": "Doc",
"content": "Body",
"source_type": "manual",
"priority": "normal",
"category": "core"
}
}))
.unwrap();
assert_eq!(request.config.model_name, DEFAULT_MEMORY_EXTRACTION_MODEL);
assert_eq!(request.config.extraction_mode, ExtractionMode::Sentence);
}
}
+43 -23
View File
@@ -1,24 +1,56 @@
//! Memory system for OpenHuman.
//! Memory orchestration.
//!
//! This module provides the core abstractions and implementations for the memory system,
//! including semantic search, ingestion pipelines, document management, and knowledge graph
//! operations. It integrates vector search, keyword search, and relational data to provide
//! a unified memory interface for AI agents.
//! This module is the high-level routing + policy layer over the memory
//! stack. Owns the ingest pipeline, background job handlers, scoring,
//! tree-building policy (tree_global / tree_topic), recall ranking, and
//! the RPC surface. Storage primitives all live in sibling memory_*
//! modules — see [`README.md`](README.md) for the full map.
//!
//! No SQLite, no on-disk md, no vector tables here — those belong one
//! layer down in [`memory_store`](crate::openhuman::memory_store).
pub mod chunker;
pub mod conversations;
// Legacy memory modules
pub mod global;
pub mod ingestion;
pub mod ops;
pub mod preferences;
pub mod rpc_models;
pub mod safety;
pub mod schemas;
pub mod stm_recall;
pub mod store;
pub mod sync_status;
pub mod tool_memory;
pub mod traits;
// Tool-scoped memory moved to top-level `memory_tools`. Re-exported here so
// existing `memory::tool_memory::*` paths still resolve during the migration.
pub use crate::openhuman::memory_tools as tool_memory;
// Modules moved from memory_tree (Phase 3)
pub mod chat;
pub mod ingest_pipeline;
pub mod read_rpc;
pub mod retrieval;
pub mod schema;
pub mod score;
pub mod tree_rpc;
pub mod util;
// Conversation storage moved to top-level `memory_conversations`. Re-exported
// here so existing `memory::conversations::*` paths still resolve during the
// migration.
pub use crate::openhuman::memory_conversations as conversations;
// Async memory job queue moved to top-level `memory_queue`. Re-exported here
// so existing `memory::jobs::*` paths still resolve during the migration.
pub use crate::openhuman::memory_queue as jobs;
pub use crate::openhuman::memory_store::{
create_memory, create_memory_for_migration, create_memory_with_local_ai,
effective_embedding_settings, effective_memory_backend_name, MemoryClient, MemoryClientRef,
MemoryItemKind, MemoryState, NamespaceDocumentInput, NamespaceMemoryHit, NamespaceQueryResult,
NamespaceRetrievalContext, RetrievalScoreBreakdown, UnifiedMemory,
};
pub use crate::openhuman::memory_sync::sync_status::{
all_memory_sync_status_controller_schemas, all_memory_sync_status_registered_controllers,
FreshnessLabel, MemorySyncStatus,
};
pub use ingestion::{
ExtractedEntity, ExtractedRelation, ExtractionMode, IngestionJob, IngestionQueue,
IngestionState, IngestionStatusSnapshot, MemoryIngestionConfig, MemoryIngestionRequest,
@@ -31,18 +63,6 @@ pub use schemas::{
all_controller_schemas as all_memory_controller_schemas,
all_registered_controllers as all_memory_registered_controllers,
};
pub use store::{
create_memory, create_memory_for_migration, create_memory_with_local_ai,
create_memory_with_storage, create_memory_with_storage_and_routes,
effective_embedding_settings, effective_embedding_settings_probed,
effective_memory_backend_name, MemoryClient, MemoryClientRef, MemoryItemKind, MemoryState,
NamespaceDocumentInput, NamespaceMemoryHit, NamespaceQueryResult, NamespaceRetrievalContext,
RetrievalScoreBreakdown, UnifiedMemory,
};
pub use sync_status::{
all_memory_sync_status_controller_schemas, all_memory_sync_status_registered_controllers,
FreshnessLabel, MemorySyncStatus,
};
pub use tool_memory::{
render_tool_memory_rules, tool_memory_namespace, ToolMemoryCaptureHook, ToolMemoryPriority,
ToolMemoryRule, ToolMemoryRulesSection, ToolMemorySource, ToolMemoryStore, TOOL_MEMORY_HEADING,
+219
View File
@@ -489,3 +489,222 @@ pub async fn memory_recall_memories(
Err(message) => Ok(error_envelope("memory.recall_memories_failed", message)),
}
}
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use std::sync::OnceLock;
use serde_json::json;
use tempfile::TempDir;
use super::*;
fn ensure_memory_client() {
static WORKSPACE: OnceLock<PathBuf> = OnceLock::new();
let workspace = WORKSPACE.get_or_init(|| {
let tmp = TempDir::new().expect("tempdir");
let path = tmp.path().join("workspace");
std::fs::create_dir_all(&path).expect("workspace dir");
std::mem::forget(tmp);
path
});
let _ = crate::openhuman::memory::global::init(workspace.clone());
}
fn unique_namespace(prefix: &str) -> String {
format!("{prefix}-{}", uuid::Uuid::new_v4())
}
fn sample_put(namespace: String, key: String, title: &str, content: &str) -> PutDocParams {
PutDocParams {
namespace,
key,
title: title.into(),
content: content.into(),
source_type: default_source_type(),
priority: default_priority(),
tags: vec!["test".into()],
metadata: json!({"source": "test"}),
category: default_category(),
session_id: Some("session-docs".into()),
document_id: None,
}
}
#[tokio::test]
async fn direct_document_handlers_roundtrip_through_namespace() {
ensure_memory_client();
let namespace = unique_namespace("memory-docs-direct");
let key = format!("note-{}", uuid::Uuid::new_v4());
let put = doc_put(sample_put(
namespace.clone(),
key.clone(),
"Rust ownership",
"Ownership and borrowing let Rust enforce memory safety.",
))
.await
.expect("doc_put");
let document_id = put.value.document_id.clone();
assert!(!document_id.is_empty());
let listed = doc_list(Some(NamespaceOnlyParams {
namespace: namespace.clone(),
}))
.await
.expect("doc_list");
let docs = listed
.value
.get("documents")
.and_then(|v| v.as_array())
.expect("documents array");
assert!(docs.iter().any(|doc| doc["key"] == key));
let queried = context_query(QueryNamespaceParams {
namespace: namespace.clone(),
query: "ownership".into(),
limit: Some(5),
})
.await
.expect("context_query");
assert!(
queried.value.to_lowercase().contains("ownership"),
"query result should mention the stored concept"
);
let recalled = context_recall(RecallNamespaceParams {
namespace: namespace.clone(),
limit: Some(5),
})
.await
.expect("context_recall");
assert!(recalled.value.is_some());
let deleted = doc_delete(DeleteDocParams {
namespace: namespace.clone(),
document_id: document_id.clone(),
})
.await
.expect("doc_delete");
assert_eq!(deleted.logs, vec!["memory document deleted".to_string()]);
let deleted_flag = deleted
.value
.get("deleted")
.and_then(|v| v.as_bool())
.unwrap_or(false);
assert!(deleted_flag, "delete result should report success");
let after = doc_list(Some(NamespaceOnlyParams { namespace }))
.await
.expect("doc_list after delete");
let after_docs = after
.value
.get("documents")
.and_then(|v| v.as_array())
.expect("documents array after delete");
assert!(after_docs.is_empty());
}
#[tokio::test]
async fn envelope_memory_handlers_report_counts_and_statuses() {
ensure_memory_client();
let namespace = unique_namespace("memory-docs-envelope");
let key = format!("env-{}", uuid::Uuid::new_v4());
let _ = memory_init(MemoryInitRequest { jwt_token: None })
.await
.expect("memory_init");
let direct = doc_put(sample_put(
namespace.clone(),
key.clone(),
"Borrow checker",
"The borrow checker enforces aliasing and mutation rules.",
))
.await
.expect("seed document");
let document_id = direct.value.document_id;
let listed = memory_list_documents(ListDocumentsRequest {
namespace: Some(namespace.clone()),
})
.await
.expect("memory_list_documents");
let listed_data = listed.value.data.expect("list envelope data");
assert_eq!(listed_data.count, 1);
assert_eq!(listed_data.documents[0].key, key);
assert_eq!(
listed
.value
.meta
.counts
.as_ref()
.and_then(|m| m.get("num_documents")),
Some(&1)
);
let namespaces = memory_list_namespaces(EmptyRequest {})
.await
.expect("memory_list_namespaces");
let namespace_data = namespaces.value.data.expect("namespace data");
assert!(
namespace_data.namespaces.iter().any(|ns| ns == &namespace),
"expected namespace list to include the seeded namespace"
);
let queried = memory_query_namespace(QueryNamespaceRequest {
namespace: namespace.clone(),
query: "borrow checker".into(),
limit: Some(5),
max_chunks: None,
include_references: Some(true),
document_ids: None,
})
.await
.expect("memory_query_namespace");
let query_data = queried.value.data.expect("query data");
assert!(query_data.llm_context_message.is_some());
assert!(query_data.context.is_some());
let recalled = memory_recall_memories(RecallMemoriesRequest {
namespace: namespace.clone(),
min_retention: None,
as_of: None,
limit: Some(5),
max_chunks: None,
top_k: None,
})
.await
.expect("memory_recall_memories");
let recall_data = recalled.value.data.expect("recall data");
assert_eq!(recall_data.memories.len(), 1);
assert_eq!(recall_data.memories[0].kind, "document");
let deleted = memory_delete_document(DeleteDocumentRequest {
namespace: namespace.clone(),
document_id,
})
.await
.expect("memory_delete_document");
let deleted_data = deleted.value.data.expect("delete envelope data");
assert_eq!(deleted_data.status, "completed");
assert!(deleted_data.deleted);
let cleared = clear_namespace(ClearNamespaceParams {
namespace: namespace.clone(),
})
.await
.expect("clear_namespace");
assert!(cleared.value.cleared);
let listed_after = memory_list_documents(ListDocumentsRequest {
namespace: Some(namespace),
})
.await
.expect("memory_list_documents after clear");
let after_data = listed_after.value.data.expect("after clear data");
assert_eq!(after_data.count, 0);
}
}
+31
View File
@@ -80,3 +80,34 @@ pub(crate) fn error_envelope<T: Serialize>(
vec![],
)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn memory_counts_collects_named_entries() {
let counts = memory_counts([("documents", 2), ("entities", 5)]);
assert_eq!(counts.get("documents"), Some(&2));
assert_eq!(counts.get("entities"), Some(&5));
}
#[test]
fn envelope_wraps_data_with_meta() {
let outcome = envelope(serde_json::json!({"ok": true}), None, None);
let value = outcome.into_cli_compatible_json().unwrap();
assert_eq!(value["data"]["ok"], true);
assert!(value["meta"]["request_id"].as_str().is_some());
assert!(value["error"].is_null());
}
#[test]
fn error_envelope_wraps_code_and_message() {
let outcome: RpcOutcome<ApiEnvelope<serde_json::Value>> =
error_envelope("bad_request", "boom".to_string());
let value = outcome.into_cli_compatible_json().unwrap();
assert_eq!(value["error"]["code"], "bad_request");
assert_eq!(value["error"]["message"], "boom");
assert!(value["data"].is_null());
}
}
+277
View File
@@ -102,3 +102,280 @@ pub async fn ai_write_memory_file(
None,
))
}
#[cfg(test)]
mod tests {
use std::ffi::OsString;
use std::sync::{Mutex, OnceLock};
use tempfile::TempDir;
use super::*;
fn env_mutex() -> &'static Mutex<()> {
static ENV_MUTEX: OnceLock<Mutex<()>> = OnceLock::new();
ENV_MUTEX.get_or_init(|| Mutex::new(()))
}
struct WorkspaceEnvGuard {
previous: Option<OsString>,
}
impl WorkspaceEnvGuard {
fn set(path: &std::path::Path) -> Self {
let previous = std::env::var_os("OPENHUMAN_WORKSPACE");
unsafe {
std::env::set_var("OPENHUMAN_WORKSPACE", path);
}
Self { previous }
}
}
impl Drop for WorkspaceEnvGuard {
fn drop(&mut self) {
unsafe {
if let Some(previous) = self.previous.take() {
std::env::set_var("OPENHUMAN_WORKSPACE", previous);
} else {
std::env::remove_var("OPENHUMAN_WORKSPACE");
}
}
}
}
#[tokio::test]
async fn write_read_and_list_memory_files_roundtrip() {
let _guard = env_mutex()
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let tmp = TempDir::new().expect("tempdir");
let _workspace = WorkspaceEnvGuard::set(tmp.path());
let write = ai_write_memory_file(WriteMemoryFileRequest {
relative_path: "notes/today.md".to_string(),
content: "remember this".to_string(),
})
.await
.expect("write should succeed");
let write_data = write.value.data.expect("write data");
assert_eq!(write_data.relative_path, "notes/today.md");
assert!(write_data.written);
assert_eq!(write_data.bytes_written, "remember this".len());
let read = ai_read_memory_file(ReadMemoryFileRequest {
relative_path: "notes/today.md".to_string(),
})
.await
.expect("read should succeed");
let read_data = read.value.data.expect("read data");
assert_eq!(read_data.relative_path, "notes/today.md");
assert_eq!(read_data.content, "remember this");
let memory_root = super::super::helpers::resolve_existing_memory_path("")
.await
.expect("resolve memory root");
tokio::fs::write(memory_root.join("b.md"), "b")
.await
.expect("write b");
tokio::fs::write(memory_root.join("a.md"), "a")
.await
.expect("write a");
tokio::fs::create_dir_all(memory_root.join("nested"))
.await
.expect("create nested dir");
tokio::fs::write(memory_root.join("nested").join("hidden.md"), "hidden")
.await
.expect("write nested file");
let listed = ai_list_memory_files(ListMemoryFilesRequest {
relative_dir: String::new(),
})
.await
.expect("list should succeed");
let listed_data = listed.value.data.expect("list data");
assert_eq!(listed_data.relative_dir, "");
assert_eq!(listed_data.files, vec!["a.md", "b.md"]);
assert_eq!(listed_data.count, 2);
}
#[tokio::test]
async fn list_memory_files_rejects_non_directory_target() {
let _guard = env_mutex()
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let tmp = TempDir::new().expect("tempdir");
let _workspace = WorkspaceEnvGuard::set(tmp.path());
tokio::fs::create_dir_all(tmp.path().join("memory"))
.await
.expect("create memory root");
tokio::fs::write(tmp.path().join("memory").join("single.md"), "hello")
.await
.expect("write file");
let err = ai_list_memory_files(ListMemoryFilesRequest {
relative_dir: "single.md".to_string(),
})
.await
.expect_err("listing a file path should fail");
assert!(
err.contains("memory directory not found") || err.contains("resolve memory path"),
"unexpected error: {err}"
);
}
#[tokio::test]
async fn read_and_write_memory_files_reject_path_traversal() {
let _guard = env_mutex()
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let tmp = TempDir::new().expect("tempdir");
let _workspace = WorkspaceEnvGuard::set(tmp.path());
let write_err = ai_write_memory_file(WriteMemoryFileRequest {
relative_path: "../secrets.txt".to_string(),
content: "nope".to_string(),
})
.await
.expect_err("path traversal should fail for writes");
assert!(write_err.contains("path traversal is not allowed"));
let read_err = ai_read_memory_file(ReadMemoryFileRequest {
relative_path: "../secrets.txt".to_string(),
})
.await
.expect_err("path traversal should fail for reads");
assert!(read_err.contains("path traversal is not allowed"));
}
#[tokio::test]
async fn list_and_read_memory_files_reject_absolute_paths() {
let _guard = env_mutex()
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let tmp = TempDir::new().expect("tempdir");
let _workspace = WorkspaceEnvGuard::set(tmp.path());
let list_err = ai_list_memory_files(ListMemoryFilesRequest {
relative_dir: "/tmp".to_string(),
})
.await
.expect_err("absolute list path should fail");
assert!(list_err.contains("absolute paths are not allowed"));
let read_err = ai_read_memory_file(ReadMemoryFileRequest {
relative_path: "/tmp/secret.txt".to_string(),
})
.await
.expect_err("absolute read path should fail");
assert!(read_err.contains("absolute paths are not allowed"));
let write_err = ai_write_memory_file(WriteMemoryFileRequest {
relative_path: "/tmp/secret.txt".to_string(),
content: "nope".to_string(),
})
.await
.expect_err("absolute write path should fail");
assert!(write_err.contains("absolute paths are not allowed"));
}
#[tokio::test]
async fn read_memory_file_surfaces_missing_file_error() {
let _guard = env_mutex()
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let tmp = TempDir::new().expect("tempdir");
let _workspace = WorkspaceEnvGuard::set(tmp.path());
let err = ai_read_memory_file(ReadMemoryFileRequest {
relative_path: "missing.md".to_string(),
})
.await
.expect_err("missing file should fail");
assert!(
err.contains("resolve memory path") || err.contains("read memory file"),
"unexpected error: {err}"
);
}
#[tokio::test]
async fn read_memory_file_surfaces_invalid_utf8_error() {
let _guard = env_mutex()
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let tmp = TempDir::new().expect("tempdir");
let _workspace = WorkspaceEnvGuard::set(tmp.path());
let memory_root = super::super::helpers::resolve_existing_memory_path("")
.await
.expect("resolve memory root");
tokio::fs::write(memory_root.join("binary.bin"), [0xff, 0xfe, 0xfd])
.await
.expect("write invalid utf8 file");
let err = ai_read_memory_file(ReadMemoryFileRequest {
relative_path: "binary.bin".to_string(),
})
.await
.expect_err("invalid utf8 file should fail");
assert!(err.contains("read memory file"));
}
#[cfg(unix)]
#[tokio::test]
async fn write_memory_file_rejects_symlink_targets() {
use std::os::unix::fs::symlink;
let _guard = env_mutex()
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let tmp = TempDir::new().expect("tempdir");
let _workspace = WorkspaceEnvGuard::set(tmp.path());
let memory_root = super::super::helpers::resolve_existing_memory_path("")
.await
.expect("resolve memory root");
let real = memory_root.join("real.md");
tokio::fs::write(&real, "hello")
.await
.expect("write real file");
symlink(&real, memory_root.join("alias.md")).expect("create symlink");
let err = ai_write_memory_file(WriteMemoryFileRequest {
relative_path: "alias.md".to_string(),
content: "mutate".to_string(),
})
.await
.expect_err("writing through symlink should fail");
assert!(err.contains("refusing to write through symlink"));
}
#[cfg(unix)]
#[tokio::test]
async fn list_memory_files_skips_symlink_entries() {
use std::os::unix::fs::symlink;
let _guard = env_mutex()
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let tmp = TempDir::new().expect("tempdir");
let _workspace = WorkspaceEnvGuard::set(tmp.path());
let memory_root = super::super::helpers::resolve_existing_memory_path("")
.await
.expect("resolve memory root");
let real = memory_root.join("real.md");
tokio::fs::write(&real, "hello")
.await
.expect("write real file");
symlink(&real, memory_root.join("alias.md")).expect("create symlink");
let listed = ai_list_memory_files(ListMemoryFilesRequest {
relative_dir: String::new(),
})
.await
.expect("list should succeed");
let listed_data = listed.value.data.expect("list data");
assert_eq!(listed_data.files, vec!["real.md"]);
}
}
+75 -1
View File
@@ -9,12 +9,12 @@ use serde::Deserialize;
use serde_json::{json, Value};
use crate::openhuman::config::Config;
use crate::openhuman::memory::store::GraphRelationRecord;
use crate::openhuman::memory::{
MemoryClient, MemoryClientRef, MemoryDocumentSummary, MemoryItemKind, MemoryRetrievalChunk,
MemoryRetrievalContext, MemoryRetrievalEntity, MemoryRetrievalRelation, NamespaceMemoryHit,
QueryNamespaceRequest,
};
use crate::openhuman::memory_store::GraphRelationRecord;
// ---------------------------------------------------------------------------
// Formatting helpers
@@ -225,6 +225,80 @@ pub(crate) fn format_llm_context_message(
Some(parts.join("\n\n"))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::openhuman::memory::RetrievalScoreBreakdown;
fn sample_hit(kind: MemoryItemKind) -> NamespaceMemoryHit {
NamespaceMemoryHit {
id: "hit-1".into(),
kind,
namespace: "global".into(),
key: "note-1".into(),
title: Some("Title".into()),
content: "Body text".into(),
category: "core".into(),
source_type: Some("manual".into()),
updated_at: 1.5,
score: 0.7,
score_breakdown: RetrievalScoreBreakdown::default(),
document_id: Some("doc-1".into()),
chunk_id: Some("chunk-1".into()),
supporting_relations: vec![GraphRelationRecord {
namespace: Some("global".into()),
subject: "Alice".into(),
predicate: "OWNS".into(),
object: "OpenHuman".into(),
attrs: json!({"entity_types": {"subject": "PERSON", "object": "PRODUCT"}}),
updated_at: 2.0,
evidence_count: 1,
order_index: Some(0),
document_ids: vec!["doc-1".into()],
chunk_ids: vec!["chunk-1".into()],
}],
}
}
#[test]
fn timestamp_to_rfc3339_rejects_invalid_values() {
assert!(timestamp_to_rfc3339(f64::NAN).is_none());
assert!(timestamp_to_rfc3339(f64::INFINITY).is_none());
assert!(timestamp_to_rfc3339(-1.0).is_none());
assert!(timestamp_to_rfc3339(1.5).is_some());
}
#[test]
fn relation_identity_and_metadata_include_namespace_and_attrs() {
let relation = sample_hit(MemoryItemKind::Document)
.supporting_relations
.remove(0);
assert_eq!(relation_identity(&relation), "global|Alice|OWNS|OpenHuman");
let meta = relation_metadata(&relation);
assert_eq!(meta["namespace"], "global");
assert_eq!(meta["attrs"]["entity_types"]["subject"], "PERSON");
}
#[test]
fn build_retrieval_context_deduplicates_relations_and_entities() {
let hit = sample_hit(MemoryItemKind::Document);
let ctx = build_retrieval_context(&[hit.clone(), hit]);
assert_eq!(ctx.chunks.len(), 2);
assert_eq!(ctx.relations.len(), 1);
assert!(ctx.entities.iter().any(|e| e.name == "Alice"));
assert!(ctx.entities.iter().any(|e| e.name == "OpenHuman"));
}
#[test]
fn format_llm_context_message_includes_query_and_relation_text() {
let hit = sample_hit(MemoryItemKind::Document);
let text = format_llm_context_message(Some("who owns it"), &[hit]).unwrap();
assert!(text.contains("Query: who owns it"));
assert!(text.contains("Title: Body text"));
assert!(text.contains("Alice (PERSON) -[OWNS]-> OpenHuman (PRODUCT)"));
}
}
/// Filters memory hits to only include those matching specific document IDs.
pub(crate) fn filter_hits_by_document_ids(
hits: Vec<NamespaceMemoryHit>,
+108
View File
@@ -134,3 +134,111 @@ pub async fn graph_query(
.await?;
Ok(RpcOutcome::single_log(rows, "memory graph queried"))
}
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use std::sync::OnceLock;
use tempfile::TempDir;
use super::*;
fn ensure_memory_client() {
static WORKSPACE: OnceLock<PathBuf> = OnceLock::new();
let workspace = WORKSPACE.get_or_init(|| {
let tmp = TempDir::new().expect("tempdir");
let path = tmp.path().join("workspace");
std::fs::create_dir_all(&path).expect("workspace dir");
std::mem::forget(tmp);
path
});
let _ = crate::openhuman::memory::global::init(workspace.clone());
}
fn unique_namespace(prefix: &str) -> String {
format!("{prefix}-{}", uuid::Uuid::new_v4())
}
#[tokio::test]
async fn kv_handlers_roundtrip_scoped_values() {
ensure_memory_client();
let namespace = unique_namespace("kv-graph-kv");
let key = format!("state-{}", uuid::Uuid::new_v4());
let set = kv_set(KvSetParams {
namespace: Some(namespace.clone()),
key: key.clone(),
value: serde_json::json!({"open": true}),
})
.await
.expect("kv set");
assert!(set.value);
let get = kv_get(KvGetDeleteParams {
namespace: Some(namespace.clone()),
key: key.clone(),
})
.await
.expect("kv get");
assert_eq!(get.value, Some(serde_json::json!({"open": true})));
let listed = kv_list_namespace(super::super::documents::NamespaceOnlyParams {
namespace: namespace.clone(),
})
.await
.expect("kv list namespace");
assert!(listed
.value
.iter()
.any(|row| row["key"] == key && row["value"] == serde_json::json!({"open": true})));
let deleted = kv_delete(KvGetDeleteParams {
namespace: Some(namespace.clone()),
key: key.clone(),
})
.await
.expect("kv delete");
assert!(deleted.value);
let after = kv_get(KvGetDeleteParams {
namespace: Some(namespace),
key,
})
.await
.expect("kv get after delete");
assert!(after.value.is_none());
}
#[tokio::test]
async fn graph_handlers_roundtrip_relation_rows() {
ensure_memory_client();
let namespace = unique_namespace("kv-graph-rel");
let subject = format!("alice-{}", uuid::Uuid::new_v4());
let upsert = graph_upsert(GraphUpsertParams {
namespace: Some(namespace.clone()),
subject: subject.clone(),
predicate: "OWNS".into(),
object: "Atlas".into(),
attrs: serde_json::json!({"source": "test", "confidence": 0.9}),
})
.await
.expect("graph upsert");
assert!(upsert.value);
let queried = graph_query(GraphQueryParams {
namespace: Some(namespace),
subject: Some(subject.clone()),
predicate: Some("OWNS".into()),
})
.await
.expect("graph query");
assert_eq!(queried.logs, vec!["memory graph queried".to_string()]);
assert_eq!(queried.value.len(), 1);
assert_eq!(queried.value[0]["subject"], subject.to_uppercase());
assert_eq!(queried.value[0]["predicate"], "OWNS");
assert_eq!(queried.value[0]["object"], "ATLAS");
}
}
+193 -3
View File
@@ -111,9 +111,10 @@ pub async fn memory_learn_all(
"[memory.learn] running summarization for namespace='{}'",
namespace
);
let outcome =
crate::openhuman::memory_tree::summarizer::ops::tree_summarizer_run(&config, namespace)
.await;
let outcome = crate::openhuman::memory_tree::tree_runtime::ops::tree_summarizer_run(
&config, namespace,
)
.await;
match outcome {
Ok(_) => {
tracing::info!("[memory.learn] namespace='{}' ok", namespace);
@@ -151,3 +152,192 @@ pub async fn memory_learn_all(
vec![],
))
}
#[cfg(test)]
mod tests {
use std::ffi::OsString;
use std::path::PathBuf;
use std::sync::OnceLock;
use serde_json::json;
use tempfile::TempDir;
use super::*;
use crate::openhuman::memory_store::NamespaceDocumentInput;
fn ensure_memory_client() {
static WORKSPACE: OnceLock<PathBuf> = OnceLock::new();
let workspace = WORKSPACE.get_or_init(|| {
let tmp = TempDir::new().expect("tempdir");
let path = tmp.path().join("workspace");
std::fs::create_dir_all(&path).expect("workspace dir");
std::mem::forget(tmp);
path
});
let _ = crate::openhuman::memory::global::init(workspace.clone());
}
struct WorkspaceEnvGuard {
_lock: std::sync::MutexGuard<'static, ()>,
previous: Option<OsString>,
}
impl WorkspaceEnvGuard {
fn set(path: &std::path::Path) -> Self {
let lock = crate::openhuman::config::TEST_ENV_LOCK.lock().unwrap();
let previous = std::env::var_os("OPENHUMAN_WORKSPACE");
std::env::set_var("OPENHUMAN_WORKSPACE", path);
Self {
_lock: lock,
previous,
}
}
}
impl Drop for WorkspaceEnvGuard {
fn drop(&mut self) {
if let Some(previous) = self.previous.as_ref() {
std::env::set_var("OPENHUMAN_WORKSPACE", previous);
} else {
std::env::remove_var("OPENHUMAN_WORKSPACE");
}
}
}
async fn seed_namespace(prefix: &str) -> String {
ensure_memory_client();
let namespace = format!("{prefix}-{}", uuid::Uuid::new_v4());
let client = crate::openhuman::memory::global::client().expect("memory client");
client
.put_doc_light(NamespaceDocumentInput {
namespace: namespace.clone(),
key: format!("key-{}", uuid::Uuid::new_v4()),
title: "Test".into(),
content: "Seed content".into(),
source_type: "doc".into(),
priority: "normal".into(),
tags: vec!["test".into()],
metadata: json!({"source": "test"}),
category: "core".into(),
session_id: None,
document_id: None,
})
.await
.expect("seed namespace doc");
namespace
}
async fn write_config_with_runtime_enabled(
workspace_root: &std::path::Path,
runtime_enabled: bool,
) {
let _guard = WorkspaceEnvGuard::set(workspace_root);
let mut config = crate::openhuman::config::Config::load_or_init()
.await
.expect("load config");
config.local_ai.runtime_enabled = runtime_enabled;
config.save().await.expect("save config");
}
#[tokio::test]
async fn memory_learn_all_is_noop_for_explicit_empty_namespace_list() {
ensure_memory_client();
let outcome = memory_learn_all(LearnAllParams {
namespaces: Some(vec![]),
})
.await
.expect("empty list should early-return");
assert_eq!(outcome.value.namespaces_processed, 0);
assert!(outcome.value.results.is_empty());
assert!(outcome.logs.is_empty());
}
#[tokio::test]
async fn memory_learn_all_is_noop_when_requested_namespaces_do_not_exist() {
ensure_memory_client();
let missing = format!("missing-{}", uuid::Uuid::new_v4());
let outcome = memory_learn_all(LearnAllParams {
namespaces: Some(vec![missing]),
})
.await
.expect("unknown namespaces should filter to no-op");
assert_eq!(outcome.value.namespaces_processed, 0);
assert!(outcome.value.results.is_empty());
}
#[tokio::test]
async fn memory_learn_all_filters_missing_namespaces_and_dedupes_requested_order() {
let namespace_a = seed_namespace("memory-learn-a").await;
let namespace_b = seed_namespace("memory-learn-b").await;
let missing = format!("missing-{}", uuid::Uuid::new_v4());
let tmp = TempDir::new().expect("tempdir");
write_config_with_runtime_enabled(tmp.path(), true).await;
let _workspace = WorkspaceEnvGuard::set(tmp.path());
let outcome = memory_learn_all(LearnAllParams {
namespaces: Some(vec![
missing,
namespace_b.clone(),
namespace_a.clone(),
namespace_b.clone(),
]),
})
.await
.expect("existing namespaces with runtime enabled should run");
assert_eq!(outcome.value.namespaces_processed, 2);
assert_eq!(outcome.value.results.len(), 2);
assert_eq!(outcome.value.results[0].namespace, namespace_b);
assert_eq!(outcome.value.results[1].namespace, namespace_a);
assert!(outcome.value.results.iter().all(|r| r.status == "ok"));
assert!(outcome.value.results.iter().all(|r| r.error.is_none()));
}
#[tokio::test]
async fn memory_learn_all_requires_local_ai_once_existing_namespace_is_selected() {
let namespace = seed_namespace("memory-learn-runtime").await;
let tmp = TempDir::new().expect("tempdir");
write_config_with_runtime_enabled(tmp.path(), false).await;
let _workspace = WorkspaceEnvGuard::set(tmp.path());
let err = memory_learn_all(LearnAllParams {
namespaces: Some(vec![namespace]),
})
.await
.expect_err("runtime-disabled config should hard-fail");
assert!(err.contains("memory_learn_all requires local_ai.runtime_enabled=true"));
}
#[tokio::test]
async fn memory_learn_all_uses_all_namespaces_when_none_is_requested() {
let namespace_a = seed_namespace("memory-learn-all-a").await;
let namespace_b = seed_namespace("memory-learn-all-b").await;
let tmp = TempDir::new().expect("tempdir");
write_config_with_runtime_enabled(tmp.path(), true).await;
let _workspace = WorkspaceEnvGuard::set(tmp.path());
let outcome = memory_learn_all(LearnAllParams { namespaces: None })
.await
.expect("runtime-enabled config should process all namespaces");
assert!(
outcome.value.namespaces_processed >= 2,
"expected at least the two seeded namespaces to be processed"
);
let namespaces: std::collections::BTreeSet<_> = outcome
.value
.results
.iter()
.map(|r| r.namespace.as_str())
.collect();
assert!(namespaces.contains(namespace_a.as_str()));
assert!(namespaces.contains(namespace_b.as_str()));
assert!(outcome
.value
.results
.iter()
.filter(|r| r.namespace == namespace_a || r.namespace == namespace_b)
.all(|r| r.status == "ok" && r.error.is_none()));
}
}
+160
View File
@@ -110,3 +110,163 @@ pub async fn memory_ingestion_status() -> Result<RpcOutcome<IngestionStatusResul
vec![],
))
}
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use std::sync::{Arc, OnceLock};
use super::*;
use async_trait::async_trait;
use serde_json::json;
use tempfile::TempDir;
use tokio::sync::mpsc;
use tokio::time::{timeout, Duration};
use crate::core::event_bus::{self, DomainEvent, EventHandler};
fn test_mutex() -> &'static std::sync::Mutex<()> {
static LOCK: OnceLock<std::sync::Mutex<()>> = OnceLock::new();
LOCK.get_or_init(|| std::sync::Mutex::new(()))
}
fn ensure_memory_client() -> crate::openhuman::memory::MemoryClientRef {
static WORKSPACE: OnceLock<PathBuf> = OnceLock::new();
let workspace = WORKSPACE.get_or_init(|| {
let tmp = TempDir::new().expect("tempdir");
let path = tmp.path().join("workspace");
std::fs::create_dir_all(&path).expect("workspace dir");
std::mem::forget(tmp);
path
});
crate::openhuman::memory::global::init(workspace.clone()).expect("init memory client")
}
struct ChannelCapture {
tx: mpsc::UnboundedSender<Option<String>>,
}
#[async_trait]
impl EventHandler for ChannelCapture {
fn name(&self) -> &str {
"memory::ops::sync::tests::capture"
}
fn domains(&self) -> Option<&[&str]> {
Some(&["memory"])
}
async fn handle(&self, event: &DomainEvent) {
if let DomainEvent::MemorySyncRequested { channel_id } = event {
let _ = self.tx.send(channel_id.clone());
}
}
}
#[test]
fn sync_channel_params_deserialize_channel_id() {
let params: SyncChannelParams =
serde_json::from_value(json!({"channel_id": "channel-1"})).unwrap();
assert_eq!(params.channel_id, "channel-1");
}
#[test]
fn ingestion_status_result_default_is_idle() {
let status = IngestionStatusResult::default();
assert!(!status.running);
assert!(status.current_document_id.is_none());
assert!(status.current_title.is_none());
assert!(status.current_namespace.is_none());
assert_eq!(status.queue_depth, 0);
assert!(status.last_completed_at.is_none());
assert!(status.last_document_id.is_none());
assert!(status.last_success.is_none());
}
#[test]
fn sync_result_structs_serialize_expected_fields() {
let one = serde_json::to_value(SyncChannelResult {
requested: true,
channel_id: "abc".into(),
})
.unwrap();
assert_eq!(one, json!({"requested": true, "channel_id": "abc"}));
let all = serde_json::to_value(SyncAllResult { requested: true }).unwrap();
assert_eq!(all, json!({"requested": true}));
}
#[tokio::test]
async fn memory_sync_channel_publishes_targeted_event() {
let _guard = test_mutex()
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
event_bus::init_global(event_bus::DEFAULT_CAPACITY);
let (tx, mut rx) = mpsc::unbounded_channel();
let _subscription = event_bus::subscribe_global(Arc::new(ChannelCapture { tx }))
.expect("global bus should be initialized");
let outcome = memory_sync_channel(SyncChannelParams {
channel_id: "channel-123".into(),
})
.await
.expect("memory_sync_channel");
assert!(outcome.value.requested);
assert_eq!(outcome.value.channel_id, "channel-123");
let received = timeout(Duration::from_secs(1), rx.recv())
.await
.expect("event should arrive before timeout")
.expect("sender should still be connected");
assert_eq!(received.as_deref(), Some("channel-123"));
}
#[tokio::test]
async fn memory_sync_all_publishes_broadcast_event() {
let _guard = test_mutex()
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
event_bus::init_global(event_bus::DEFAULT_CAPACITY);
let (tx, mut rx) = mpsc::unbounded_channel();
let _subscription = event_bus::subscribe_global(Arc::new(ChannelCapture { tx }))
.expect("global bus should be initialized");
let outcome = memory_sync_all().await.expect("memory_sync_all");
assert!(outcome.value.requested);
let received = timeout(Duration::from_secs(1), rx.recv())
.await
.expect("event should arrive before timeout")
.expect("sender should still be connected");
assert!(
received.is_none(),
"sync-all should publish channel_id=None"
);
}
#[tokio::test]
async fn memory_ingestion_status_reflects_initialized_client_snapshot() {
let _guard = test_mutex()
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let client = ensure_memory_client();
let state = client.ingestion_state();
state.enqueue();
state.mark_running("doc-sync", "Sync Title", "sync-test");
let status = memory_ingestion_status()
.await
.expect("memory_ingestion_status")
.value;
assert!(status.running);
assert_eq!(status.current_document_id.as_deref(), Some("doc-sync"));
assert_eq!(status.current_title.as_deref(), Some("Sync Title"));
assert_eq!(status.current_namespace.as_deref(), Some("sync-test"));
assert_eq!(status.queue_depth, 1);
state.dequeue();
state.mark_completed("doc-sync", true, 12345);
}
}
+159
View File
@@ -172,3 +172,162 @@ pub async fn tool_rules_json(params: ToolRuleListParams) -> Result<RpcOutcome<Va
let value = store.list_rules_json(&params.tool_name).await?;
Ok(RpcOutcome::single_log(value, "tool memory rules json"))
}
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use std::sync::OnceLock;
use tempfile::TempDir;
use super::*;
use crate::openhuman::memory::ToolMemoryPriority;
fn ensure_memory_client() {
static WORKSPACE: OnceLock<PathBuf> = OnceLock::new();
let workspace = WORKSPACE.get_or_init(|| {
let tmp = TempDir::new().expect("tempdir");
let path = tmp.path().join("workspace");
std::fs::create_dir_all(&path).expect("workspace dir");
std::mem::forget(tmp);
path
});
let _ = crate::openhuman::memory::global::init(workspace.clone());
}
fn unique_tool_name() -> String {
format!("tool-memory-{}", uuid::Uuid::new_v4())
}
#[tokio::test]
async fn tool_rule_put_get_list_and_delete_roundtrip() {
ensure_memory_client();
let tool_name = unique_tool_name();
let stored = tool_rule_put(ToolRulePutParams {
tool_name: tool_name.clone(),
rule: "Always ask before sending emails".into(),
priority: None,
source: None,
tags: vec!["safety".into()],
id: Some(" ".into()),
})
.await
.expect("tool rule put")
.value;
assert_eq!(stored.tool_name, tool_name);
assert_eq!(stored.priority, ToolMemoryPriority::Normal);
assert_eq!(
stored.source,
crate::openhuman::memory::ToolMemorySource::Programmatic
);
assert_eq!(stored.tags, vec!["safety".to_string()]);
assert!(
!stored.id.trim().is_empty(),
"blank id should be regenerated"
);
let fetched = tool_rule_get(ToolRuleRefParams {
tool_name: stored.tool_name.clone(),
id: stored.id.clone(),
})
.await
.expect("tool rule get")
.value
.expect("stored rule should exist");
assert_eq!(fetched.rule, "Always ask before sending emails");
let listed = tool_rule_list(ToolRuleListParams {
tool_name: stored.tool_name.clone(),
})
.await
.expect("tool rule list")
.value;
assert_eq!(listed.len(), 1);
assert_eq!(listed[0].id, stored.id);
let deleted = tool_rule_delete(ToolRuleRefParams {
tool_name: stored.tool_name.clone(),
id: stored.id.clone(),
})
.await
.expect("tool rule delete")
.value;
assert!(deleted);
let after = tool_rule_get(ToolRuleRefParams {
tool_name: stored.tool_name,
id: stored.id,
})
.await
.expect("tool rule get after delete");
assert!(after.value.is_none());
}
#[tokio::test]
async fn tool_rules_for_prompt_sorts_by_priority_and_tool_name() {
ensure_memory_client();
let primary_tool = unique_tool_name();
let secondary_tool = unique_tool_name();
let high = tool_rule_put(ToolRulePutParams {
tool_name: primary_tool.clone(),
rule: "Use the dry-run mode first".into(),
priority: Some(ToolMemoryPriority::High),
source: None,
tags: vec![],
id: None,
})
.await
.expect("put high")
.value;
let normal = tool_rule_put(ToolRulePutParams {
tool_name: secondary_tool.clone(),
rule: "Log the final command".into(),
priority: Some(ToolMemoryPriority::Normal),
source: None,
tags: vec![],
id: None,
})
.await
.expect("put normal")
.value;
let prompt = tool_rules_for_prompt(ToolRulesForPromptParams {
tools: vec![secondary_tool.clone(), primary_tool.clone()],
})
.await
.expect("rules for prompt")
.value;
assert_eq!(prompt.rules.len(), 1, "only eager rules should be included");
assert_eq!(prompt.rules[0].id, high.id);
assert!(prompt.rendered.contains(&primary_tool));
assert!(prompt.rendered.contains("Use the dry-run mode first"));
let json_rules = tool_rules_json(ToolRuleListParams {
tool_name: secondary_tool.clone(),
})
.await
.expect("tool rules json")
.value;
assert!(json_rules.is_array(), "tool rules json should be an array");
assert!(json_rules
.as_array()
.expect("array")
.iter()
.any(|row| row["rule"] == "Log the final command"));
let _ = tool_rule_delete(ToolRuleRefParams {
tool_name: primary_tool,
id: high.id,
})
.await;
let _ = tool_rule_delete(ToolRuleRefParams {
tool_name: secondary_tool,
id: normal.id,
})
.await;
}
}
+1 -1
View File
@@ -4,8 +4,8 @@
use serde_json::json;
use super::{build_retrieval_context, filter_hits_by_document_ids, format_llm_context_message};
use crate::openhuman::memory::store::GraphRelationRecord;
use crate::openhuman::memory::{MemoryItemKind, NamespaceMemoryHit, RetrievalScoreBreakdown};
use crate::openhuman::memory_store::GraphRelationRecord;
fn sample_hit() -> NamespaceMemoryHit {
NamespaceMemoryHit {
@@ -31,11 +31,11 @@ use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use crate::openhuman::config::Config;
use crate::openhuman::memory_tree::content_store::read as content_read;
use crate::openhuman::memory_tree::retrieval::types::NodeKind;
use crate::openhuman::memory_tree::score::store as score_store;
use crate::openhuman::memory_tree::store::{self as chunk_store, with_connection};
use crate::openhuman::memory_tree::types::SourceKind;
use crate::openhuman::memory::retrieval::types::NodeKind;
use crate::openhuman::memory::score::store as score_store;
use crate::openhuman::memory_store::chunks::store::{self as chunk_store, with_connection};
use crate::openhuman::memory_store::chunks::types::SourceKind;
use crate::openhuman::memory_store::content::read as content_read;
use crate::rpc::RpcOutcome;
const PREVIEW_MAX_CHARS: usize = 500;
@@ -46,7 +46,7 @@ const MAX_LIST_LIMIT: u32 = 1_000;
/// Wire-shape chunk returned by the read RPCs.
///
/// Distinct from [`crate::openhuman::memory_tree::types::Chunk`] in two
/// Distinct from [`crate::openhuman::memory_store::chunks::types::Chunk`] in two
/// ways: serialised timestamps are ms-since-epoch (matches the rest of the
/// JSON-RPC surface) and the body is replaced with a `≤500-char preview`
/// + a flag indicating whether the row has an embedding. UIs needing the
@@ -462,7 +462,7 @@ pub async fn recall_rpc(
// Reuse the source-tree retrieval path which already does cosine
// rerank against query embeddings. We pull more summaries than `k`
// because each summary expands into multiple leaves.
let resp = crate::openhuman::memory_tree::retrieval::query_source(
let resp = crate::openhuman::memory::retrieval::query_source(
config,
None,
None,
@@ -780,7 +780,7 @@ pub async fn chunk_score_rpc(
ScoreBreakdown {
signals,
total: r.total,
threshold: crate::openhuman::memory_tree::score::DEFAULT_DROP_THRESHOLD,
threshold: crate::openhuman::memory::score::DEFAULT_DROP_THRESHOLD,
kept: !r.dropped,
llm_consulted,
}
@@ -843,7 +843,7 @@ pub async fn delete_chunk_rpc(
if e.kind() != std::io::ErrorKind::NotFound {
log::warn!(
"[memory_tree::read::delete] failed to remove chunk file path_hash={}: {e}",
crate::openhuman::memory_tree::util::redact::redact(&rel),
crate::openhuman::memory::util::redact::redact(&rel),
);
}
}
@@ -1001,7 +1001,7 @@ pub async fn graph_export_rpc(
mode,
resp.nodes.len(),
resp.edges.len(),
crate::openhuman::memory_tree::util::redact::redact(&resp.content_root_abs),
crate::openhuman::memory::util::redact::redact(&resp.content_root_abs),
);
Ok(RpcOutcome::single_log(resp, log))
}
@@ -1371,7 +1371,7 @@ pub async fn wipe_all_rpc(config: &Config) -> Result<RpcOutcome<WipeAllResponse>
/// keyed under [`crate::openhuman::composio::providers::sync_state::KV_NAMESPACE`].
///
/// We open the SQLite file directly rather than going through
/// [`crate::openhuman::memory::store::client::MemoryClientRef`] so
/// [`crate::openhuman::memory_store::client::MemoryClientRef`] so
/// `wipe_all` stays a pure synchronous operation runnable from
/// `spawn_blocking` without dragging in the full memory-store init
/// path. The `kv_namespace` table is created up-front by
@@ -1430,8 +1430,8 @@ pub struct ResetTreeResponse {
/// outside `spawn_blocking`) so the on-disk removal can use
/// async retry without blocking the worker thread.
pub async fn reset_tree_rpc(config: &Config) -> Result<RpcOutcome<ResetTreeResponse>, String> {
use crate::openhuman::memory_tree::jobs::store as jobs_store;
use crate::openhuman::memory_tree::jobs::types::{ExtractChunkPayload, NewJob};
use crate::openhuman::memory::jobs::store as jobs_store;
use crate::openhuman::memory::jobs::types::{ExtractChunkPayload, NewJob};
let cfg = config.clone();
let (tree_rows_deleted, chunks_requeued, jobs_enqueued) =
@@ -1535,7 +1535,7 @@ pub async fn reset_tree_rpc(config: &Config) -> Result<RpcOutcome<ResetTreeRespo
// Wake the worker pool. Done after the on-disk cleanup so jobs don't
// start racing against an in-progress directory removal; the small
// delay (at most the retry window on Windows) is acceptable.
crate::openhuman::memory_tree::jobs::wake_workers();
crate::openhuman::memory::jobs::wake_workers();
let resp = ResetTreeResponse {
tree_rows_deleted,
@@ -1579,9 +1579,9 @@ pub struct FlushNowResponse {
/// where `<block>` is the current 3-hour UTC block (0..=7), so
/// spamming the button within the same window doesn't queue duplicates.
pub async fn flush_now_rpc(config: &Config) -> Result<RpcOutcome<FlushNowResponse>, String> {
use crate::openhuman::memory_tree::jobs::store as jobs_store;
use crate::openhuman::memory_tree::jobs::types::{FlushStalePayload, NewJob};
use crate::openhuman::memory_tree::tree_source::store as tree_store;
use crate::openhuman::memory::jobs::store as jobs_store;
use crate::openhuman::memory::jobs::types::{FlushStalePayload, NewJob};
use crate::openhuman::memory_tree::tree::store as tree_store;
let cfg = config.clone();
let resp = tokio::task::spawn_blocking(move || -> Result<FlushNowResponse> {
@@ -1827,9 +1827,14 @@ fn parse_source_kind_str(s: &str) -> Option<SourceKind> {
#[cfg(test)]
mod tests {
use super::*;
use crate::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage};
use crate::openhuman::memory_tree::ingest::ingest_chat;
use crate::openhuman::composio::providers::sync_state::KV_NAMESPACE;
use crate::openhuman::embeddings::NoopEmbedding;
use crate::openhuman::memory::ingest_pipeline::ingest_chat;
use crate::openhuman::memory_store::unified::UnifiedMemory;
use crate::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage};
use chrono::{TimeZone, Utc};
use rusqlite::params;
use std::sync::Arc;
use tempfile::TempDir;
fn test_config() -> (TempDir, Config) {
@@ -1866,6 +1871,45 @@ mod tests {
.unwrap();
}
fn update_chunk_timestamp(cfg: &Config, chunk_id: &str, timestamp_ms: i64) {
with_connection(cfg, |conn| {
conn.execute(
"UPDATE mem_tree_chunks
SET timestamp_ms = ?1,
time_range_start_ms = ?1,
time_range_end_ms = ?1
WHERE id = ?2",
params![timestamp_ms, chunk_id],
)?;
Ok(())
})
.unwrap();
}
fn insert_raw_chunk(
cfg: &Config,
id: &str,
source_kind: &str,
source_id: &str,
timestamp_ms: i64,
tags_json: &str,
content: &str,
token_count: i64,
) {
with_connection(cfg, |conn| {
conn.execute(
"INSERT INTO mem_tree_chunks (
id, source_kind, source_id, source_ref, owner, timestamp_ms,
time_range_start_ms, time_range_end_ms, tags_json, content,
token_count, seq_in_source, created_at_ms, lifecycle_status, content_path
) VALUES (?1, ?2, ?3, NULL, 'tester', ?4, ?4, ?4, ?5, ?6, ?7, 0, ?4, 'seeded', NULL)",
params![id, source_kind, source_id, timestamp_ms, tags_json, content, token_count],
)?;
Ok(())
})
.unwrap();
}
#[tokio::test]
async fn list_chunks_returns_seeded_chunk() {
let (_tmp, cfg) = test_config();
@@ -1912,11 +1956,143 @@ mod tests {
.await
.unwrap()
.value;
assert!(resp.chunks.iter().any(|c| c
.content_preview
.as_deref()
.unwrap_or("")
.contains("phoenix")));
assert!(resp.chunks.iter().any(|c| {
c.content_preview
.as_deref()
.unwrap_or("")
.contains("phoenix")
}));
}
#[tokio::test]
async fn list_chunks_filters_by_source_kind_and_applies_limit_offset() {
let (_tmp, cfg) = test_config();
seed_chat_chunk(&cfg, "slack:#a", "first chat").await;
seed_chat_chunk(&cfg, "slack:#b", "second chat").await;
let filtered = list_chunks_rpc(
&cfg,
ChunkFilter {
source_kinds: Some(vec!["chat".into()]),
limit: Some(1),
offset: Some(1),
..ChunkFilter::default()
},
)
.await
.unwrap()
.value;
assert_eq!(filtered.chunks.len(), 1);
assert_eq!(filtered.total, 2);
assert!(filtered.chunks.iter().all(|c| c.source_kind == "chat"));
}
#[tokio::test]
async fn list_chunks_filters_by_entity_id_and_time_window() {
let (_tmp, cfg) = test_config();
seed_chat_chunk(&cfg, "slack:#eng", "alice@example.com handles phoenix").await;
seed_chat_chunk(&cfg, "slack:#eng", "bob@example.com handles atlas").await;
let seeded = list_chunks_rpc(&cfg, ChunkFilter::default())
.await
.unwrap()
.value
.chunks;
let alice = seeded
.iter()
.find(|chunk| {
chunk
.content_preview
.as_deref()
.unwrap_or("")
.contains("alice@example.com")
})
.expect("alice chunk present");
let bob = seeded
.iter()
.find(|chunk| {
chunk
.content_preview
.as_deref()
.unwrap_or("")
.contains("bob@example.com")
})
.expect("bob chunk present");
update_chunk_timestamp(&cfg, &alice.id, 1_700_000_000_100);
update_chunk_timestamp(&cfg, &bob.id, 1_700_000_000_900);
let filtered = list_chunks_rpc(
&cfg,
ChunkFilter {
entity_ids: Some(vec!["email:alice@example.com".into()]),
since_ms: Some(1_700_000_000_000),
until_ms: Some(1_700_000_000_500),
..ChunkFilter::default()
},
)
.await
.unwrap()
.value;
assert_eq!(filtered.total, 1);
assert_eq!(filtered.chunks.len(), 1);
assert_eq!(filtered.chunks[0].id, alice.id);
}
#[tokio::test]
async fn list_chunks_ignores_empty_filter_lists_and_blank_query() {
let (_tmp, cfg) = test_config();
seed_chat_chunk(&cfg, "slack:#a", "alpha").await;
seed_chat_chunk(&cfg, "slack:#b", "beta").await;
let resp = list_chunks_rpc(
&cfg,
ChunkFilter {
source_kinds: Some(vec![]),
source_ids: Some(vec![]),
entity_ids: Some(vec![]),
query: Some(" ".into()),
limit: Some(10),
..ChunkFilter::default()
},
)
.await
.unwrap()
.value;
assert_eq!(resp.total, 2);
assert_eq!(resp.chunks.len(), 2);
}
#[tokio::test]
async fn list_chunks_normalizes_invalid_tags_negative_tokens_and_empty_content() {
let (_tmp, cfg) = test_config();
insert_raw_chunk(
&cfg,
"raw-empty",
"document",
"notion:page-1",
1_700_000_000_123,
"not-json",
"",
-7,
);
let resp = list_chunks_rpc(&cfg, ChunkFilter::default())
.await
.unwrap()
.value;
let row = resp
.chunks
.into_iter()
.find(|chunk| chunk.id == "raw-empty")
.expect("raw chunk listed");
assert_eq!(row.token_count, 0);
assert_eq!(row.tags, Vec::<String>::new());
assert_eq!(row.content_preview, None);
assert!(!row.has_embedding);
}
#[tokio::test]
@@ -1938,6 +2114,33 @@ mod tests {
assert_eq!(b.chunk_count, 1);
}
#[tokio::test]
async fn list_sources_formats_email_threads_with_trimmed_user_hint() {
let (_tmp, cfg) = test_config();
insert_raw_chunk(
&cfg,
"email-thread",
"email",
"gmail:Alice@Example.com|bob@example.com|carol@example.com",
1_700_000_000_123,
"[]",
"thread body",
12,
);
let sources = list_sources_rpc(&cfg, Some(" alice@example.com ".into()))
.await
.unwrap()
.value;
let source = sources
.iter()
.find(|row| {
row.source_id == "gmail:Alice@Example.com|bob@example.com|carol@example.com"
})
.expect("email thread source present");
assert_eq!(source.display_name, "bob@example.com, carol@example.com");
}
#[tokio::test]
async fn entity_index_for_returns_extracted_entities() {
let (_tmp, cfg) = test_config();
@@ -1956,6 +2159,25 @@ mod tests {
);
}
#[tokio::test]
async fn chunks_for_entity_returns_leaf_chunk_ids_only() {
let (_tmp, cfg) = test_config();
seed_chat_chunk(&cfg, "slack:#eng", "alice@example.com owns it").await;
let chunk_id = list_chunks_rpc(&cfg, ChunkFilter::default())
.await
.unwrap()
.value
.chunks[0]
.id
.clone();
let rows = chunks_for_entity_rpc(&cfg, "email:alice@example.com".into())
.await
.unwrap()
.value;
assert_eq!(rows, vec![chunk_id]);
}
#[tokio::test]
async fn top_entities_returns_most_frequent() {
let (_tmp, cfg) = test_config();
@@ -2030,11 +2252,74 @@ mod tests {
seed_chat_chunk(&cfg, "slack:#eng", "phoenix migration scheduled friday").await;
seed_chat_chunk(&cfg, "slack:#eng", "different unrelated text").await;
let hits = search_rpc(&cfg, "phoenix".into(), 10).await.unwrap().value;
assert!(hits.iter().any(|c| c
assert!(hits.iter().any(|c| {
c.content_preview
.as_deref()
.unwrap_or("")
.contains("phoenix")
}));
}
#[tokio::test]
async fn read_chunk_row_returns_preview_and_metadata() {
let (_tmp, cfg) = test_config();
seed_chat_chunk(
&cfg,
"slack:#eng",
"phoenix migration scheduled friday with context and source refs",
)
.await;
let chunk = list_chunks_rpc(&cfg, ChunkFilter::default())
.await
.unwrap()
.value
.chunks
.into_iter()
.next()
.expect("seeded chunk");
let row = read_chunk_row(&cfg, &chunk.id).unwrap().expect("chunk row");
assert_eq!(row.id, chunk.id);
assert_eq!(row.source_kind, "chat");
assert_eq!(row.source_id, "slack:#eng");
assert_eq!(row.source_ref.as_deref(), Some("slack://x"));
assert_eq!(row.owner, "alice");
assert_eq!(row.lifecycle_status, "pending_extraction");
assert!(row.content_path.is_some());
assert!(row
.content_preview
.as_deref()
.unwrap_or("")
.contains("phoenix")));
.contains("phoenix migration scheduled friday"));
}
#[tokio::test]
async fn read_chunk_row_falls_back_to_sqlite_preview_when_file_missing() {
let (_tmp, cfg) = test_config();
let body = "sqlite preview survives missing file";
seed_chat_chunk(&cfg, "slack:#eng", body).await;
let chunk = list_chunks_rpc(&cfg, ChunkFilter::default())
.await
.unwrap()
.value
.chunks
.into_iter()
.next()
.expect("seeded chunk");
let rel_path = chunk.content_path.clone().expect("content path present");
let abs_path = cfg.memory_tree_content_root().join(rel_path);
std::fs::remove_file(&abs_path).expect("remove chunk file");
let row = read_chunk_row(&cfg, &chunk.id).unwrap().expect("chunk row");
assert_eq!(row.content_path, chunk.content_path);
assert!(row.content_preview.as_deref().unwrap_or("").contains(body));
}
#[test]
fn read_chunk_row_returns_none_for_missing_chunk() {
let (_tmp, cfg) = test_config();
assert!(read_chunk_row(&cfg, "missing-chunk").unwrap().is_none());
}
#[tokio::test]
@@ -2338,8 +2623,92 @@ mod tests {
);
}
#[test]
fn display_name_handles_multiple_participants_and_trimmed_hint() {
let name = display_name_for_source(
"gmail:Alice@Example.com|bob@example.com|carol@example.com",
Some(" alice@example.com "),
);
assert_eq!(name, "bob@example.com, carol@example.com");
}
#[test]
fn display_name_handles_no_prefix() {
assert_eq!(display_name_for_source("loose-id", None), "loose-id");
}
#[test]
fn sanitize_basename_replaces_windows_illegal_characters() {
assert_eq!(
sanitize_basename(r#"chat:slack/#eng\name*?"<>|"#),
"chat-slack-#eng-name------"
);
assert_eq!(sanitize_basename("safe-name.md"), "safe-name.md");
}
#[test]
fn parse_source_kind_str_accepts_known_values_only() {
assert_eq!(parse_source_kind_str("chat"), Some(SourceKind::Chat));
assert_eq!(parse_source_kind_str("email"), Some(SourceKind::Email));
assert_eq!(
parse_source_kind_str("document"),
Some(SourceKind::Document)
);
assert_eq!(parse_source_kind_str("unknown"), None);
}
#[tokio::test]
async fn get_llm_rpc_includes_current_backend_in_value_and_log() {
let (_tmp, mut cfg) = test_config();
cfg.memory_tree.llm_backend = crate::openhuman::config::LlmBackend::Local;
let outcome = get_llm_rpc(&cfg).await.unwrap();
assert_eq!(outcome.value.current, "local");
assert_eq!(
outcome.logs,
vec!["memory_tree::read: get_llm current=local".to_string()]
);
}
#[test]
fn clear_composio_sync_state_removes_only_target_namespace() {
let tmp = TempDir::new().unwrap();
let _memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap();
let db_path = tmp.path().join("memory").join("memory.db");
let conn = rusqlite::Connection::open(&db_path).unwrap();
conn.execute(
"INSERT INTO kv_namespace (namespace, key, value_json, updated_at)
VALUES (?1, 'cursor', '{}', 1.0)",
params![KV_NAMESPACE],
)
.unwrap();
conn.execute(
"INSERT INTO kv_namespace (namespace, key, value_json, updated_at)
VALUES ('other-namespace', 'cursor', '{}', 2.0)",
[],
)
.unwrap();
drop(conn);
let removed = clear_composio_sync_state(&db_path).unwrap();
assert_eq!(removed, 1);
let conn = rusqlite::Connection::open(&db_path).unwrap();
let composio_count: i64 = conn
.query_row(
"SELECT COUNT(*) FROM kv_namespace WHERE namespace = ?1",
params![KV_NAMESPACE],
|row| row.get(0),
)
.unwrap();
let other_count: i64 = conn
.query_row(
"SELECT COUNT(*) FROM kv_namespace WHERE namespace = 'other-namespace'",
[],
|row| row.get(0),
)
.unwrap();
assert_eq!(composio_count, 0);
assert_eq!(other_count, 1);
}
}
@@ -21,13 +21,13 @@ use chrono::{TimeZone, Utc};
use tempfile::TempDir;
use crate::openhuman::config::Config;
use crate::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage};
use crate::openhuman::memory_tree::ingest::ingest_chat;
use crate::openhuman::memory_tree::jobs::testing::drain_until_idle;
use crate::openhuman::memory_tree::retrieval::{
use crate::openhuman::memory::ingest_pipeline::ingest_chat;
use crate::openhuman::memory::jobs::testing::drain_until_idle;
use crate::openhuman::memory::retrieval::{
fetch_leaves, query_source, query_topic, search_entities,
};
use crate::openhuman::memory_tree::types::SourceKind;
use crate::openhuman::memory_store::chunks::types::SourceKind;
use crate::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage};
/// Shared test config — disables embedding for deterministic inert behaviour.
fn bench_config() -> (TempDir, Config) {
@@ -23,13 +23,11 @@ use std::collections::VecDeque;
use anyhow::Result;
use crate::openhuman::config::Config;
use crate::openhuman::memory_tree::content_store::read as content_read;
use crate::openhuman::memory_tree::retrieval::types::{
hit_from_chunk, hit_from_summary, RetrievalHit,
};
use crate::openhuman::memory_tree::score::embed::{build_embedder_from_config, cosine_similarity};
use crate::openhuman::memory_tree::store::{get_chunk, get_chunk_embedding};
use crate::openhuman::memory_tree::tree_source::store;
use crate::openhuman::memory::retrieval::types::{hit_from_chunk, hit_from_summary, RetrievalHit};
use crate::openhuman::memory::score::embed::{build_embedder_from_config, cosine_similarity};
use crate::openhuman::memory_store::chunks::store::{get_chunk, get_chunk_embedding};
use crate::openhuman::memory_store::content::read as content_read;
use crate::openhuman::memory_tree::tree::store;
/// Walk the summary hierarchy down one step (or more if `max_depth > 1`)
/// and return the hydrated child hits. Children at level 1 are raw chunks;
@@ -257,16 +255,17 @@ fn walk_with_embeddings(
#[cfg(test)]
mod tests {
use super::*;
use crate::openhuman::memory_tree::content_store;
use crate::openhuman::memory_tree::store::upsert_chunks;
use crate::openhuman::memory_tree::tree_source::bucket_seal::{
append_leaf, LabelStrategy, LeafRef,
use crate::openhuman::memory::chat::{test_override, ChatProvider, StaticChatProvider};
use crate::openhuman::memory_store::chunks::store::upsert_chunks;
use crate::openhuman::memory_store::chunks::types::{
chunk_id, Chunk, Metadata, SourceKind, SourceRef,
};
use crate::openhuman::memory_tree::tree_source::registry::get_or_create_source_tree;
use crate::openhuman::memory_tree::tree_source::summariser::inert::InertSummariser;
use crate::openhuman::memory_tree::tree_source::types::TreeKind;
use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef};
use crate::openhuman::memory_store::content as content_store;
use crate::openhuman::memory_store::trees::types::TreeKind;
use crate::openhuman::memory_tree::sources::registry::get_or_create_source_tree;
use crate::openhuman::memory_tree::tree::bucket_seal::{append_leaf, LabelStrategy, LeafRef};
use chrono::Utc;
use std::sync::Arc;
use tempfile::TempDir;
fn test_config() -> (TempDir, Config) {
@@ -284,7 +283,8 @@ mod tests {
// Seed two 6k-token leaves so the L0 buffer seals into an L1 node.
let ts = Utc::now();
let tree = get_or_create_source_tree(cfg, "slack:#eng").unwrap();
let summariser = InertSummariser::new();
let provider: Arc<dyn ChatProvider> =
Arc::new(StaticChatProvider::new("test summary content"));
let content_root = cfg.memory_tree_content_root();
std::fs::create_dir_all(&content_root).unwrap();
let mut leaf_ids: Vec<String> = Vec::new();
@@ -301,8 +301,7 @@ mod tests {
tags: vec![],
source_ref: Some(SourceRef::new("slack://x")),
},
token_count: crate::openhuman::memory_tree::tree_source::types::INPUT_TOKEN_BUDGET
* 6
token_count: crate::openhuman::memory_store::trees::types::INPUT_TOKEN_BUDGET * 6
/ 10,
seq_in_source: seq,
created_at: ts,
@@ -312,33 +311,32 @@ mod tests {
// Stage to disk so `hydrate_leaf_inputs` can read the full body
// via `read_chunk_body` during the seal triggered by `append_leaf`.
let staged = content_store::stage_chunks(&content_root, &[c.clone()]).unwrap();
crate::openhuman::memory_tree::store::with_connection(cfg, |conn| {
crate::openhuman::memory_store::chunks::store::with_connection(cfg, |conn| {
let tx = conn.unchecked_transaction()?;
crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?;
crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(
&tx, &staged,
)?;
tx.commit()?;
Ok(())
})
.unwrap();
leaf_ids.push(c.id.clone());
append_leaf(
cfg,
&tree,
&LeafRef {
chunk_id: c.id.clone(),
token_count:
crate::openhuman::memory_tree::tree_source::types::INPUT_TOKEN_BUDGET * 6
/ 10,
timestamp: ts,
content: c.content.clone(),
entities: vec![],
topics: vec![],
score: 0.5,
},
&summariser,
&LabelStrategy::Empty,
)
.await
.unwrap();
let leaf = LeafRef {
chunk_id: c.id.clone(),
token_count: crate::openhuman::memory_store::trees::types::INPUT_TOKEN_BUDGET * 6
/ 10,
timestamp: ts,
content: c.content.clone(),
entities: vec![],
topics: vec![],
score: 0.5,
};
test_override::with_provider(Arc::clone(&provider), async {
append_leaf(cfg, &tree, &leaf, &LabelStrategy::Empty)
.await
.unwrap()
})
.await;
}
// Fetch the sealed L1 summary id from the tree row.
let refreshed = store::get_tree(cfg, &tree.id).unwrap().unwrap();
@@ -430,9 +428,9 @@ mod tests {
// (or similar — the key invariant is that BFS returns all siblings at
// one depth before any descendant at a deeper depth).
use crate::openhuman::memory_tree::store::with_connection;
use crate::openhuman::memory_tree::tree_source::store as tree_store;
use crate::openhuman::memory_tree::tree_source::types::{SummaryNode, Tree, TreeStatus};
use crate::openhuman::memory_store::chunks::store::with_connection;
use crate::openhuman::memory_store::trees::types::{SummaryNode, Tree, TreeStatus};
use crate::openhuman::memory_tree::tree::store as tree_store;
/// Build a tiny 2-level tree directly via store inserts so we can
/// assert BFS ordering without needing ~100 leaves to cascade L1→L2
@@ -499,9 +497,9 @@ mod tests {
let content_root = cfg.memory_tree_content_root();
std::fs::create_dir_all(&content_root).unwrap();
let staged = content_store::stage_chunks(&content_root, &all_leaves).unwrap();
crate::openhuman::memory_tree::store::with_connection(cfg, |conn| {
crate::openhuman::memory_store::chunks::store::with_connection(cfg, |conn| {
let tx = conn.unchecked_transaction()?;
crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?;
crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(&tx, &staged)?;
tx.commit()?;
Ok(())
})
@@ -13,10 +13,10 @@
use anyhow::Result;
use crate::openhuman::config::Config;
use crate::openhuman::memory_tree::content_store::read as content_read;
use crate::openhuman::memory_tree::retrieval::types::{hit_from_chunk, RetrievalHit};
use crate::openhuman::memory_tree::score::store::get_score;
use crate::openhuman::memory_tree::store::get_chunk;
use crate::openhuman::memory::retrieval::types::{hit_from_chunk, RetrievalHit};
use crate::openhuman::memory::score::store::get_score;
use crate::openhuman::memory_store::chunks::store::get_chunk;
use crate::openhuman::memory_store::content::read as content_read;
/// Max batch size. Callers that pass more than this get truncated with a
/// warn log — no error surface so the LLM sees a partial result.
@@ -96,9 +96,11 @@ pub async fn fetch_leaves(config: &Config, chunk_ids: &[String]) -> Result<Vec<R
#[cfg(test)]
mod tests {
use super::*;
use crate::openhuman::memory_tree::content_store;
use crate::openhuman::memory_tree::store::upsert_chunks;
use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef};
use crate::openhuman::memory_store::chunks::store::upsert_chunks;
use crate::openhuman::memory_store::chunks::types::{
chunk_id, Chunk, Metadata, SourceKind, SourceRef,
};
use crate::openhuman::memory_store::content as content_store;
use chrono::{TimeZone, Utc};
use tempfile::TempDir;
@@ -107,9 +109,9 @@ mod tests {
std::fs::create_dir_all(&content_root).expect("create content_root for test");
let staged = content_store::stage_chunks(&content_root, chunks)
.expect("stage_chunks for test chunks");
crate::openhuman::memory_tree::store::with_connection(cfg, |conn| {
crate::openhuman::memory_store::chunks::store::with_connection(cfg, |conn| {
let tx = conn.unchecked_transaction()?;
crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?;
crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(&tx, &staged)?;
tx.commit()?;
Ok(())
})
@@ -13,10 +13,10 @@ use anyhow::Result;
use chrono::Duration;
use crate::openhuman::config::Config;
use crate::openhuman::memory_tree::retrieval::types::{NodeKind, QueryResponse, RetrievalHit};
use crate::openhuman::memory_tree::tree_global::recap::{recap, RecapOutput};
use crate::openhuman::memory_tree::tree_global::registry::get_or_create_global_tree;
use crate::openhuman::memory_tree::tree_source::types::TreeKind;
use crate::openhuman::memory::retrieval::types::{NodeKind, QueryResponse, RetrievalHit};
use crate::openhuman::memory_store::trees::registry::get_or_create_global_tree;
use crate::openhuman::memory_store::trees::types::TreeKind;
use crate::openhuman::memory_tree::global::recap::{recap, RecapOutput};
/// Return the global digest for the given window in days. Always returns a
/// [`QueryResponse`]; the response is empty if the global tree has no
@@ -87,16 +87,17 @@ fn recap_to_hits(recap: RecapOutput, tree_id: &str, tree_scope: &str) -> Vec<Ret
#[cfg(test)]
mod tests {
use super::*;
use crate::openhuman::memory_tree::content_store;
use crate::openhuman::memory_tree::store::upsert_chunks;
use crate::openhuman::memory_tree::tree_global::digest::{end_of_day_digest, DigestOutcome};
use crate::openhuman::memory_tree::tree_source::bucket_seal::{
append_leaf, LabelStrategy, LeafRef,
use crate::openhuman::memory::chat::{test_override, ChatProvider, StaticChatProvider};
use crate::openhuman::memory_store::chunks::store::upsert_chunks;
use crate::openhuman::memory_store::chunks::types::{
chunk_id, Chunk, Metadata, SourceKind, SourceRef,
};
use crate::openhuman::memory_tree::tree_source::registry::get_or_create_source_tree;
use crate::openhuman::memory_tree::tree_source::summariser::inert::InertSummariser;
use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef};
use crate::openhuman::memory_store::content as content_store;
use crate::openhuman::memory_tree::global::digest::{end_of_day_digest, DigestOutcome};
use crate::openhuman::memory_tree::sources::registry::get_or_create_source_tree;
use crate::openhuman::memory_tree::tree::bucket_seal::{append_leaf, LabelStrategy, LeafRef};
use chrono::{DateTime, Utc};
use std::sync::Arc;
use tempfile::TempDir;
fn stage_test_chunks(cfg: &Config, chunks: &[Chunk]) {
@@ -104,9 +105,9 @@ mod tests {
std::fs::create_dir_all(&content_root).expect("create content_root for test");
let staged = content_store::stage_chunks(&content_root, chunks)
.expect("stage_chunks for test chunks");
crate::openhuman::memory_tree::store::with_connection(cfg, |conn| {
crate::openhuman::memory_store::chunks::store::with_connection(cfg, |conn| {
let tx = conn.unchecked_transaction()?;
crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?;
crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(&tx, &staged)?;
tx.commit()?;
Ok(())
})
@@ -125,16 +126,21 @@ mod tests {
}
async fn seed_daily_digest(cfg: &Config) {
let summariser = InertSummariser::new();
let provider: Arc<dyn ChatProvider> =
Arc::new(StaticChatProvider::new("test summary content"));
let day = Utc::now().date_naive();
let ts = day.and_hms_opt(12, 0, 0).unwrap().and_utc();
seed_source_for_day(cfg, "slack:#eng", ts).await;
end_of_day_digest(cfg, day, &summariser).await.unwrap();
test_override::with_provider(provider, async {
end_of_day_digest(cfg, day).await.unwrap()
})
.await;
}
async fn seed_source_for_day(cfg: &Config, scope: &str, ts: DateTime<Utc>) {
let tree = get_or_create_source_tree(cfg, scope).unwrap();
let summariser = InertSummariser::new();
let provider: Arc<dyn ChatProvider> =
Arc::new(StaticChatProvider::new("test summary content"));
for seq in 0..2u32 {
let c = Chunk {
id: chunk_id(SourceKind::Chat, scope, seq, "test-content"),
@@ -155,23 +161,21 @@ mod tests {
};
upsert_chunks(cfg, &[c.clone()]).unwrap();
stage_test_chunks(cfg, &[c.clone()]);
append_leaf(
cfg,
&tree,
&LeafRef {
chunk_id: c.id.clone(),
token_count: 30_000,
timestamp: ts,
content: c.content.clone(),
entities: vec![],
topics: vec![],
score: 0.5,
},
&summariser,
&LabelStrategy::Empty,
)
.await
.unwrap();
let leaf = LeafRef {
chunk_id: c.id.clone(),
token_count: 30_000,
timestamp: ts,
content: c.content.clone(),
entities: vec![],
topics: vec![],
score: 0.5,
};
test_override::with_provider(Arc::clone(&provider), async {
append_leaf(cfg, &tree, &leaf, &LabelStrategy::Empty)
.await
.unwrap()
})
.await;
}
}
@@ -205,11 +209,15 @@ mod tests {
// if this ever returned Skipped the rest of the suite would trivially
// pass which would be misleading.
let (_tmp, cfg) = test_config();
let summariser = InertSummariser::new();
let provider: Arc<dyn ChatProvider> =
Arc::new(StaticChatProvider::new("test summary content"));
let day = Utc::now().date_naive();
let ts = day.and_hms_opt(12, 0, 0).unwrap().and_utc();
seed_source_for_day(&cfg, "slack:#eng", ts).await;
let outcome = end_of_day_digest(&cfg, day, &summariser).await.unwrap();
let outcome = test_override::with_provider(provider, async {
end_of_day_digest(&cfg, day).await.unwrap()
})
.await;
assert!(matches!(outcome, DigestOutcome::Emitted { .. }));
}
}
@@ -14,12 +14,12 @@ use chrono::{TimeZone, Utc};
use tempfile::TempDir;
use crate::openhuman::config::Config;
use crate::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage};
use crate::openhuman::memory_tree::ingest::ingest_chat;
use crate::openhuman::memory_tree::retrieval::{
use crate::openhuman::memory::ingest_pipeline::ingest_chat;
use crate::openhuman::memory::retrieval::{
drill_down, fetch_leaves, query_global, query_source, query_topic, search_entities,
};
use crate::openhuman::memory_tree::types::SourceKind;
use crate::openhuman::memory_store::chunks::types::SourceKind;
use crate::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage};
fn test_config() -> (TempDir, Config) {
let tmp = TempDir::new().unwrap();
@@ -124,7 +124,7 @@ async fn end_to_end_three_chat_batches() {
// ── fetch_leaves: find a guaranteed leaf hit from alice's topic results
// and assert that fetch_leaves hydrates it correctly.
use crate::openhuman::memory_tree::retrieval::types::NodeKind;
use crate::openhuman::memory::retrieval::types::NodeKind;
let leaf_hit = by_email
.hits
.iter()
@@ -164,9 +164,9 @@ async fn topic_entity_surfaces_after_ingest() {
/// handler, so the test drains the queue before inspecting.
#[tokio::test]
async fn ingest_populates_chunk_embeddings() {
use crate::openhuman::memory_tree::jobs::drain_until_idle;
use crate::openhuman::memory_tree::score::embed::EMBEDDING_DIM;
use crate::openhuman::memory_tree::store::get_chunk_embedding;
use crate::openhuman::memory::jobs::drain_until_idle;
use crate::openhuman::memory::score::embed::EMBEDDING_DIM;
use crate::openhuman::memory_store::chunks::store::get_chunk_embedding;
let (_tmp, cfg) = test_config();
let out = ingest_chat(&cfg, "slack:#eng", "alice", vec![], chat_about_phoenix(0))
@@ -192,20 +192,21 @@ async fn ingest_populates_chunk_embeddings() {
/// the seal from firing on short batches.
#[tokio::test]
async fn seal_populates_summary_embedding() {
use crate::openhuman::memory_tree::content_store;
use crate::openhuman::memory_tree::score::embed::EMBEDDING_DIM;
use crate::openhuman::memory_tree::store::upsert_chunks;
use crate::openhuman::memory_tree::tree_source::bucket_seal::{
append_leaf, LabelStrategy, LeafRef,
use crate::openhuman::memory::chat::{test_override, ChatProvider, StaticChatProvider};
use crate::openhuman::memory::score::embed::EMBEDDING_DIM;
use crate::openhuman::memory_store::chunks::store::upsert_chunks;
use crate::openhuman::memory_store::chunks::types::{
chunk_id, Chunk, Metadata, SourceKind, SourceRef,
};
use crate::openhuman::memory_tree::tree_source::registry::get_or_create_source_tree;
use crate::openhuman::memory_tree::tree_source::store as src_store;
use crate::openhuman::memory_tree::tree_source::summariser::inert::InertSummariser;
use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef};
use crate::openhuman::memory_store::content as content_store;
use crate::openhuman::memory_tree::sources::registry::get_or_create_source_tree;
use crate::openhuman::memory_tree::tree::bucket_seal::{append_leaf, LabelStrategy, LeafRef};
use crate::openhuman::memory_tree::tree::store as src_store;
use std::sync::Arc;
let (_tmp, cfg) = test_config();
let tree = get_or_create_source_tree(&cfg, "slack:#seal-test").unwrap();
let summariser = InertSummariser::new();
let provider: Arc<dyn ChatProvider> = Arc::new(StaticChatProvider::new("test summary content"));
let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap();
let mk_chunk = |seq: u32, tokens: u32| Chunk {
@@ -233,9 +234,9 @@ async fn seal_populates_summary_embedding() {
std::fs::create_dir_all(&content_root).expect("create content_root for test");
let staged = content_store::stage_chunks(&content_root, &[c1.clone(), c2.clone()])
.expect("stage_chunks for test chunks");
crate::openhuman::memory_tree::store::with_connection(&cfg, |conn| {
crate::openhuman::memory_store::chunks::store::with_connection(&cfg, |conn| {
let tx = conn.unchecked_transaction()?;
crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?;
crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(&tx, &staged)?;
tx.commit()?;
Ok(())
})
@@ -251,24 +252,18 @@ async fn seal_populates_summary_embedding() {
topics: vec![],
score: 0.5,
};
append_leaf(
&cfg,
&tree,
&leaf_of(&c1),
&summariser,
&LabelStrategy::Empty,
)
.await
.unwrap();
let sealed = append_leaf(
&cfg,
&tree,
&leaf_of(&c2),
&summariser,
&LabelStrategy::Empty,
)
.await
.unwrap();
test_override::with_provider(Arc::clone(&provider), async {
append_leaf(&cfg, &tree, &leaf_of(&c1), &LabelStrategy::Empty)
.await
.unwrap()
})
.await;
let sealed = test_override::with_provider(Arc::clone(&provider), async {
append_leaf(&cfg, &tree, &leaf_of(&c2), &LabelStrategy::Empty)
.await
.unwrap()
})
.await;
assert_eq!(sealed.len(), 1, "expected one seal at the budget crossing");
// #1574 cutover: the seal path no longer writes the legacy
@@ -8,7 +8,7 @@
use serde::{Deserialize, Serialize};
use crate::openhuman::config::Config;
use crate::openhuman::memory_tree::retrieval::{
use crate::openhuman::memory::retrieval::{
drill_down::drill_down,
fetch::fetch_leaves,
global::query_global,
@@ -17,8 +17,8 @@ use crate::openhuman::memory_tree::retrieval::{
topic::query_topic,
types::{EntityMatch, QueryResponse, RetrievalHit},
};
use crate::openhuman::memory_tree::score::extract::EntityKind;
use crate::openhuman::memory_tree::types::SourceKind;
use crate::openhuman::memory::score::extract::EntityKind;
use crate::openhuman::memory_store::chunks::types::SourceKind;
use crate::rpc::RpcOutcome;
// ── query_source ──────────────────────────────────────────────────────
@@ -307,9 +307,9 @@ mod tests {
//! initialises the schema idempotently on first access, so read-only
//! calls return empty responses rather than erroring.
use super::*;
use crate::openhuman::memory_tree::content_store;
use crate::openhuman::memory_tree::store::upsert_chunks;
use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceRef};
use crate::openhuman::memory_store::chunks::store::upsert_chunks;
use crate::openhuman::memory_store::chunks::types::{chunk_id, Chunk, Metadata, SourceRef};
use crate::openhuman::memory_store::content as content_store;
use chrono::{TimeZone, Utc};
use tempfile::TempDir;
@@ -318,9 +318,9 @@ mod tests {
std::fs::create_dir_all(&content_root).expect("create content_root for test");
let staged = content_store::stage_chunks(&content_root, chunks)
.expect("stage_chunks for test chunks");
crate::openhuman::memory_tree::store::with_connection(cfg, |conn| {
crate::openhuman::memory_store::chunks::store::with_connection(cfg, |conn| {
let tx = conn.unchecked_transaction()?;
crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?;
crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(&tx, &staged)?;
tx.commit()?;
Ok(())
})
@@ -18,7 +18,7 @@ use serde_json::{Map, Value};
use crate::core::all::{ControllerFuture, RegisteredController};
use crate::core::{ControllerSchema, FieldSchema, TypeSchema};
use crate::openhuman::config::rpc as config_rpc;
use crate::openhuman::memory_tree::retrieval::rpc as retrieval_rpc;
use crate::openhuman::memory::retrieval::rpc as retrieval_rpc;
use crate::rpc::RpcOutcome;
const NAMESPACE: &str = "memory_tree";
@@ -385,3 +385,49 @@ fn parse_value<T: DeserializeOwned>(v: Value) -> Result<T, String> {
fn to_json<T: serde::Serialize>(outcome: RpcOutcome<T>) -> Result<Value, String> {
outcome.into_cli_compatible_json()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn all_controller_schemas_cover_every_registered_retrieval_function() {
let schemas = all_controller_schemas();
let functions: Vec<&str> = schemas.iter().map(|s| s.function).collect();
assert_eq!(
functions,
vec![
"query_source",
"query_global",
"query_topic",
"search_entities",
"drill_down",
"fetch_leaves",
]
);
}
#[test]
fn registered_controllers_use_memory_tree_namespace() {
let controllers = all_registered_controllers();
assert_eq!(controllers.len(), 6);
assert!(controllers.iter().all(|c| c.schema.namespace == NAMESPACE));
}
#[test]
fn unknown_schema_returns_error_output() {
let schema = schemas("not_a_real_function");
assert_eq!(schema.namespace, NAMESPACE);
assert_eq!(schema.function, "unknown");
assert_eq!(schema.outputs.len(), 1);
assert_eq!(schema.outputs[0].name, "error");
}
#[test]
fn query_global_schema_requires_time_window_days() {
let schema = schemas("query_global");
assert_eq!(schema.inputs.len(), 1);
assert_eq!(schema.inputs[0].name, "time_window_days");
assert!(schema.inputs[0].required);
}
}
@@ -19,9 +19,9 @@ use anyhow::{Context, Result};
use rusqlite::params_from_iter;
use crate::openhuman::config::Config;
use crate::openhuman::memory_tree::retrieval::types::EntityMatch;
use crate::openhuman::memory_tree::score::extract::EntityKind;
use crate::openhuman::memory_tree::store::with_connection;
use crate::openhuman::memory::retrieval::types::EntityMatch;
use crate::openhuman::memory::score::extract::EntityKind;
use crate::openhuman::memory_store::chunks::store::with_connection;
const DEFAULT_LIMIT: usize = 5;
const MAX_LIMIT: usize = 100;
@@ -156,8 +156,8 @@ fn row_to_match(row: &rusqlite::Row<'_>) -> rusqlite::Result<EntityMatch> {
#[cfg(test)]
mod tests {
use super::*;
use crate::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage};
use crate::openhuman::memory_tree::ingest::ingest_chat;
use crate::openhuman::memory::ingest_pipeline::ingest_chat;
use crate::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage};
use chrono::{TimeZone, Utc};
use tempfile::TempDir;
@@ -21,14 +21,12 @@ use anyhow::Result;
use chrono::{Duration, Utc};
use crate::openhuman::config::Config;
use crate::openhuman::memory_tree::content_store::read as content_read;
use crate::openhuman::memory_tree::retrieval::types::{
hit_from_summary, QueryResponse, RetrievalHit,
};
use crate::openhuman::memory_tree::score::embed::{build_embedder_from_config, cosine_similarity};
use crate::openhuman::memory_tree::tree_source::store;
use crate::openhuman::memory_tree::tree_source::types::{SummaryNode, Tree, TreeKind};
use crate::openhuman::memory_tree::types::SourceKind;
use crate::openhuman::memory::retrieval::types::{hit_from_summary, QueryResponse, RetrievalHit};
use crate::openhuman::memory::score::embed::{build_embedder_from_config, cosine_similarity};
use crate::openhuman::memory_store::chunks::types::SourceKind;
use crate::openhuman::memory_store::content::read as content_read;
use crate::openhuman::memory_store::trees::types::{SummaryNode, Tree, TreeKind};
use crate::openhuman::memory_tree::tree::store;
const DEFAULT_LIMIT: usize = 10;
@@ -306,15 +304,16 @@ fn filter_by_window(hits: Vec<RetrievalHit>, window_days: u32) -> Vec<RetrievalH
#[cfg(test)]
mod tests {
use super::*;
use crate::openhuman::memory_tree::content_store;
use crate::openhuman::memory_tree::store::upsert_chunks;
use crate::openhuman::memory_tree::tree_source::bucket_seal::{
append_leaf, LabelStrategy, LeafRef,
use crate::openhuman::memory::chat::{test_override, ChatProvider, StaticChatProvider};
use crate::openhuman::memory_store::chunks::store::upsert_chunks;
use crate::openhuman::memory_store::chunks::types::{
chunk_id, Chunk, Metadata, SourceKind, SourceRef,
};
use crate::openhuman::memory_tree::tree_source::registry::get_or_create_source_tree;
use crate::openhuman::memory_tree::tree_source::summariser::inert::InertSummariser;
use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef};
use crate::openhuman::memory_store::content as content_store;
use crate::openhuman::memory_tree::sources::registry::get_or_create_source_tree;
use crate::openhuman::memory_tree::tree::bucket_seal::{append_leaf, LabelStrategy, LeafRef};
use chrono::{DateTime, TimeZone};
use std::sync::Arc;
use tempfile::TempDir;
fn test_config() -> (TempDir, Config) {
@@ -330,7 +329,8 @@ mod tests {
async fn seed_source(cfg: &Config, scope: &str, ts: DateTime<Utc>) {
let tree = get_or_create_source_tree(cfg, scope).unwrap();
let summariser = InertSummariser::new();
let provider: Arc<dyn ChatProvider> =
Arc::new(StaticChatProvider::new("test summary content"));
let content_root = cfg.memory_tree_content_root();
std::fs::create_dir_all(&content_root).unwrap();
for seq in 0..2u32 {
@@ -346,8 +346,7 @@ mod tests {
tags: vec!["eng".into()],
source_ref: Some(SourceRef::new(format!("slack://{scope}/{seq}"))),
},
token_count: crate::openhuman::memory_tree::tree_source::types::INPUT_TOKEN_BUDGET
* 6
token_count: crate::openhuman::memory_store::trees::types::INPUT_TOKEN_BUDGET * 6
/ 10,
seq_in_source: seq,
created_at: ts,
@@ -358,32 +357,31 @@ mod tests {
// via `read_chunk_body` during the seal triggered by `append_leaf`,
// and `collect_hits_and_nodes` can read summary bodies for the API.
let staged = content_store::stage_chunks(&content_root, &[c.clone()]).unwrap();
crate::openhuman::memory_tree::store::with_connection(cfg, |conn| {
crate::openhuman::memory_store::chunks::store::with_connection(cfg, |conn| {
let tx = conn.unchecked_transaction()?;
crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?;
crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(
&tx, &staged,
)?;
tx.commit()?;
Ok(())
})
.unwrap();
append_leaf(
cfg,
&tree,
&LeafRef {
chunk_id: c.id.clone(),
token_count:
crate::openhuman::memory_tree::tree_source::types::INPUT_TOKEN_BUDGET * 6
/ 10,
timestamp: ts,
content: c.content.clone(),
entities: vec![],
topics: vec![],
score: 0.5,
},
&summariser,
&LabelStrategy::Empty,
)
.await
.unwrap();
let leaf = LeafRef {
chunk_id: c.id.clone(),
token_count: crate::openhuman::memory_store::trees::types::INPUT_TOKEN_BUDGET * 6
/ 10,
timestamp: ts,
content: c.content.clone(),
entities: vec![],
topics: vec![],
score: 0.5,
};
test_override::with_provider(Arc::clone(&provider), async {
append_leaf(cfg, &tree, &leaf, &LabelStrategy::Empty)
.await
.unwrap()
})
.await;
}
}
@@ -527,8 +525,8 @@ mod tests {
/// the ingest path writes by default.
#[tokio::test]
async fn query_reranks_by_cosine_similarity() {
use crate::openhuman::memory_tree::score::embed::{pack_embedding, EMBEDDING_DIM};
use crate::openhuman::memory_tree::tree_source::store as src_store;
use crate::openhuman::memory::score::embed::{pack_embedding, EMBEDDING_DIM};
use crate::openhuman::memory_tree::tree::store as src_store;
let (_tmp, cfg) = test_config();
let ts = Utc::now();
@@ -548,17 +546,17 @@ mod tests {
// Write directly via raw UPDATE so we replace whatever the
// seal-time inert embedder wrote.
use crate::openhuman::memory_tree::store::with_connection;
use crate::openhuman::memory_store::chunks::store::with_connection;
let phoenix_tree = src_store::get_tree_by_scope(
&cfg,
crate::openhuman::memory_tree::tree_source::types::TreeKind::Source,
crate::openhuman::memory_store::trees::types::TreeKind::Source,
"slack:#phoenix",
)
.unwrap()
.unwrap();
let unrelated_tree = src_store::get_tree_by_scope(
&cfg,
crate::openhuman::memory_tree::tree_source::types::TreeKind::Source,
crate::openhuman::memory_store::trees::types::TreeKind::Source,
"slack:#unrelated",
)
.unwrap()
@@ -597,7 +595,7 @@ mod tests {
// The practical test here: construct a hypothetical query
// vector equal to phoenix_vec, then verify that running the
// rerank helper with that vector places phoenix first.
use crate::openhuman::memory_tree::score::embed::cosine_similarity;
use crate::openhuman::memory::score::embed::cosine_similarity;
let query_vec = phoenix_vec.clone();
let phoenix_sim = cosine_similarity(&query_vec, &phoenix_vec);
let unrelated_sim = cosine_similarity(&query_vec, &unrelated_vec);
@@ -629,9 +627,9 @@ mod tests {
/// summaries that do have embeddings when a `query` is supplied.
#[tokio::test]
async fn legacy_null_embedding_rows_sort_last() {
use crate::openhuman::memory_tree::score::embed::{pack_embedding, EMBEDDING_DIM};
use crate::openhuman::memory_tree::tree_source::store as src_store;
use crate::openhuman::memory_tree::tree_source::types::TreeKind;
use crate::openhuman::memory::score::embed::{pack_embedding, EMBEDDING_DIM};
use crate::openhuman::memory_store::trees::types::TreeKind;
use crate::openhuman::memory_tree::tree::store as src_store;
let (_tmp, cfg) = test_config();
let ts = Utc::now();
@@ -655,7 +653,7 @@ mod tests {
v[0] = 1.0;
let blob = pack_embedding(&v);
use crate::openhuman::memory_tree::store::with_connection;
use crate::openhuman::memory_store::chunks::store::with_connection;
with_connection(&cfg, |conn| {
conn.execute(
"UPDATE mem_tree_summaries SET embedding = ?1 WHERE id = ?2",
@@ -17,14 +17,12 @@ use anyhow::Result;
use chrono::{Duration, TimeZone, Utc};
use crate::openhuman::config::Config;
use crate::openhuman::memory_tree::content_store::read as content_read;
use crate::openhuman::memory_tree::retrieval::types::{
hit_from_summary, QueryResponse, RetrievalHit,
};
use crate::openhuman::memory_tree::score::embed::{build_embedder_from_config, cosine_similarity};
use crate::openhuman::memory_tree::score::store::{lookup_entity, EntityHit};
use crate::openhuman::memory_tree::tree_source::store;
use crate::openhuman::memory_tree::tree_source::types::{Tree, TreeKind};
use crate::openhuman::memory::retrieval::types::{hit_from_summary, QueryResponse, RetrievalHit};
use crate::openhuman::memory::score::embed::{build_embedder_from_config, cosine_similarity};
use crate::openhuman::memory::score::store::{lookup_entity, EntityHit};
use crate::openhuman::memory_store::content::read as content_read;
use crate::openhuman::memory_store::trees::types::{Tree, TreeKind};
use crate::openhuman::memory_tree::tree::store;
const DEFAULT_LIMIT: usize = 10;
/// How many rows we pull from the entity index before filtering. We give
@@ -175,9 +173,9 @@ async fn rerank_by_semantic_similarity(
query: &str,
hits: Vec<RetrievalHit>,
) -> Result<Vec<RetrievalHit>> {
use crate::openhuman::memory_tree::retrieval::types::NodeKind;
use crate::openhuman::memory_tree::store::get_chunk_embedding;
use crate::openhuman::memory_tree::tree_source::store as src_store;
use crate::openhuman::memory::retrieval::types::NodeKind;
use crate::openhuman::memory_store::chunks::store::get_chunk_embedding;
use crate::openhuman::memory_tree::tree::store as src_store;
let embedder = build_embedder_from_config(config)?;
let query_vec = embedder.embed(query).await?;
@@ -318,8 +316,8 @@ async fn entity_hit_to_retrieval_hit(
return Ok(Some(h));
}
// Leaf: fetch chunk and hydrate.
use crate::openhuman::memory_tree::retrieval::types::hit_from_chunk;
use crate::openhuman::memory_tree::store::get_chunk;
use crate::openhuman::memory::retrieval::types::hit_from_chunk;
use crate::openhuman::memory_store::chunks::store::get_chunk;
let mut chunk = match get_chunk(&config_owned, &node_id)? {
Some(c) => c,
None => {
@@ -368,8 +366,14 @@ fn filter_by_window(hits: Vec<RetrievalHit>, window_days: u32) -> Vec<RetrievalH
#[cfg(test)]
mod tests {
use super::*;
use crate::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage};
use crate::openhuman::memory_tree::ingest::ingest_chat;
use crate::openhuman::memory::ingest_pipeline::ingest_chat;
use crate::openhuman::memory::score::extract::EntityKind;
use crate::openhuman::memory::score::store::EntityHit;
use crate::openhuman::memory_store::chunks::store::upsert_chunks;
use crate::openhuman::memory_store::chunks::types::{
chunk_id, Chunk, Metadata, SourceKind, SourceRef,
};
use crate::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage};
use chrono::TimeZone;
use tempfile::TempDir;
@@ -522,6 +526,71 @@ mod tests {
}
}
#[tokio::test]
async fn zero_limit_uses_default_limit() {
let (_tmp, cfg) = test_config();
for i in 0..12 {
let source = format!("slack:#c{i}");
let batch = ChatBatch {
platform: "slack".into(),
channel_label: format!("#c{i}"),
messages: vec![ChatMessage {
author: "alice".into(),
timestamp: Utc::now(),
text: format!(
"Meeting {i} about Phoenix migration. alice@example.com owns it. \
Launch status looks good."
),
source_ref: None,
}],
};
ingest_chat(&cfg, &source, "alice", vec![], batch)
.await
.unwrap();
}
let resp = query_topic(&cfg, "email:alice@example.com", None, None, 0)
.await
.unwrap();
assert!(resp.hits.len() <= DEFAULT_LIMIT);
if resp.total > DEFAULT_LIMIT {
assert!(resp.truncated);
}
}
#[tokio::test]
async fn topic_tree_root_missing_summary_row_is_ignored() {
use crate::openhuman::memory_store::chunks::store::with_connection;
use crate::openhuman::memory_store::trees::types::{Tree, TreeKind, TreeStatus};
use crate::openhuman::memory_tree::tree::store as tree_store;
let (_tmp, cfg) = test_config();
let ts = Utc::now();
let entity_id = "topic:phoenix";
let tree = Tree {
id: "test:phoenix-missing-root".into(),
kind: TreeKind::Topic,
scope: entity_id.into(),
root_id: Some("summary:missing".into()),
max_level: 1,
status: TreeStatus::Active,
created_at: ts,
last_sealed_at: Some(ts),
};
with_connection(&cfg, |conn| {
let tx = conn.unchecked_transaction()?;
tree_store::insert_tree_conn(&tx, &tree)?;
tx.commit()?;
Ok(())
})
.unwrap();
let resp = query_topic(&cfg, entity_id, None, None, 10).await.unwrap();
assert!(resp.hits.is_empty());
assert_eq!(resp.total, 0);
}
// Regression: the same node_id must only appear once in `hits`, even
// when the topic-tree root overlaps with its own entity-index row.
// Flagged on PR #831 CodeRabbit review — see the HashMap-based merge
@@ -529,14 +598,13 @@ mod tests {
// caller would see two rows for the same summary.
#[tokio::test]
async fn duplicate_node_is_deduplicated_across_index_and_topic_tree_root() {
use crate::openhuman::memory_tree::score::extract::EntityKind;
use crate::openhuman::memory_tree::score::resolver::CanonicalEntity;
use crate::openhuman::memory_tree::score::store as score_store;
use crate::openhuman::memory_tree::store::with_connection;
use crate::openhuman::memory_tree::tree_source::store as tree_store;
use crate::openhuman::memory_tree::tree_source::types::{
use crate::openhuman::memory::score::resolver::CanonicalEntity;
use crate::openhuman::memory::score::store as score_store;
use crate::openhuman::memory_store::chunks::store::with_connection;
use crate::openhuman::memory_store::trees::types::{
SummaryNode, Tree, TreeKind, TreeStatus,
};
use crate::openhuman::memory_tree::tree::store as tree_store;
let (_tmp, cfg) = test_config();
let ts = Utc::now();
@@ -625,4 +693,144 @@ mod tests {
"total should count distinct nodes, not raw row occurrences"
);
}
#[tokio::test]
async fn stale_leaf_entity_index_row_is_skipped() {
let (_tmp, cfg) = test_config();
let hit = EntityHit {
entity_id: "email:alice@example.com".into(),
node_id: "chunk:missing".into(),
node_kind: "leaf".into(),
entity_kind: EntityKind::Email,
surface: "alice@example.com".into(),
score: 0.7,
timestamp_ms: Utc::now().timestamp_millis(),
tree_id: Some("tree:missing".into()),
is_user: false,
};
let converted = entity_hit_to_retrieval_hit(&cfg, &hit).await.unwrap();
assert!(converted.is_none());
}
#[tokio::test]
async fn leaf_hit_falls_back_to_source_scope_when_tree_lookup_misses() {
let (_tmp, cfg) = test_config();
let ts = Utc.timestamp_millis_opt(1_700_123_456_789).unwrap();
let chunk = Chunk {
id: chunk_id(
SourceKind::Chat,
"slack:#eng",
0,
"Phoenix owner is alice@example.com",
),
content: "Phoenix owner is alice@example.com".into(),
metadata: Metadata {
source_kind: SourceKind::Chat,
source_id: "slack:#eng".into(),
owner: "alice".into(),
timestamp: ts,
time_range: (ts, ts),
tags: vec!["phoenix".into()],
source_ref: Some(SourceRef::new("slack://eng/1")),
},
token_count: 8,
seq_in_source: 0,
created_at: ts,
partial_message: false,
};
upsert_chunks(&cfg, &[chunk.clone()]).unwrap();
let hit = EntityHit {
entity_id: "email:alice@example.com".into(),
node_id: chunk.id.clone(),
node_kind: "leaf".into(),
entity_kind: EntityKind::Email,
surface: "alice@example.com".into(),
score: 0.91,
timestamp_ms: ts.timestamp_millis(),
tree_id: Some("tree:missing".into()),
is_user: false,
};
let converted = entity_hit_to_retrieval_hit(&cfg, &hit)
.await
.unwrap()
.unwrap();
assert_eq!(converted.tree_scope, "slack:#eng");
assert_eq!(converted.tree_id, "tree:missing");
assert_eq!(converted.score, 0.91);
assert_eq!(converted.source_ref.as_deref(), Some("slack://eng/1"));
assert_eq!(converted.topics, vec!["phoenix"]);
}
#[tokio::test]
async fn summary_hit_without_tree_id_uses_empty_scope_and_index_score() {
use crate::openhuman::memory_store::chunks::store::with_connection;
use crate::openhuman::memory_store::trees::types::{
SummaryNode, Tree, TreeKind, TreeStatus,
};
use crate::openhuman::memory_tree::tree::store as tree_store;
let (_tmp, cfg) = test_config();
let ts = Utc::now();
let tree = Tree {
id: "tree:topic:phoenix".into(),
kind: TreeKind::Topic,
scope: "topic:phoenix".into(),
root_id: Some("summary:l1:phoenix".into()),
max_level: 1,
status: TreeStatus::Active,
created_at: ts,
last_sealed_at: Some(ts),
};
let summary = SummaryNode {
id: "summary:l1:phoenix".into(),
tree_id: tree.id.clone(),
tree_kind: TreeKind::Topic,
level: 1,
parent_id: None,
child_ids: vec!["chunk:a".into()],
content: "Phoenix recap preview".into(),
token_count: 16,
entities: vec!["topic:phoenix".into()],
topics: vec!["phoenix".into()],
time_range_start: ts,
time_range_end: ts,
score: 0.12,
sealed_at: ts,
deleted: false,
embedding: None,
};
with_connection(&cfg, |conn| {
let tx = conn.unchecked_transaction()?;
tree_store::insert_tree_conn(&tx, &tree)?;
tree_store::insert_summary_tx(&tx, &summary, None, "test")?;
tx.commit()?;
Ok(())
})
.unwrap();
let hit = EntityHit {
entity_id: "topic:phoenix".into(),
node_id: summary.id.clone(),
node_kind: "summary".into(),
entity_kind: EntityKind::Topic,
surface: "phoenix".into(),
score: 0.88,
timestamp_ms: ts.timestamp_millis(),
tree_id: None,
is_user: false,
};
let converted = entity_hit_to_retrieval_hit(&cfg, &hit)
.await
.unwrap()
.unwrap();
assert_eq!(converted.tree_scope, "");
assert_eq!(converted.tree_id, tree.id);
assert_eq!(converted.score, 0.88);
assert_eq!(converted.content, "Phoenix recap preview");
}
}
@@ -18,9 +18,9 @@
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use crate::openhuman::memory_tree::score::extract::EntityKind;
use crate::openhuman::memory_tree::tree_source::types::{SummaryNode, Tree, TreeKind};
use crate::openhuman::memory_tree::types::{Chunk, SourceKind};
use crate::openhuman::memory::score::extract::EntityKind;
use crate::openhuman::memory_store::chunks::types::{Chunk, SourceKind};
use crate::openhuman::memory_store::trees::types::{SummaryNode, Tree, TreeKind};
/// Whether a hit represents a leaf (raw chunk) or a summary node.
#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)]
@@ -16,8 +16,8 @@ use serde_json::{Map, Value};
use crate::core::all::{ControllerFuture, RegisteredController};
use crate::core::{ControllerSchema, FieldSchema, TypeSchema};
use crate::openhuman::config::rpc as config_rpc;
use crate::openhuman::memory_tree::read_rpc;
use crate::openhuman::memory_tree::rpc as tree_rpc;
use crate::openhuman::memory::read_rpc;
use crate::openhuman::memory::tree_rpc;
use crate::rpc::RpcOutcome;
const NAMESPACE: &str = "memory_tree";
@@ -207,26 +207,31 @@ pub fn schemas(function: &str) -> ControllerSchema {
"list_chunks" => ControllerSchema {
namespace: NAMESPACE,
function: "list_chunks",
description:
"Paginated list of chunks with optional filters by source kind / source id / \
description: "Paginated list of chunks with optional filters by source kind / source id / \
entity ids / time window / keyword. Returns chunks plus total match count for \
pagination.",
inputs: vec![
FieldSchema {
name: "source_kinds",
ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new(TypeSchema::String)))),
ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new(
TypeSchema::String,
)))),
comment: "Restrict to one or more source kinds (chat / email / document).",
required: false,
},
FieldSchema {
name: "source_ids",
ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new(TypeSchema::String)))),
ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new(
TypeSchema::String,
)))),
comment: "Restrict to one or more logical source ids.",
required: false,
},
FieldSchema {
name: "entity_ids",
ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new(TypeSchema::String)))),
ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new(
TypeSchema::String,
)))),
comment: "Restrict to chunks indexed against any of these canonical entity ids.",
required: false,
},
@@ -296,8 +301,7 @@ pub fn schemas(function: &str) -> ControllerSchema {
"list_sources" => ControllerSchema {
namespace: NAMESPACE,
function: "list_sources",
description:
"Distinct (source_kind, source_id) pairs with chunk counts and most-recent timestamps. \
description: "Distinct (source_kind, source_id) pairs with chunk counts and most-recent timestamps. \
`display_name` is computed from the source_id (un-slug + strip user email when known).",
inputs: vec![FieldSchema {
name: "user_email_hint",
@@ -316,8 +320,7 @@ pub fn schemas(function: &str) -> ControllerSchema {
"search" => ControllerSchema {
namespace: NAMESPACE,
function: "search",
description:
"Keyword LIKE-search over chunk bodies. Cheap, deterministic; useful as a \
description: "Keyword LIKE-search over chunk bodies. Cheap, deterministic; useful as a \
fallback when semantic recall is unavailable.",
inputs: vec![
FieldSchema {
@@ -343,8 +346,7 @@ pub fn schemas(function: &str) -> ControllerSchema {
"recall" => ControllerSchema {
namespace: NAMESPACE,
function: "recall",
description:
"Semantic recall — runs the Phase 4 cosine rerank against the query embedding \
description: "Semantic recall — runs the Phase 4 cosine rerank against the query embedding \
and returns leaf chunks (not summaries) for UI display.",
inputs: vec![
FieldSchema {
@@ -395,14 +397,12 @@ pub fn schemas(function: &str) -> ControllerSchema {
"chunks_for_entity" => ControllerSchema {
namespace: NAMESPACE,
function: "chunks_for_entity",
description:
"Return chunk IDs that reference an entity_id (inverse of entity_index_for). \
description: "Return chunk IDs that reference an entity_id (inverse of entity_index_for). \
Used by the Memory tab's People/Topics lenses to filter the chunk list.",
inputs: vec![FieldSchema {
name: "entity_id",
ty: TypeSchema::String,
comment:
"Canonical entity id (e.g. `person:Steven Enamakel`, \
comment: "Canonical entity id (e.g. `person:Steven Enamakel`, \
`email:alice@example.com`).",
required: true,
}],
@@ -416,8 +416,7 @@ pub fn schemas(function: &str) -> ControllerSchema {
"top_entities" => ControllerSchema {
namespace: NAMESPACE,
function: "top_entities",
description:
"Most-frequent canonical entities across the workspace, optionally narrowed by kind.",
description: "Most-frequent canonical entities across the workspace, optionally narrowed by kind.",
inputs: vec![
FieldSchema {
name: "kind",
@@ -442,8 +441,7 @@ pub fn schemas(function: &str) -> ControllerSchema {
"chunk_score" => ControllerSchema {
namespace: NAMESPACE,
function: "chunk_score",
description:
"Score breakdown stored in `mem_tree_score` for one chunk — used by the Memory \
description: "Score breakdown stored in `mem_tree_score` for one chunk — used by the Memory \
tab's 'why was this kept / dropped' panel.",
inputs: vec![FieldSchema {
name: "chunk_id",
@@ -461,8 +459,7 @@ pub fn schemas(function: &str) -> ControllerSchema {
"delete_chunk" => ControllerSchema {
namespace: NAMESPACE,
function: "delete_chunk",
description:
"Purge one chunk plus its score row, entity-index rows, and on-disk .md file. \
description: "Purge one chunk plus its score row, entity-index rows, and on-disk .md file. \
Idempotent missing chunk returns deleted=false. Does NOT cascade through \
sealed summaries; UIs warn the user.",
inputs: vec![FieldSchema {
@@ -509,8 +506,7 @@ pub fn schemas(function: &str) -> ControllerSchema {
"set_llm" => ControllerSchema {
namespace: NAMESPACE,
function: "set_llm",
description:
"Update the LLM backend selector and (optionally) per-role model choices \
description: "Update the LLM backend selector and (optionally) per-role model choices \
(`cloud_model`, `extract_model`, `summariser_model`) and persist the \
result to config.toml in a single atomic write. Absent model fields \
leave the corresponding config key unchanged so a caller flipping just \
@@ -961,3 +957,51 @@ fn parse_value<T: DeserializeOwned>(v: Value) -> Result<T, String> {
fn to_json<T: serde::Serialize>(outcome: RpcOutcome<T>) -> Result<Value, String> {
outcome.into_cli_compatible_json()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn all_controller_schemas_and_registered_controllers_stay_in_sync() {
let schemas = all_controller_schemas();
let controllers = all_registered_controllers();
assert_eq!(schemas.len(), controllers.len());
assert!(schemas.iter().all(|s| s.namespace == NAMESPACE));
assert!(controllers.iter().all(|c| c.schema.namespace == NAMESPACE));
}
#[test]
fn unknown_function_schema_returns_error_output() {
let schema = schemas("not_real");
assert_eq!(schema.namespace, NAMESPACE);
assert_eq!(schema.function, "unknown");
assert_eq!(schema.outputs.len(), 1);
assert_eq!(schema.outputs[0].name, "error");
}
#[test]
fn ingest_schema_requires_source_kind_source_id_and_payload() {
let schema = schemas("ingest");
assert_eq!(schema.function, "ingest");
let required: Vec<&str> = schema
.inputs
.iter()
.filter(|f| f.required)
.map(|f| f.name)
.collect();
assert!(required.contains(&"source_kind"));
assert!(required.contains(&"source_id"));
assert!(required.contains(&"payload"));
}
#[test]
fn set_llm_schema_exposes_backend_update_fields() {
let schema = schemas("set_llm");
let names: Vec<&str> = schema.inputs.iter().map(|f| f.name).collect();
assert!(names.contains(&"backend"));
assert!(names.contains(&"cloud_model"));
assert!(names.contains(&"extract_model"));
assert!(names.contains(&"summariser_model"));
}
}
+39
View File
@@ -535,3 +535,42 @@ fn handle_clear_namespace(params: Map<String, Value>) -> ControllerFuture {
to_json(rpc::clear_namespace(payload).await?)
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn documents_schema_exposes_all_functions() {
assert_eq!(controllers().len(), FUNCTIONS.len());
assert!(FUNCTIONS.contains(&"init"));
assert!(FUNCTIONS.contains(&"doc_ingest"));
assert!(FUNCTIONS.contains(&"clear_namespace"));
}
#[test]
fn unknown_document_schema_returns_none() {
assert!(schema("not_real").is_none());
}
#[test]
fn query_namespace_schema_requires_namespace_and_query() {
let schema = schema("query_namespace").unwrap();
let required: Vec<&str> = schema
.inputs
.iter()
.filter(|f| f.required)
.map(|f| f.name)
.collect();
assert!(required.contains(&"namespace"));
assert!(required.contains(&"query"));
}
#[test]
fn clear_namespace_schema_requires_namespace() {
let schema = schema("clear_namespace").unwrap();
assert_eq!(schema.inputs.len(), 1);
assert_eq!(schema.inputs[0].name, "namespace");
assert!(schema.inputs[0].required);
}
}
+29
View File
@@ -126,3 +126,32 @@ fn handle_write_file(params: Map<String, Value>) -> ControllerFuture {
to_json(rpc::ai_write_memory_file(payload).await?)
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn file_schema_exposes_all_functions() {
assert_eq!(FUNCTIONS, &["list_files", "read_file", "write_file"]);
assert_eq!(controllers().len(), FUNCTIONS.len());
}
#[test]
fn unknown_file_schema_returns_none() {
assert!(schema("not_real").is_none());
}
#[test]
fn write_file_schema_requires_path_and_content() {
let schema = schema("write_file").unwrap();
let required: Vec<&str> = schema
.inputs
.iter()
.filter(|f| f.required)
.map(|f| f.name)
.collect();
assert!(required.contains(&"relative_path"));
assert!(required.contains(&"content"));
}
}
+40
View File
@@ -267,3 +267,43 @@ fn handle_graph_query(params: Map<String, Value>) -> ControllerFuture {
to_json(rpc::graph_query(payload).await?)
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn kv_graph_schema_exposes_all_functions() {
assert_eq!(
FUNCTIONS,
&[
"kv_set",
"kv_get",
"kv_delete",
"kv_list_namespace",
"graph_upsert",
"graph_query",
]
);
assert_eq!(controllers().len(), FUNCTIONS.len());
}
#[test]
fn unknown_kv_graph_schema_returns_none() {
assert!(schema("not_real").is_none());
}
#[test]
fn graph_upsert_schema_requires_subject_predicate_and_object() {
let schema = schema("graph_upsert").unwrap();
let required: Vec<&str> = schema
.inputs
.iter()
.filter(|f| f.required)
.map(|f| f.name)
.collect();
assert!(required.contains(&"subject"));
assert!(required.contains(&"predicate"));
assert!(required.contains(&"object"));
}
}
+24
View File
@@ -54,3 +54,27 @@ fn handle_learn_all(params: Map<String, Value>) -> ControllerFuture {
to_json(rpc::memory_learn_all(payload).await?)
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn learn_schema_only_exposes_learn_all() {
assert_eq!(FUNCTIONS, &["learn_all"]);
assert_eq!(controllers().len(), 1);
}
#[test]
fn unknown_learn_schema_returns_none() {
assert!(schema("not_real").is_none());
}
#[test]
fn learn_all_schema_has_optional_namespaces_input() {
let schema = schema("learn_all").unwrap();
assert_eq!(schema.inputs.len(), 1);
assert_eq!(schema.inputs[0].name, "namespaces");
assert!(!schema.inputs[0].required);
}
}
+24
View File
@@ -85,3 +85,27 @@ fn handle_sync_all(_params: Map<String, Value>) -> ControllerFuture {
fn handle_ingestion_status(_params: Map<String, Value>) -> ControllerFuture {
Box::pin(async move { to_json(rpc::memory_ingestion_status().await?) })
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn sync_schema_exposes_all_functions() {
assert_eq!(FUNCTIONS, &["sync_channel", "sync_all", "ingestion_status"]);
assert_eq!(controllers().len(), FUNCTIONS.len());
}
#[test]
fn unknown_sync_schema_returns_none() {
assert!(schema("not_real").is_none());
}
#[test]
fn sync_channel_schema_requires_channel_id() {
let schema = schema("sync_channel").unwrap();
assert_eq!(schema.inputs.len(), 1);
assert_eq!(schema.inputs[0].name, "channel_id");
assert!(schema.inputs[0].required);
}
}
@@ -226,6 +226,52 @@ pub(super) fn schema(function: &str) -> Option<ControllerSchema> {
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn schema_exposes_all_tool_memory_functions() {
let functions: Vec<&str> = FUNCTIONS.to_vec();
assert_eq!(
functions,
vec![
"tool_rule_put",
"tool_rule_get",
"tool_rule_list",
"tool_rule_delete",
"tool_rules_for_prompt",
"tool_rules_json",
]
);
}
#[test]
fn controllers_match_function_count() {
let registered = controllers();
assert_eq!(registered.len(), FUNCTIONS.len());
assert!(registered.iter().all(|c| c.schema.namespace == "memory"));
}
#[test]
fn schema_returns_none_for_unknown_function() {
assert!(schema("not_real").is_none());
}
#[test]
fn tool_rule_put_schema_requires_tool_name_and_rule() {
let schema = schema("tool_rule_put").unwrap();
let input_names: Vec<&str> = schema.inputs.iter().map(|f| f.name).collect();
assert!(input_names.contains(&"tool_name"));
assert!(input_names.contains(&"rule"));
assert!(schema
.inputs
.iter()
.any(|f| f.name == "tool_name" && f.required));
assert!(schema.inputs.iter().any(|f| f.name == "rule" && f.required));
}
}
fn handle_tool_rule_put(params: Map<String, Value>) -> ControllerFuture {
Box::pin(async move {
let payload = parse_params::<ToolRulePutParams>(params)?;
@@ -77,7 +77,7 @@ impl EntityExtractor for CompositeExtractor {
#[cfg(test)]
mod tests {
use super::*;
use crate::openhuman::memory_tree::score::extract::EntityKind;
use crate::openhuman::memory::score::extract::EntityKind;
#[tokio::test]
async fn regex_only_extractor_works() {
@@ -35,7 +35,7 @@ use serde::Deserialize;
use super::types::{EntityKind, ExtractedEntities, ExtractedEntity, ExtractedTopic};
use super::EntityExtractor;
use crate::openhuman::memory_tree::chat::{ChatPrompt, ChatProvider};
use crate::openhuman::memory::chat::{ChatPrompt, ChatProvider};
// ── Configuration ────────────────────────────────────────────────────────
@@ -101,7 +101,7 @@ impl Default for LlmExtractorConfig {
/// Holds an `Arc<dyn ChatProvider>` rather than a per-instance HTTP
/// client. The provider abstraction lets a single workspace choose
/// cloud vs local at runtime (see
/// [`crate::openhuman::memory_tree::chat::build_chat_provider`]). Tests
/// [`crate::openhuman::memory::chat::build_chat_provider`]). Tests
/// can mock the provider to assert the prompt / parse behaviour without
/// a real Ollama or backend.
pub struct LlmEntityExtractor {
@@ -193,7 +193,7 @@ async fn extract_soft_fallback_on_provider_failure() {
// Provider always errors. extract() must NOT return Err — it must
// return an empty ExtractedEntities with a warn log after retry
// exhaustion.
use crate::openhuman::memory_tree::chat::{ChatPrompt, ChatProvider};
use crate::openhuman::memory::chat::{ChatPrompt, ChatProvider};
use async_trait::async_trait;
use std::sync::Arc;
@@ -220,7 +220,7 @@ async fn extract_routes_through_chat_provider_and_parses_response() {
// Mock provider returns canned NER+importance JSON. Verify the
// extractor parses it, recovers spans by string search, and emits the
// expected entities + importance signal.
use crate::openhuman::memory_tree::chat::{ChatPrompt, ChatProvider};
use crate::openhuman::memory::chat::{ChatPrompt, ChatProvider};
use async_trait::async_trait;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
@@ -266,7 +266,7 @@ async fn extract_returns_empty_on_malformed_provider_response() {
// Provider returns garbage. Caller must NOT see an Err — the parse
// failure path returns empty entities (retrying the same input would
// yield the same garbage, so we don't burn retries).
use crate::openhuman::memory_tree::chat::{ChatPrompt, ChatProvider};
use crate::openhuman::memory::chat::{ChatPrompt, ChatProvider};
use async_trait::async_trait;
use std::sync::Arc;
@@ -386,7 +386,7 @@ fn into_extracted_entities_disallowed_known_kind_falls_back_to_misc() {
#[test]
fn build_prompt_carries_user_text_and_kind_tag() {
use crate::openhuman::memory_tree::chat::{ChatPrompt, ChatProvider};
use crate::openhuman::memory::chat::{ChatPrompt, ChatProvider};
use async_trait::async_trait;
use std::sync::Arc;
+63
View File
@@ -0,0 +1,63 @@
//! Entity extraction (Phase 2 / #708).
//!
//! Exposes [`EntityExtractor`] as a pluggable interface and a default
//! [`CompositeExtractor`] that runs a chain of extractors and merges their
//! output. Phase 2 ships with the mechanical regex extractor only; semantic
//! NER (GLiNER / LLM) plugs in later without changing any call sites.
mod extractor;
pub mod llm;
pub mod regex;
pub mod types;
use std::sync::Arc;
use crate::openhuman::config::Config;
use crate::openhuman::memory::chat::build_chat_runtime;
pub use extractor::{CompositeExtractor, EntityExtractor, RegexEntityExtractor};
pub use llm::{LlmEntityExtractor, LlmExtractorConfig};
pub use types::{EntityKind, ExtractedEntities, ExtractedEntity, ExtractedTopic};
/// Build the extractor used by seal handlers to label new summary nodes.
///
/// Composition:
/// - regex extractor — always on, mechanical, near-zero cost
/// - LLM extractor with `emit_topics: true` — added when the unified
/// summarization workload can be built from inference routing.
///
/// Differs from [`super::ScoringConfig::from_config`] (the chunk-admission
/// builder) in two ways: returns *just* an extractor (no thresholds /
/// weights / drop logic — none of which apply at seal time), and flips
/// `emit_topics` on so summaries surface thematic labels alongside
/// entities. Leaf-side scoring is unchanged.
pub fn build_summary_extractor(config: &Config) -> Arc<dyn EntityExtractor> {
let (provider, model) = match build_chat_runtime(config) {
Ok(runtime) => runtime,
Err(err) => {
log::warn!(
"[memory_tree::extract] summary extractor: build_chat_runtime failed: \
{err:#} falling back to regex-only"
);
return Arc::new(CompositeExtractor::regex_only());
}
};
let cfg = LlmExtractorConfig {
model: model.clone(),
emit_topics: true,
output_language: config.output_language.clone(),
..LlmExtractorConfig::default()
};
log::debug!(
"[memory_tree::extract] summary extractor: regex + LLM provider={} model={} \
emit_topics=true",
provider.name(),
model
);
Arc::new(CompositeExtractor::new(vec![
Box::new(RegexEntityExtractor),
Box::new(LlmEntityExtractor::new(cfg, provider)),
]))
}
@@ -22,7 +22,7 @@ use serde::{Deserialize, Serialize};
use self::extract::{EntityExtractor, ExtractedEntities};
use self::resolver::{canonicalise, CanonicalEntity};
use self::signals::{ScoreSignals, SignalWeights};
use crate::openhuman::memory_tree::types::{approx_token_count, Chunk, SourceKind};
use crate::openhuman::memory_store::chunks::types::{approx_token_count, Chunk, SourceKind};
/// Default drop threshold. Chunks with `total < DEFAULT_DROP_THRESHOLD`
/// are tombstoned and never reach the chunk store.
@@ -105,29 +105,20 @@ impl ScoringConfig {
}
}
/// Build a [`ScoringConfig`] from the workspace [`Config`]. The
/// resolution rules match `build_summary_extractor`:
/// Build a [`ScoringConfig`] from the workspace [`Config`].
///
/// - `llm_backend = "cloud"` (default): always wires the LLM extractor
/// against the cloud provider, using the configured
/// `cloud_llm_model` (defaulting to `summarization-v1`).
/// - `llm_backend = "local"`: wires the LLM extractor only when both
/// `llm_extractor_endpoint` and `llm_extractor_model` are set;
/// otherwise falls back to [`Self::default_regex_only`].
///
/// Construction errors in the chat provider (rare — only client-builder
/// failures) fall back to regex-only with a warn log; scoring never
/// blocks on LLM availability.
/// The LLM extractor follows the unified summarization workload routing.
/// Construction errors fall back to regex-only with a warn log; scoring
/// never blocks on LLM availability.
pub fn from_config(config: &crate::openhuman::config::Config) -> Self {
use crate::openhuman::memory_tree::chat::{build_chat_provider, ChatConsumer};
use crate::openhuman::memory::chat::build_chat_runtime;
let model = match extract::resolve_extractor_model(config) {
Some(m) => m,
None => {
log::debug!(
"[memory_tree::score] llm_extractor not resolvable for memory_provider={:?} \
using regex-only",
config.memory_provider.as_deref().unwrap_or("cloud")
let (provider, model) = match build_chat_runtime(config) {
Ok(runtime) => runtime,
Err(err) => {
log::warn!(
"[memory::score] build_chat_runtime failed: {err:#} — \
falling back to regex-only"
);
return Self::default_regex_only();
}
@@ -139,23 +130,12 @@ impl ScoringConfig {
..extract::LlmExtractorConfig::default()
};
match build_chat_provider(config, ChatConsumer::Extract) {
Ok(provider) => {
log::info!(
"[memory_tree::score] using LlmEntityExtractor provider={} model={}",
provider.name(),
model
);
Self::with_llm_extractor(Arc::new(extract::LlmEntityExtractor::new(cfg, provider)))
}
Err(err) => {
log::warn!(
"[memory_tree::score] build_chat_provider failed: {err:#} — \
falling back to regex-only"
);
Self::default_regex_only()
}
}
log::info!(
"[memory::score] using LlmEntityExtractor provider={} model={}",
provider.name(),
model
);
Self::with_llm_extractor(Arc::new(extract::LlmEntityExtractor::new(cfg, provider)))
}
}
@@ -175,7 +155,7 @@ impl ScoringConfig {
/// 4. Apply final admission gate against `drop_threshold`.
pub async fn score_chunk(chunk: &Chunk, cfg: &ScoringConfig) -> Result<ScoreResult> {
log::debug!(
"[memory_tree::score] score_chunk chunk_id={} tokens={}",
"[memory::score] score_chunk chunk_id={} tokens={}",
chunk.id,
chunk.token_count
);
@@ -201,7 +181,7 @@ pub async fn score_chunk(chunk: &Chunk, cfg: &ScoringConfig) -> Result<ScoreResu
let llm_consulted = if in_band {
if let Some(llm) = cfg.llm_extractor.as_ref() {
log::debug!(
"[memory_tree::score] borderline chunk_id={} cheap_total={:.3} — consulting LLM",
"[memory::score] borderline chunk_id={} cheap_total={:.3} — consulting LLM",
chunk.id,
cheap_total
);
@@ -219,7 +199,7 @@ pub async fn score_chunk(chunk: &Chunk, cfg: &ScoringConfig) -> Result<ScoreResu
}
Err(e) => {
log::warn!(
"[memory_tree::score] LLM extractor `{}` failed: {e} — \
"[memory::score] LLM extractor `{}` failed: {e} — \
falling back to cheap signals only",
llm.name()
);
@@ -231,7 +211,7 @@ pub async fn score_chunk(chunk: &Chunk, cfg: &ScoringConfig) -> Result<ScoreResu
}
} else {
log::debug!(
"[memory_tree::score] short-circuit chunk_id={} cheap_total={:.3} \
"[memory::score] short-circuit chunk_id={} cheap_total={:.3} \
({}, skipping LLM)",
chunk.id,
cheap_total,
@@ -289,7 +269,7 @@ pub async fn score_chunk(chunk: &Chunk, cfg: &ScoringConfig) -> Result<ScoreResu
if !kept {
log::debug!(
"[memory_tree::score] drop chunk_id={} total={:.3} reason={:?} llm_consulted={}",
"[memory::score] drop chunk_id={} total={:.3} reason={:?} llm_consulted={}",
chunk.id,
total,
drop_reason,
@@ -1,5 +1,5 @@
use super::*;
use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind};
use crate::openhuman::memory_store::chunks::types::{chunk_id, Chunk, Metadata, SourceKind};
use chrono::Utc;
fn test_chunk(content: &str) -> Chunk {
@@ -7,7 +7,7 @@ fn test_chunk(content: &str) -> Chunk {
Chunk {
id: chunk_id(SourceKind::Email, "t1", 0, "test-content"),
content: content.to_string(),
token_count: crate::openhuman::memory_tree::types::approx_token_count(content),
token_count: crate::openhuman::memory_store::chunks::types::approx_token_count(content),
metadata: meta,
seq_in_source: 0,
created_at: Utc::now(),
@@ -9,7 +9,7 @@
use serde::{Deserialize, Serialize};
use crate::openhuman::memory_tree::score::extract::{EntityKind, ExtractedEntities};
use crate::openhuman::memory::score::extract::{EntityKind, ExtractedEntities};
/// Canonicalised entity — same shape as [`ExtractedEntity`] plus a stable
/// `canonical_id` suitable for indexing.
@@ -103,7 +103,7 @@ pub fn canonical_id_for(kind: EntityKind, surface: &str) -> String {
#[cfg(test)]
mod tests {
use super::*;
use crate::openhuman::memory_tree::score::extract::ExtractedEntity;
use crate::openhuman::memory::score::extract::ExtractedEntity;
fn entity(kind: EntityKind, text: &str) -> ExtractedEntity {
ExtractedEntity {
@@ -173,7 +173,7 @@ mod tests {
// ── Topic canonicalisation (#709 / Phase 3c topic-tree scope) ────
use crate::openhuman::memory_tree::score::extract::ExtractedTopic;
use crate::openhuman::memory::score::extract::ExtractedTopic;
fn topic(label: &str, score: f32) -> ExtractedTopic {
ExtractedTopic {
@@ -13,7 +13,7 @@
//! Ingest adapters can attach these tags during canonicalisation when the
//! upstream source supports the distinction. Absent tags → neutral score.
use crate::openhuman::memory_tree::types::Metadata;
use crate::openhuman::memory_store::chunks::types::Metadata;
/// Tag set when the user replied to this message/thread.
pub const TAG_REPLY: &str = "reply";
@@ -67,7 +67,7 @@ pub fn score(meta: &Metadata) -> f32 {
#[cfg(test)]
mod tests {
use super::*;
use crate::openhuman::memory_tree::types::SourceKind;
use crate::openhuman::memory_store::chunks::types::SourceKind;
use chrono::Utc;
fn meta(tags: &[&str]) -> Metadata {
@@ -8,7 +8,7 @@
//! context (e.g., channel size, thread participant count) is a future
//! refinement when we actually have that metadata at ingest.
use crate::openhuman::memory_tree::types::{Metadata, SourceKind};
use crate::openhuman::memory_store::chunks::types::{Metadata, SourceKind};
/// Base weight for each source kind.
///
@@ -3,8 +3,8 @@
use super::{interaction, metadata_weight, source_weight, token_count, unique_words};
use super::{ScoreSignals, SignalWeights};
use crate::openhuman::memory_tree::score::extract::ExtractedEntities;
use crate::openhuman::memory_tree::types::Metadata;
use crate::openhuman::memory::score::extract::ExtractedEntities;
use crate::openhuman::memory_store::chunks::types::Metadata;
/// Compute all signals for a chunk.
///
@@ -95,10 +95,10 @@ pub fn combine_cheap_only(signals: &ScoreSignals, w: &SignalWeights) -> f32 {
#[cfg(test)]
mod tests {
use super::*;
use crate::openhuman::memory_tree::score::extract::{
use crate::openhuman::memory::score::extract::{
EntityKind, ExtractedEntities, ExtractedEntity,
};
use crate::openhuman::memory_tree::types::SourceKind;
use crate::openhuman::memory_store::chunks::types::SourceKind;
use chrono::Utc;
fn meta(tags: &[&str], kind: SourceKind) -> Metadata {
@@ -10,7 +10,7 @@
//! Finer distinction (DM vs channel on Slack specifically) requires richer
//! ingest-time metadata and is deferred.
use crate::openhuman::memory_tree::types::{DataSource, Metadata, SourceKind};
use crate::openhuman::memory_store::chunks::types::{DataSource, Metadata, SourceKind};
const PROVIDER_PREFIX: &str = "provider:";
@@ -63,3 +63,45 @@ impl SignalWeights {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn score_signals_default_to_zero() {
let signals = ScoreSignals::default();
assert_eq!(signals.token_count, 0.0);
assert_eq!(signals.unique_words, 0.0);
assert_eq!(signals.metadata_weight, 0.0);
assert_eq!(signals.source_weight, 0.0);
assert_eq!(signals.interaction, 0.0);
assert_eq!(signals.entity_density, 0.0);
assert_eq!(signals.llm_importance, 0.0);
}
#[test]
fn signal_weights_default_match_expected_priorities() {
let weights = SignalWeights::default();
assert_eq!(weights.token_count, 1.0);
assert_eq!(weights.unique_words, 1.0);
assert_eq!(weights.metadata_weight, 1.5);
assert_eq!(weights.source_weight, 1.5);
assert_eq!(weights.interaction, 3.0);
assert_eq!(weights.entity_density, 1.0);
assert_eq!(weights.llm_importance, 0.0);
}
#[test]
fn with_llm_enabled_only_changes_llm_weight() {
let default = SignalWeights::default();
let enabled = SignalWeights::with_llm_enabled();
assert_eq!(enabled.token_count, default.token_count);
assert_eq!(enabled.unique_words, default.unique_words);
assert_eq!(enabled.metadata_weight, default.metadata_weight);
assert_eq!(enabled.source_weight, default.source_weight);
assert_eq!(enabled.interaction, default.interaction);
assert_eq!(enabled.entity_density, default.entity_density);
assert_eq!(enabled.llm_importance, 2.0);
}
}
@@ -14,10 +14,10 @@ use serde::{Deserialize, Serialize};
use crate::openhuman::composio::providers::profile::{is_self_identity_any_toolkit, IdentityKind};
use crate::openhuman::config::Config;
use crate::openhuman::memory_tree::score::extract::EntityKind;
use crate::openhuman::memory_tree::score::resolver::CanonicalEntity;
use crate::openhuman::memory_tree::score::signals::ScoreSignals;
use crate::openhuman::memory_tree::store::with_connection;
use crate::openhuman::memory::score::extract::EntityKind;
use crate::openhuman::memory::score::resolver::CanonicalEntity;
use crate::openhuman::memory::score::signals::ScoreSignals;
use crate::openhuman::memory_store::chunks::store::with_connection;
/// Map a memory-tree `EntityKind` to the Composio identity-registry
/// [`IdentityKind`] used for self-matching, or `None` for kinds that
@@ -304,7 +304,7 @@ pub(crate) fn index_summary_entity_ids_tx(
Some((kind, _)) => kind,
None => {
log::warn!(
"[memory_tree::score::store] summary entity id missing ':' — \
"[memory::score::store] summary entity id missing ':' — \
storing as-is: {canonical_id}"
);
canonical_id.as_str()

Some files were not shown because too many files have changed in this diff Show More