mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-27 21:08:00 +00:00
test(memory): extend memory coverage across retrieval and tooling (#2574)
This commit is contained in:
@@ -14,6 +14,7 @@ import { waitForApp, waitForAppReady } from '../helpers/app-helpers';
|
|||||||
import { hasAppChrome } from '../helpers/element-helpers';
|
import { hasAppChrome } from '../helpers/element-helpers';
|
||||||
import { resetApp } from '../helpers/reset-app';
|
import { resetApp } from '../helpers/reset-app';
|
||||||
import { waitForHomePage } from '../helpers/shared-flows';
|
import { waitForHomePage } from '../helpers/shared-flows';
|
||||||
|
import { startMockServer, stopMockServer } from '../mock-server';
|
||||||
|
|
||||||
const USER_ID = 'e2e-smoke';
|
const USER_ID = 'e2e-smoke';
|
||||||
|
|
||||||
@@ -21,10 +22,15 @@ describe('Smoke', function () {
|
|||||||
this.timeout(120_000);
|
this.timeout(120_000);
|
||||||
|
|
||||||
before(async () => {
|
before(async () => {
|
||||||
|
await startMockServer();
|
||||||
await waitForApp();
|
await waitForApp();
|
||||||
await resetApp(USER_ID);
|
await resetApp(USER_ID);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
after(async () => {
|
||||||
|
await stopMockServer();
|
||||||
|
});
|
||||||
|
|
||||||
it('has a live WebDriver session', async () => {
|
it('has a live WebDriver session', async () => {
|
||||||
const sessionId = browser.sessionId;
|
const sessionId = browser.sessionId;
|
||||||
expect(sessionId).toBeDefined();
|
expect(sessionId).toBeDefined();
|
||||||
|
|||||||
@@ -41,13 +41,13 @@ use openhuman_core::openhuman::composio::providers::registry::{
|
|||||||
get_provider, init_default_providers,
|
get_provider, init_default_providers,
|
||||||
};
|
};
|
||||||
use openhuman_core::openhuman::config::Config;
|
use openhuman_core::openhuman::config::Config;
|
||||||
use openhuman_core::openhuman::memory_tree::content_store::read::{
|
use openhuman_core::openhuman::memory::jobs::drain_until_idle;
|
||||||
verify_chunk_file, verify_summary_file, VerifyResult,
|
use openhuman_core::openhuman::memory_store::chunks::store::{
|
||||||
};
|
|
||||||
use openhuman_core::openhuman::memory_tree::jobs::drain_until_idle;
|
|
||||||
use openhuman_core::openhuman::memory_tree::store::{
|
|
||||||
get_chunk_content_pointers, list_chunks, list_summaries_with_content_path, ListChunksQuery,
|
get_chunk_content_pointers, list_chunks, list_summaries_with_content_path, ListChunksQuery,
|
||||||
};
|
};
|
||||||
|
use openhuman_core::openhuman::memory_store::content::read::{
|
||||||
|
verify_chunk_file, verify_summary_file, VerifyResult,
|
||||||
|
};
|
||||||
|
|
||||||
#[derive(Parser, Debug)]
|
#[derive(Parser, Debug)]
|
||||||
#[command(
|
#[command(
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ use std::sync::atomic::{AtomicUsize, Ordering};
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use openhuman_core::openhuman::config::Config;
|
use openhuman_core::openhuman::config::Config;
|
||||||
use openhuman_core::openhuman::memory_tree::store::with_connection;
|
use openhuman_core::openhuman::memory_store::chunks::store::with_connection;
|
||||||
|
|
||||||
fn main() -> ExitCode {
|
fn main() -> ExitCode {
|
||||||
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info"))
|
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info"))
|
||||||
|
|||||||
@@ -211,8 +211,8 @@ async fn main() -> Result<()> {
|
|||||||
|
|
||||||
if cli.seal_probe {
|
if cli.seal_probe {
|
||||||
use chrono::{Duration, Utc};
|
use chrono::{Duration, Utc};
|
||||||
use openhuman_core::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage};
|
use openhuman_core::openhuman::memory::ingest_pipeline::ingest_chat;
|
||||||
use openhuman_core::openhuman::memory_tree::ingest::ingest_chat;
|
use openhuman_core::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage};
|
||||||
|
|
||||||
let connection_id = cli.connection_id.clone().ok_or_else(|| {
|
let connection_id = cli.connection_id.clone().ok_or_else(|| {
|
||||||
anyhow::anyhow!(
|
anyhow::anyhow!(
|
||||||
|
|||||||
+3
-1
@@ -68,7 +68,9 @@ pub fn run_from_cli_args(args: &[String]) -> Result<()> {
|
|||||||
}
|
}
|
||||||
"text-input" => crate::openhuman::text_input::cli::run_text_input_command(&args[1..]),
|
"text-input" => crate::openhuman::text_input::cli::run_text_input_command(&args[1..]),
|
||||||
"tree-summarizer" => {
|
"tree-summarizer" => {
|
||||||
crate::openhuman::memory_tree::summarizer::cli::run_tree_summarizer_command(&args[1..])
|
crate::openhuman::memory_tree::tree_runtime::cli::run_tree_summarizer_command(
|
||||||
|
&args[1..],
|
||||||
|
)
|
||||||
}
|
}
|
||||||
"memory" => crate::core::memory_cli::run_memory_command(&args[1..]),
|
"memory" => crate::core::memory_cli::run_memory_command(&args[1..]),
|
||||||
"agent" => {
|
"agent" => {
|
||||||
|
|||||||
+1
-1
@@ -1558,7 +1558,7 @@ fn register_domain_subscribers(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
crate::openhuman::memory_tree::jobs::start(config.clone());
|
crate::openhuman::memory::jobs::start(config.clone());
|
||||||
|
|
||||||
// Restart requests go through a subscriber so every trigger path shares
|
// Restart requests go through a subscriber so every trigger path shares
|
||||||
// the same respawn logic.
|
// the same respawn logic.
|
||||||
|
|||||||
@@ -18,23 +18,18 @@
|
|||||||
|
|
||||||
use crate::openhuman::agent::hooks::{PostTurnHook, TurnContext};
|
use crate::openhuman::agent::hooks::{PostTurnHook, TurnContext};
|
||||||
use crate::openhuman::config::Config;
|
use crate::openhuman::config::Config;
|
||||||
use crate::openhuman::memory::store::events::{self, EventRecord, EventType};
|
use crate::openhuman::memory::chat::ChatProvider;
|
||||||
use crate::openhuman::memory::store::fts5::{self, EpisodicEntry};
|
use crate::openhuman::memory::ingest_pipeline;
|
||||||
use crate::openhuman::memory::store::profile::{self, FacetType};
|
use crate::openhuman::memory::score::embed::{build_embedder_from_config, Embedder};
|
||||||
use crate::openhuman::memory::store::segments::{
|
use crate::openhuman::memory_store::events::{self, EventRecord, EventType};
|
||||||
|
use crate::openhuman::memory_store::fts5::{self, EpisodicEntry};
|
||||||
|
use crate::openhuman::memory_store::profile::{self, FacetType};
|
||||||
|
use crate::openhuman::memory_store::segments::{
|
||||||
self, BoundaryConfig, BoundaryDecision, ConversationSegment,
|
self, BoundaryConfig, BoundaryDecision, ConversationSegment,
|
||||||
};
|
};
|
||||||
use crate::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage};
|
use crate::openhuman::memory_store::trees::types::TreeKind;
|
||||||
use crate::openhuman::memory_tree::chat::{ChatConsumer, ChatProvider};
|
use crate::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage};
|
||||||
use crate::openhuman::memory_tree::ingest;
|
use crate::openhuman::memory_tree::summarise::{summarise, SummaryContext, SummaryInput};
|
||||||
use crate::openhuman::memory_tree::score::embed::{build_embedder_from_config, Embedder};
|
|
||||||
use crate::openhuman::memory_tree::tree_source::summariser::llm::{
|
|
||||||
LlmSummariser, LlmSummariserConfig,
|
|
||||||
};
|
|
||||||
use crate::openhuman::memory_tree::tree_source::summariser::{
|
|
||||||
Summariser, SummaryContext, SummaryInput,
|
|
||||||
};
|
|
||||||
use crate::openhuman::memory_tree::tree_source::types::TreeKind;
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
use rusqlite::Connection;
|
use rusqlite::Connection;
|
||||||
@@ -98,10 +93,7 @@ impl ArchivistHook {
|
|||||||
pub fn with_config(mut self, config: Config) -> Self {
|
pub fn with_config(mut self, config: Config) -> Self {
|
||||||
// Build the LLM chat provider for segment recap.
|
// Build the LLM chat provider for segment recap.
|
||||||
let chat_provider: Option<Arc<dyn ChatProvider>> =
|
let chat_provider: Option<Arc<dyn ChatProvider>> =
|
||||||
match crate::openhuman::memory_tree::chat::build_chat_provider(
|
match crate::openhuman::memory::chat::build_chat_provider(&config) {
|
||||||
&config,
|
|
||||||
ChatConsumer::Summarise,
|
|
||||||
) {
|
|
||||||
Ok(p) => {
|
Ok(p) => {
|
||||||
tracing::debug!("[archivist] segment recap provider={} registered", p.name());
|
tracing::debug!("[archivist] segment recap provider={} registered", p.name());
|
||||||
Some(p)
|
Some(p)
|
||||||
@@ -207,6 +199,7 @@ impl ArchivistHook {
|
|||||||
timestamp: f64,
|
timestamp: f64,
|
||||||
user_message: &str,
|
user_message: &str,
|
||||||
current_episodic_id: i64,
|
current_episodic_id: i64,
|
||||||
|
current_seq: Option<u32>,
|
||||||
) -> Option<ConversationSegment> {
|
) -> Option<ConversationSegment> {
|
||||||
let now = Self::now_timestamp();
|
let now = Self::now_timestamp();
|
||||||
|
|
||||||
@@ -241,6 +234,7 @@ impl ArchivistHook {
|
|||||||
conn,
|
conn,
|
||||||
&segment.segment_id,
|
&segment.segment_id,
|
||||||
current_episodic_id,
|
current_episodic_id,
|
||||||
|
current_seq,
|
||||||
timestamp,
|
timestamp,
|
||||||
now,
|
now,
|
||||||
) {
|
) {
|
||||||
@@ -269,6 +263,7 @@ impl ArchivistHook {
|
|||||||
session_id,
|
session_id,
|
||||||
"global",
|
"global",
|
||||||
current_episodic_id,
|
current_episodic_id,
|
||||||
|
current_seq,
|
||||||
timestamp,
|
timestamp,
|
||||||
now,
|
now,
|
||||||
) {
|
) {
|
||||||
@@ -293,6 +288,7 @@ impl ArchivistHook {
|
|||||||
session_id,
|
session_id,
|
||||||
"global",
|
"global",
|
||||||
current_episodic_id,
|
current_episodic_id,
|
||||||
|
current_seq,
|
||||||
timestamp,
|
timestamp,
|
||||||
now,
|
now,
|
||||||
) {
|
) {
|
||||||
@@ -318,8 +314,10 @@ impl ArchivistHook {
|
|||||||
session_id: &str,
|
session_id: &str,
|
||||||
now: f64,
|
now: f64,
|
||||||
) {
|
) {
|
||||||
// Gather the conversation text for this segment from episodic entries.
|
// Gather the conversation text for this segment. Prefer the
|
||||||
let entries = fts5::episodic_session_entries(conn, session_id).unwrap_or_default();
|
// md-backed memory_archivist read when config is available; fall
|
||||||
|
// back to FTS5 in test paths or when config isn't wired.
|
||||||
|
let entries = self.read_session_entries(conn, session_id);
|
||||||
|
|
||||||
// Filter entries that fall within the segment's time window.
|
// Filter entries that fall within the segment's time window.
|
||||||
// Use <= for end_timestamp (entries at the boundary are part of this
|
// Use <= for end_timestamp (entries at the boundary are part of this
|
||||||
@@ -583,6 +581,56 @@ impl PostTurnHook for ArchivistHook {
|
|||||||
|
|
||||||
tracing::debug!("[archivist] episodic rows written: session={session_id}");
|
tracing::debug!("[archivist] episodic rows written: session={session_id}");
|
||||||
|
|
||||||
|
// Dual-write into memory_archivist::store (md-backed) so we can
|
||||||
|
// validate the FTS5 → md migration before flipping the read side.
|
||||||
|
// Best-effort: a write failure here must not break the turn. The
|
||||||
|
// user turn's assigned seq is captured into `current_seq` so the
|
||||||
|
// segment ops can store it alongside the FTS5 episodic id.
|
||||||
|
let mut current_seq: Option<u32> = None;
|
||||||
|
if let Some(cfg) = self.config.as_ref() {
|
||||||
|
let ts_ms = (timestamp * 1000.0) as i64;
|
||||||
|
let user_turn = crate::openhuman::memory_archivist::ArchivedTurn {
|
||||||
|
session_id: session_id.to_string(),
|
||||||
|
seq: 0, // assigned by record_turn
|
||||||
|
timestamp_ms: ts_ms,
|
||||||
|
role: "user".to_string(),
|
||||||
|
content: ctx.user_message.clone(),
|
||||||
|
lesson: None,
|
||||||
|
tool_calls_json: None,
|
||||||
|
cost_microdollars: 0,
|
||||||
|
};
|
||||||
|
match crate::openhuman::memory_archivist::store::record_turn(cfg, user_turn) {
|
||||||
|
Ok(stored) => current_seq = Some(stored.seq),
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!("[archivist] memory_archivist user dual-write failed: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Assistant turn carries the tool_calls_json + lesson the FTS5
|
||||||
|
// insert just wrote. Re-derive locally so we don't depend on
|
||||||
|
// FTS5 having returned.
|
||||||
|
let assistant_lesson = extract_lesson_from_tools(&ctx.tool_calls);
|
||||||
|
let assistant_tool_calls = if ctx.tool_calls.is_empty() {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(serde_json::to_string(&ctx.tool_calls).unwrap_or_default())
|
||||||
|
};
|
||||||
|
let assistant_turn = crate::openhuman::memory_archivist::ArchivedTurn {
|
||||||
|
session_id: session_id.to_string(),
|
||||||
|
seq: 0,
|
||||||
|
timestamp_ms: ts_ms + 1,
|
||||||
|
role: "assistant".to_string(),
|
||||||
|
content: ctx.assistant_response.clone(),
|
||||||
|
lesson: assistant_lesson,
|
||||||
|
tool_calls_json: assistant_tool_calls,
|
||||||
|
cost_microdollars: 0,
|
||||||
|
};
|
||||||
|
if let Err(e) =
|
||||||
|
crate::openhuman::memory_archivist::store::record_turn(cfg, assistant_turn)
|
||||||
|
{
|
||||||
|
tracing::warn!("[archivist] memory_archivist assistant dual-write failed: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Manage conversation segmentation (sync boundary detection + SQLite
|
// Manage conversation segmentation (sync boundary detection + SQLite
|
||||||
// operations). Returns the just-closed segment when a boundary fired.
|
// operations). Returns the just-closed segment when a boundary fired.
|
||||||
let closed_segment = self.manage_segment_sync(
|
let closed_segment = self.manage_segment_sync(
|
||||||
@@ -591,6 +639,7 @@ impl PostTurnHook for ArchivistHook {
|
|||||||
timestamp,
|
timestamp,
|
||||||
&ctx.user_message,
|
&ctx.user_message,
|
||||||
current_episodic_id,
|
current_episodic_id,
|
||||||
|
current_seq,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Run async recap + embed + segment-tree ingest on the closed segment
|
// Run async recap + embed + segment-tree ingest on the closed segment
|
||||||
@@ -607,6 +656,47 @@ impl PostTurnHook for ArchivistHook {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl ArchivistHook {
|
impl ArchivistHook {
|
||||||
|
/// Read every entry recorded for `session_id`, preferring the
|
||||||
|
/// md-backed `memory_archivist::store` when `self.config` is set and
|
||||||
|
/// falling back to the legacy FTS5 episodic table otherwise.
|
||||||
|
///
|
||||||
|
/// Returns `EpisodicEntry` so the existing call sites (segment
|
||||||
|
/// gathering, recap rendering, tree push) keep their shape unchanged
|
||||||
|
/// during the FTS5 retirement migration.
|
||||||
|
fn read_session_entries(
|
||||||
|
&self,
|
||||||
|
conn: &Arc<Mutex<Connection>>,
|
||||||
|
session_id: &str,
|
||||||
|
) -> Vec<EpisodicEntry> {
|
||||||
|
if let Some(cfg) = self.config.as_ref() {
|
||||||
|
match crate::openhuman::memory_archivist::store::session_entries(cfg, session_id) {
|
||||||
|
Ok(turns) => {
|
||||||
|
return turns
|
||||||
|
.into_iter()
|
||||||
|
.map(|t| EpisodicEntry {
|
||||||
|
id: None,
|
||||||
|
session_id: t.session_id,
|
||||||
|
// ArchivedTurn stores epoch-ms; EpisodicEntry
|
||||||
|
// takes epoch-seconds as f64.
|
||||||
|
timestamp: (t.timestamp_ms as f64) / 1000.0,
|
||||||
|
role: t.role,
|
||||||
|
content: t.content,
|
||||||
|
lesson: t.lesson,
|
||||||
|
tool_calls_json: t.tool_calls_json,
|
||||||
|
cost_microdollars: t.cost_microdollars,
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!(
|
||||||
|
"[archivist] memory_archivist read failed (falling back to FTS5): {e}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fts5::episodic_session_entries(conn, session_id).unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
/// Shared summarize helper — the **single LLM summarizer** used by both
|
/// Shared summarize helper — the **single LLM summarizer** used by both
|
||||||
/// the finalize path (`on_segment_closed`) and the rolling-recap path
|
/// the finalize path (`on_segment_closed`) and the rolling-recap path
|
||||||
/// (`rolling_segment_recap`).
|
/// (`rolling_segment_recap`).
|
||||||
@@ -650,7 +740,7 @@ impl ArchivistHook {
|
|||||||
.iter()
|
.iter()
|
||||||
.filter(|e| !e.content.trim().is_empty())
|
.filter(|e| !e.content.trim().is_empty())
|
||||||
.map(|e| {
|
.map(|e| {
|
||||||
use crate::openhuman::memory_tree::types::approx_token_count;
|
use crate::openhuman::memory_store::chunks::types::approx_token_count;
|
||||||
let content = e.content.clone();
|
let content = e.content.clone();
|
||||||
let token_count = approx_token_count(&content);
|
let token_count = approx_token_count(&content);
|
||||||
let ts = chrono::DateTime::from_timestamp(e.timestamp as i64, 0)
|
let ts = chrono::DateTime::from_timestamp(e.timestamp as i64, 0)
|
||||||
@@ -678,53 +768,60 @@ impl ArchivistHook {
|
|||||||
let first = entries.first().map(|e| e.content.as_str()).unwrap_or("");
|
let first = entries.first().map(|e| e.content.as_str()).unwrap_or("");
|
||||||
let last = entries.last().map(|e| e.content.as_str()).unwrap_or(first);
|
let last = entries.last().map(|e| e.content.as_str()).unwrap_or(first);
|
||||||
|
|
||||||
if let Some(ref provider) = self.chat_provider {
|
if self.chat_provider.is_some() {
|
||||||
let cfg = LlmSummariserConfig {
|
if let Some(ref config) = self.config {
|
||||||
model: provider.name().to_string(),
|
tracing::debug!(
|
||||||
structured_facet_extraction: false,
|
"[archivist] summarize_entries: LLM recap segment={segment_id} entries={}",
|
||||||
output_language: self
|
entries.len()
|
||||||
.config
|
);
|
||||||
.as_ref()
|
#[cfg(test)]
|
||||||
.and_then(|cfg| cfg.output_language.clone()),
|
let summary_result = if let Some(provider) = self.chat_provider.as_ref() {
|
||||||
};
|
crate::openhuman::memory::chat::test_override::with_provider(
|
||||||
let summariser = LlmSummariser::new(cfg, Arc::clone(provider));
|
Arc::clone(provider),
|
||||||
tracing::debug!(
|
summarise(config, &corpus_inputs, &summary_ctx),
|
||||||
"[archivist] summarize_entries: LLM recap segment={segment_id} \
|
)
|
||||||
provider={} entries={}",
|
.await
|
||||||
provider.name(),
|
} else {
|
||||||
entries.len()
|
summarise(config, &corpus_inputs, &summary_ctx).await
|
||||||
);
|
};
|
||||||
match summariser.summarise(&corpus_inputs, &summary_ctx).await {
|
#[cfg(not(test))]
|
||||||
Ok(output) if !output.content.is_empty() => {
|
let summary_result = summarise(config, &corpus_inputs, &summary_ctx).await;
|
||||||
tracing::debug!(
|
|
||||||
"[archivist] summarize_entries: LLM recap ok segment={segment_id} \
|
match summary_result {
|
||||||
chars={}",
|
Ok(output) if !output.content.is_empty() => {
|
||||||
output.content.len()
|
tracing::debug!(
|
||||||
);
|
"[archivist] summarize_entries: LLM recap ok segment={segment_id} \
|
||||||
(output.content, true)
|
chars={}",
|
||||||
}
|
output.content.len()
|
||||||
Ok(_) => {
|
);
|
||||||
tracing::debug!(
|
return (output.content, true);
|
||||||
"[archivist] summarize_entries: LLM returned empty — \
|
}
|
||||||
heuristic fallback segment={segment_id}"
|
Ok(_) => {
|
||||||
);
|
tracing::debug!(
|
||||||
(segments::fallback_summary(first, last, turn_count), false)
|
"[archivist] summarize_entries: LLM returned empty — \
|
||||||
}
|
heuristic fallback segment={segment_id}"
|
||||||
Err(e) => {
|
);
|
||||||
tracing::warn!(
|
}
|
||||||
"[archivist] summarize_entries: LLM recap failed (non-fatal) \
|
Err(e) => {
|
||||||
segment={segment_id}: {e} — heuristic fallback"
|
tracing::warn!(
|
||||||
);
|
"[archivist] summarize_entries: LLM recap failed (non-fatal) \
|
||||||
(segments::fallback_summary(first, last, turn_count), false)
|
segment={segment_id}: {e} — heuristic fallback"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
tracing::debug!(
|
||||||
|
"[archivist] summarize_entries: no config — \
|
||||||
|
heuristic fallback segment={segment_id}"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
tracing::debug!(
|
tracing::debug!(
|
||||||
"[archivist] summarize_entries: no chat provider — \
|
"[archivist] summarize_entries: no chat provider — \
|
||||||
heuristic fallback segment={segment_id}"
|
heuristic fallback segment={segment_id}"
|
||||||
);
|
);
|
||||||
(segments::fallback_summary(first, last, turn_count), false)
|
|
||||||
}
|
}
|
||||||
|
(segments::fallback_summary(first, last, turn_count), false)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Produce a rolling recap of the **currently-open** segment for
|
/// Produce a rolling recap of the **currently-open** segment for
|
||||||
@@ -782,7 +879,7 @@ impl ArchivistHook {
|
|||||||
};
|
};
|
||||||
|
|
||||||
// Gather the episodic entries for this session so far.
|
// Gather the episodic entries for this session so far.
|
||||||
let all_entries = fts5::episodic_session_entries(conn, session_id).unwrap_or_default();
|
let all_entries = self.read_session_entries(conn, session_id);
|
||||||
|
|
||||||
// Keep only entries within the open segment's time window (start →
|
// Keep only entries within the open segment's time window (start →
|
||||||
// now, inclusive). An open segment has `end_timestamp = None`.
|
// now, inclusive). An open segment has `end_timestamp = None`.
|
||||||
@@ -867,7 +964,7 @@ impl ArchivistHook {
|
|||||||
async fn pipe_segment_to_tree(
|
async fn pipe_segment_to_tree(
|
||||||
&self,
|
&self,
|
||||||
config: &Config,
|
config: &Config,
|
||||||
segment: &crate::openhuman::memory::store::segments::ConversationSegment,
|
segment: &crate::openhuman::memory_store::segments::ConversationSegment,
|
||||||
session_id: &str,
|
session_id: &str,
|
||||||
entries: &[&fts5::EpisodicEntry],
|
entries: &[&fts5::EpisodicEntry],
|
||||||
) {
|
) {
|
||||||
@@ -945,7 +1042,7 @@ impl ArchivistHook {
|
|||||||
segment={segment_id} ep_span={start_ep}-{end_ep} provenance={provenance}"
|
segment={segment_id} ep_span={start_ep}-{end_ep} provenance={provenance}"
|
||||||
);
|
);
|
||||||
|
|
||||||
match ingest::ingest_chat(config, source_id, owner, tags, batch).await {
|
match ingest_pipeline::ingest_chat(config, source_id, owner, tags, batch).await {
|
||||||
Ok(result) => {
|
Ok(result) => {
|
||||||
tracing::debug!(
|
tracing::debug!(
|
||||||
"[archivist] tree ingest ok: source_id={source_id} \
|
"[archivist] tree ingest ok: source_id={source_id} \
|
||||||
@@ -1118,7 +1215,7 @@ impl ArchivistHook {
|
|||||||
conn: Some(conn),
|
conn: Some(conn),
|
||||||
enabled: true,
|
enabled: true,
|
||||||
boundary_config: BoundaryConfig::default(),
|
boundary_config: BoundaryConfig::default(),
|
||||||
config: None,
|
config: Some(Config::default()),
|
||||||
chat_provider: Some(chat_provider),
|
chat_provider: Some(chat_provider),
|
||||||
embedder: Some(embedder),
|
embedder: Some(embedder),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
use super::*;
|
use super::*;
|
||||||
use crate::openhuman::agent::hooks::{ToolCallRecord, TurnContext};
|
use crate::openhuman::agent::hooks::{ToolCallRecord, TurnContext};
|
||||||
use crate::openhuman::memory::store::{events as ev, fts5, segments as seg};
|
use crate::openhuman::memory::chat::ChatPrompt;
|
||||||
use crate::openhuman::memory_tree::chat::ChatPrompt;
|
use crate::openhuman::memory_store::{events as ev, fts5, segments as seg};
|
||||||
|
|
||||||
fn setup_conn() -> Arc<Mutex<Connection>> {
|
fn setup_conn() -> Arc<Mutex<Connection>> {
|
||||||
let conn = Connection::open_in_memory().unwrap();
|
let conn = Connection::open_in_memory().unwrap();
|
||||||
@@ -343,7 +343,7 @@ async fn phase0_episodic_rows_and_segment_without_learning_enabled() {
|
|||||||
struct StubChatProvider;
|
struct StubChatProvider;
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait]
|
||||||
impl crate::openhuman::memory_tree::chat::ChatProvider for StubChatProvider {
|
impl crate::openhuman::memory::chat::ChatProvider for StubChatProvider {
|
||||||
fn name(&self) -> &str {
|
fn name(&self) -> &str {
|
||||||
"stub:test"
|
"stub:test"
|
||||||
}
|
}
|
||||||
@@ -361,7 +361,7 @@ impl crate::openhuman::memory_tree::chat::ChatProvider for StubChatProvider {
|
|||||||
struct StubEmbedder;
|
struct StubEmbedder;
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait]
|
||||||
impl crate::openhuman::memory_tree::score::embed::Embedder for StubEmbedder {
|
impl crate::openhuman::memory::score::embed::Embedder for StubEmbedder {
|
||||||
fn name(&self) -> &'static str {
|
fn name(&self) -> &'static str {
|
||||||
"stub-embedder-v1"
|
"stub-embedder-v1"
|
||||||
}
|
}
|
||||||
@@ -546,7 +546,7 @@ async fn phase1_flush_open_segment_finalizes_trailing_segment() {
|
|||||||
// g) flush_open_segment also triggers tree ingest.
|
// g) flush_open_segment also triggers tree ingest.
|
||||||
|
|
||||||
use crate::openhuman::config::Config;
|
use crate::openhuman::config::Config;
|
||||||
use crate::openhuman::memory_tree::store::{count_chunks, list_chunks, ListChunksQuery};
|
use crate::openhuman::memory_store::chunks::store::{count_chunks, list_chunks, ListChunksQuery};
|
||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
|
|
||||||
/// Build a Config that points at a temp workspace, suitable for tree-ingest tests.
|
/// Build a Config that points at a temp workspace, suitable for tree-ingest tests.
|
||||||
@@ -736,7 +736,7 @@ async fn phase2_provenance_stamped_on_leaf_and_source_id_is_constant() {
|
|||||||
.iter()
|
.iter()
|
||||||
.find(|s| {
|
.find(|s| {
|
||||||
s.session_id == session
|
s.session_id == session
|
||||||
&& s.status != crate::openhuman::memory::store::segments::SegmentStatus::Open
|
&& s.status != crate::openhuman::memory_store::segments::SegmentStatus::Open
|
||||||
})
|
})
|
||||||
.expect("Expected a closed segment after flush");
|
.expect("Expected a closed segment after flush");
|
||||||
|
|
||||||
@@ -836,7 +836,9 @@ async fn phase2_ingested_content_is_raw_prose_not_recap() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The raw prose text MUST appear in at least one chunk.
|
// The raw prose text MUST appear in at least one chunk.
|
||||||
let has_user_prose = chunks.iter().any(|c| c.content.contains("lifetimes"));
|
let has_user_prose = chunks
|
||||||
|
.iter()
|
||||||
|
.any(|c| c.content.to_ascii_lowercase().contains("lifetimes"));
|
||||||
assert!(
|
assert!(
|
||||||
has_user_prose,
|
has_user_prose,
|
||||||
"Expected at least one chunk body to contain raw prose from the turn \
|
"Expected at least one chunk body to contain raw prose from the turn \
|
||||||
|
|||||||
@@ -309,7 +309,7 @@ impl PayloadSummarizer for SubagentPayloadSummarizer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Rough token estimate: ~4 characters per token. Mirrors
|
/// Rough token estimate: ~4 characters per token. Mirrors
|
||||||
/// [`crate::openhuman::memory_tree::summarizer::types::estimate_tokens`] but
|
/// [`crate::openhuman::memory_tree::tree_runtime::types::estimate_tokens`] but
|
||||||
/// returns `usize` (not `u32`) and lives here to avoid a cross-module
|
/// returns `usize` (not `u32`) and lives here to avoid a cross-module
|
||||||
/// dependency from the agent harness on the tree summarizer.
|
/// dependency from the agent harness on the tree summarizer.
|
||||||
fn estimate_tokens(text: &str) -> usize {
|
fn estimate_tokens(text: &str) -> usize {
|
||||||
|
|||||||
@@ -2150,7 +2150,7 @@ impl Agent {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Wrapper around
|
/// Wrapper around
|
||||||
/// [`crate::openhuman::memory_tree::summarizer::store::collect_root_summaries_with_caps`]
|
/// [`crate::openhuman::memory_tree::tree_runtime::store::collect_root_summaries_with_caps`]
|
||||||
/// that takes user-resolved per-namespace and total caps. The actual
|
/// that takes user-resolved per-namespace and total caps. The actual
|
||||||
/// limits are derived from the active
|
/// limits are derived from the active
|
||||||
/// [`crate::openhuman::config::schema::agent::MemoryContextWindow`]
|
/// [`crate::openhuman::config::schema::agent::MemoryContextWindow`]
|
||||||
@@ -2160,7 +2160,7 @@ fn collect_tree_root_summaries(
|
|||||||
per_namespace_cap: usize,
|
per_namespace_cap: usize,
|
||||||
total_cap: usize,
|
total_cap: usize,
|
||||||
) -> Vec<(String, String)> {
|
) -> Vec<(String, String)> {
|
||||||
crate::openhuman::memory_tree::summarizer::store::collect_root_summaries_with_caps(
|
crate::openhuman::memory_tree::tree_runtime::store::collect_root_summaries_with_caps(
|
||||||
workspace_dir,
|
workspace_dir,
|
||||||
per_namespace_cap,
|
per_namespace_cap,
|
||||||
total_cap,
|
total_cap,
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ use std::sync::Mutex as StdMutex;
|
|||||||
/// cache instead of being pushed into history raw. Token count is
|
/// cache instead of being pushed into history raw. Token count is
|
||||||
/// estimated at ~4 chars/token (mirrors
|
/// estimated at ~4 chars/token (mirrors
|
||||||
/// `crate::openhuman::agent::harness::payload_summarizer` and
|
/// `crate::openhuman::agent::harness::payload_summarizer` and
|
||||||
/// `crate::openhuman::memory_tree::summarizer::types::estimate_tokens`).
|
/// `crate::openhuman::memory_tree::tree_runtime::types::estimate_tokens`).
|
||||||
///
|
///
|
||||||
/// Set at `50_000` so the clean Gmail / Notion envelopes emitted by provider
|
/// Set at `50_000` so the clean Gmail / Notion envelopes emitted by provider
|
||||||
/// post-processing fit through unchanged for normal workloads — only
|
/// post-processing fit through unchanged for normal workloads — only
|
||||||
|
|||||||
@@ -21,7 +21,7 @@
|
|||||||
//! concatenate without branching.
|
//! concatenate without branching.
|
||||||
|
|
||||||
use crate::openhuman::config::Config;
|
use crate::openhuman::config::Config;
|
||||||
use crate::openhuman::memory_tree::retrieval::query_global;
|
use crate::openhuman::memory::retrieval::query_global;
|
||||||
|
|
||||||
/// Default lookback window for the eager digest. Mirrors the language in
|
/// Default lookback window for the eager digest. Mirrors the language in
|
||||||
/// the orchestrator prompt ("7-day digest pre-loaded into session context").
|
/// the orchestrator prompt ("7-day digest pre-loaded into session context").
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ use chrono::{DateTime, Utc};
|
|||||||
use rusqlite::{params, types::Type, Connection};
|
use rusqlite::{params, types::Type, Connection};
|
||||||
|
|
||||||
use crate::openhuman::config::Config;
|
use crate::openhuman::config::Config;
|
||||||
use crate::openhuman::memory::safety::sanitize_text;
|
use crate::openhuman::memory_store::safety::sanitize_text;
|
||||||
|
|
||||||
use super::types::{ApprovalAuditEntry, ApprovalDecision, ExecutionOutcome, PendingApproval};
|
use super::types::{ApprovalAuditEntry, ApprovalDecision, ExecutionOutcome, PendingApproval};
|
||||||
|
|
||||||
|
|||||||
@@ -587,7 +587,7 @@ pub async fn start_channels(config: Config) -> Result<()> {
|
|||||||
};
|
};
|
||||||
// Register the tree summarizer event subscriber for observability logging.
|
// Register the tree summarizer event subscriber for observability logging.
|
||||||
let _tree_summarizer_handle = bus.subscribe(Arc::new(
|
let _tree_summarizer_handle = bus.subscribe(Arc::new(
|
||||||
crate::openhuman::memory_tree::summarizer::bus::TreeSummarizerEventSubscriber::new(),
|
crate::openhuman::memory_tree::tree_runtime::bus::TreeSummarizerEventSubscriber::new(),
|
||||||
));
|
));
|
||||||
|
|
||||||
let max_in_flight_messages = compute_max_in_flight_messages(channels.len());
|
let max_in_flight_messages = compute_max_in_flight_messages(channels.len());
|
||||||
|
|||||||
@@ -576,8 +576,8 @@ async fn composio_execute_via_mock_propagates_backend_error() {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn composio_sync_gmail_via_mock_archives_raw_email_and_updates_outcome() {
|
async fn composio_sync_gmail_via_mock_archives_raw_email_and_updates_outcome() {
|
||||||
use crate::openhuman::config::TEST_ENV_LOCK;
|
use crate::openhuman::config::TEST_ENV_LOCK;
|
||||||
use crate::openhuman::memory_tree::content_store::raw::{raw_rel_path, RawKind};
|
use crate::openhuman::memory::tree_rpc::{list_chunks_rpc, ListChunksRequest};
|
||||||
use crate::openhuman::memory_tree::rpc::{list_chunks_rpc, ListChunksRequest};
|
use crate::openhuman::memory_store::content::raw::{raw_rel_path, RawKind};
|
||||||
let _cache_guard = CACHE_TEST_GUARD.lock().unwrap_or_else(|e| e.into_inner());
|
let _cache_guard = CACHE_TEST_GUARD.lock().unwrap_or_else(|e| e.into_inner());
|
||||||
let _env_guard = TEST_ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
let _env_guard = TEST_ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||||
|
|
||||||
|
|||||||
@@ -21,14 +21,14 @@ use anyhow::Result;
|
|||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
|
||||||
use crate::openhuman::config::Config;
|
use crate::openhuman::config::Config;
|
||||||
use crate::openhuman::memory_tree::canonicalize::email::{EmailMessage, EmailThread};
|
use crate::openhuman::memory::ingest_pipeline::{ingest_email, IngestResult};
|
||||||
use crate::openhuman::memory_tree::canonicalize::email_clean::{extract_email, parse_message_date};
|
use crate::openhuman::memory::util::redact::redact;
|
||||||
use crate::openhuman::memory_tree::content_store::raw::{
|
use crate::openhuman::memory_store::chunks::store::{set_chunk_raw_refs, RawRef};
|
||||||
|
use crate::openhuman::memory_store::content::raw::{
|
||||||
self as raw_store, raw_rel_path, slug_account_email, RawItem, RawKind,
|
self as raw_store, raw_rel_path, slug_account_email, RawItem, RawKind,
|
||||||
};
|
};
|
||||||
use crate::openhuman::memory_tree::ingest::{ingest_email, IngestResult};
|
use crate::openhuman::memory_sync::canonicalize::email::{EmailMessage, EmailThread};
|
||||||
use crate::openhuman::memory_tree::store::{set_chunk_raw_refs, RawRef};
|
use crate::openhuman::memory_sync::canonicalize::email_clean::{extract_email, parse_message_date};
|
||||||
use crate::openhuman::memory_tree::util::redact::redact;
|
|
||||||
|
|
||||||
/// Provider name embedded in the canonical email-thread header. Matches
|
/// Provider name embedded in the canonical email-thread header. Matches
|
||||||
/// the value `memory::tree::retrieval::source::PLATFORM_KINDS` expects.
|
/// the value `memory::tree::retrieval::source::PLATFORM_KINDS` expects.
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ use super::ProviderUserProfile;
|
|||||||
use crate::openhuman::learning::candidate::{
|
use crate::openhuman::learning::candidate::{
|
||||||
self as learning_candidate, CueFamily, EvidenceRef, FacetClass, LearningCandidate,
|
self as learning_candidate, CueFamily, EvidenceRef, FacetClass, LearningCandidate,
|
||||||
};
|
};
|
||||||
use crate::openhuman::memory::store::profile::{self, FacetType};
|
use crate::openhuman::memory_store::profile::{self, FacetType};
|
||||||
use rusqlite::params;
|
use rusqlite::params;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
@@ -32,7 +32,7 @@ use std::collections::BTreeMap;
|
|||||||
|
|
||||||
/// Shape of an identifier persisted against a connection. Mirrors the
|
/// Shape of an identifier persisted against a connection. Mirrors the
|
||||||
/// matching dimensions of the memory tree's
|
/// matching dimensions of the memory tree's
|
||||||
/// `crate::openhuman::memory_tree::score::extract::EntityKind` so the
|
/// `crate::openhuman::memory::score::extract::EntityKind` so the
|
||||||
/// self-check is a direct `(toolkit, kind, value)` lookup.
|
/// self-check is a direct `(toolkit, kind, value)` lookup.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
pub enum IdentityKind {
|
pub enum IdentityKind {
|
||||||
@@ -534,7 +534,7 @@ fn now_secs() -> f64 {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::openhuman::memory::store::profile::{profile_load_all, PROFILE_INIT_SQL};
|
use crate::openhuman::memory_store::profile::{profile_load_all, PROFILE_INIT_SQL};
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
use rusqlite::Connection;
|
use rusqlite::Connection;
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
//!
|
//!
|
||||||
//! Owns the conversion from a page of [`SlackMessage`]s (post-processed
|
//! Owns the conversion from a page of [`SlackMessage`]s (post-processed
|
||||||
//! and enriched by [`super::sync`]) into per-channel [`ChatBatch`]es and
|
//! and enriched by [`super::sync`]) into per-channel [`ChatBatch`]es and
|
||||||
//! drives [`memory_tree::ingest::ingest_chat`] per message.
|
//! drives [`memory::ingest_pipeline::ingest_chat`] per message.
|
||||||
//!
|
//!
|
||||||
//! ## Source-id scope
|
//! ## Source-id scope
|
||||||
//!
|
//!
|
||||||
@@ -30,16 +30,16 @@ use anyhow::Result;
|
|||||||
|
|
||||||
use super::types::SlackMessage;
|
use super::types::SlackMessage;
|
||||||
use crate::openhuman::config::Config;
|
use crate::openhuman::config::Config;
|
||||||
use crate::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage};
|
use crate::openhuman::memory::ingest_pipeline::ingest_chat;
|
||||||
use crate::openhuman::memory_tree::content_store::raw::{
|
use crate::openhuman::memory::util::redact::redact;
|
||||||
|
use crate::openhuman::memory_store::chunks::store::{set_chunk_raw_refs, RawRef};
|
||||||
|
use crate::openhuman::memory_store::content::raw::{
|
||||||
self as raw_store, raw_rel_path, RawItem, RawKind,
|
self as raw_store, raw_rel_path, RawItem, RawKind,
|
||||||
};
|
};
|
||||||
use crate::openhuman::memory_tree::ingest::ingest_chat;
|
use crate::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage};
|
||||||
use crate::openhuman::memory_tree::store::{set_chunk_raw_refs, RawRef};
|
|
||||||
use crate::openhuman::memory_tree::util::redact::redact;
|
|
||||||
|
|
||||||
/// Platform identifier embedded in the canonical chat transcript header.
|
/// Platform identifier embedded in the canonical chat transcript header.
|
||||||
/// Matches the value `memory_tree::retrieval::source::PLATFORM_KINDS` expects.
|
/// Matches the value `memory::retrieval::source::PLATFORM_KINDS` expects.
|
||||||
pub const SLACK_PLATFORM: &str = "slack";
|
pub const SLACK_PLATFORM: &str = "slack";
|
||||||
|
|
||||||
/// Tags attached to every Slack-ingested chunk. Stable list — retrieval
|
/// Tags attached to every Slack-ingested chunk. Stable list — retrieval
|
||||||
|
|||||||
@@ -588,7 +588,7 @@ pub async fn apply_model_settings(
|
|||||||
// so a UI embedder switch recovers prior memory under the new
|
// so a UI embedder switch recovers prior memory under the new
|
||||||
// signature. Coverage-gated + non-fatal: if the active signature did
|
// signature. Coverage-gated + non-fatal: if the active signature did
|
||||||
// not actually change, this enqueues nothing.
|
// not actually change, this enqueues nothing.
|
||||||
crate::openhuman::memory_tree::jobs::ensure_reembed_backfill(config);
|
crate::openhuman::memory::jobs::ensure_reembed_backfill(config);
|
||||||
let snapshot = snapshot_config_json(config)?;
|
let snapshot = snapshot_config_json(config)?;
|
||||||
Ok(RpcOutcome::new(
|
Ok(RpcOutcome::new(
|
||||||
snapshot,
|
snapshot,
|
||||||
@@ -638,7 +638,7 @@ pub async fn apply_memory_settings(
|
|||||||
// dark. Idempotent + non-fatal (covered space enqueues nothing; errors
|
// dark. Idempotent + non-fatal (covered space enqueues nothing; errors
|
||||||
// are logged, never fail the settings save). §7's migration is
|
// are logged, never fail the settings save). §7's migration is
|
||||||
// one-shot so it does not cover a later switch — this does.
|
// one-shot so it does not cover a later switch — this does.
|
||||||
crate::openhuman::memory_tree::jobs::ensure_reembed_backfill(config);
|
crate::openhuman::memory::jobs::ensure_reembed_backfill(config);
|
||||||
let snapshot = snapshot_config_json(config)?;
|
let snapshot = snapshot_config_json(config)?;
|
||||||
Ok(RpcOutcome::new(
|
Ok(RpcOutcome::new(
|
||||||
snapshot,
|
snapshot,
|
||||||
|
|||||||
@@ -15,8 +15,8 @@ use crate::openhuman::agent::harness::archivist::ArchivistHook;
|
|||||||
use crate::openhuman::agent::hooks::{PostTurnHook as _, TurnContext};
|
use crate::openhuman::agent::hooks::{PostTurnHook as _, TurnContext};
|
||||||
use crate::openhuman::context::summarizer::{Summarizer, SummaryStats};
|
use crate::openhuman::context::summarizer::{Summarizer, SummaryStats};
|
||||||
use crate::openhuman::inference::provider::{ChatMessage, ConversationMessage};
|
use crate::openhuman::inference::provider::{ChatMessage, ConversationMessage};
|
||||||
use crate::openhuman::memory::store::{fts5, segments as seg};
|
use crate::openhuman::memory::chat::ChatPrompt;
|
||||||
use crate::openhuman::memory_tree::chat::ChatPrompt;
|
use crate::openhuman::memory_store::{fts5, segments as seg};
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
@@ -29,9 +29,9 @@ fn setup_conn() -> Arc<Mutex<Connection>> {
|
|||||||
let conn = Connection::open_in_memory().unwrap();
|
let conn = Connection::open_in_memory().unwrap();
|
||||||
conn.execute_batch(fts5::EPISODIC_INIT_SQL).unwrap();
|
conn.execute_batch(fts5::EPISODIC_INIT_SQL).unwrap();
|
||||||
conn.execute_batch(seg::SEGMENTS_INIT_SQL).unwrap();
|
conn.execute_batch(seg::SEGMENTS_INIT_SQL).unwrap();
|
||||||
conn.execute_batch(crate::openhuman::memory::store::events::EVENTS_INIT_SQL)
|
conn.execute_batch(crate::openhuman::memory_store::events::EVENTS_INIT_SQL)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
conn.execute_batch(crate::openhuman::memory::store::profile::PROFILE_INIT_SQL)
|
conn.execute_batch(crate::openhuman::memory_store::profile::PROFILE_INIT_SQL)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
Arc::new(Mutex::new(conn))
|
Arc::new(Mutex::new(conn))
|
||||||
}
|
}
|
||||||
@@ -40,7 +40,7 @@ fn setup_conn() -> Arc<Mutex<Connection>> {
|
|||||||
struct StubChatProvider;
|
struct StubChatProvider;
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl crate::openhuman::memory_tree::chat::ChatProvider for StubChatProvider {
|
impl crate::openhuman::memory::chat::ChatProvider for StubChatProvider {
|
||||||
fn name(&self) -> &str {
|
fn name(&self) -> &str {
|
||||||
"stub:test"
|
"stub:test"
|
||||||
}
|
}
|
||||||
@@ -56,7 +56,7 @@ impl crate::openhuman::memory_tree::chat::ChatProvider for StubChatProvider {
|
|||||||
struct FailingChatProvider;
|
struct FailingChatProvider;
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl crate::openhuman::memory_tree::chat::ChatProvider for FailingChatProvider {
|
impl crate::openhuman::memory::chat::ChatProvider for FailingChatProvider {
|
||||||
fn name(&self) -> &str {
|
fn name(&self) -> &str {
|
||||||
"stub:failing"
|
"stub:failing"
|
||||||
}
|
}
|
||||||
@@ -72,7 +72,7 @@ impl crate::openhuman::memory_tree::chat::ChatProvider for FailingChatProvider {
|
|||||||
struct StubEmbedder;
|
struct StubEmbedder;
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl crate::openhuman::memory_tree::score::embed::Embedder for StubEmbedder {
|
impl crate::openhuman::memory::score::embed::Embedder for StubEmbedder {
|
||||||
fn name(&self) -> &'static str {
|
fn name(&self) -> &'static str {
|
||||||
"stub-embedder-v1"
|
"stub-embedder-v1"
|
||||||
}
|
}
|
||||||
@@ -439,13 +439,13 @@ async fn failing_provider_yields_inert_clipped_recap_used_as_compaction() {
|
|||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
// Provider present but failing → LlmSummariser inert fallback → real
|
// Provider present but failing → summarize_entries returns a non-LLM
|
||||||
// clipped content (not the bookend stub) → Some, treated as usable.
|
// fallback, which rolling_segment_recap must treat as unavailable for
|
||||||
|
// live compaction.
|
||||||
let recap = hook.rolling_segment_recap(session).await;
|
let recap = hook.rolling_segment_recap(session).await;
|
||||||
assert!(
|
assert!(
|
||||||
recap.is_some(),
|
recap.is_none(),
|
||||||
"Inert clipped-content recap (real text) is acceptable compaction \
|
"Non-LLM fallback recap text must not be used as live compaction input"
|
||||||
text — must be Some, not None"
|
|
||||||
);
|
);
|
||||||
|
|
||||||
let inner = RecordingSummarizer::new();
|
let inner = RecordingSummarizer::new();
|
||||||
@@ -464,9 +464,9 @@ async fn failing_provider_yields_inert_clipped_recap_used_as_compaction() {
|
|||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
inner.call_count(),
|
inner.call_count(),
|
||||||
0,
|
1,
|
||||||
"Inner summarizer must NOT run when an inert clipped-content recap \
|
"Inner summarizer must run when rolling recap is unavailable after \
|
||||||
is available (real content, better than no compaction)"
|
provider failure"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -817,7 +817,7 @@ fn check_memory_tree_db(config: &Config, items: &mut Vec<DiagnosticItem>) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ── Probe connection ─────────────────────────────────────────────
|
// ── Probe connection ─────────────────────────────────────────────
|
||||||
match crate::openhuman::memory_tree::store::with_connection(config, |conn| {
|
match crate::openhuman::memory_store::chunks::store::with_connection(config, |conn| {
|
||||||
let n: i64 = conn.query_row("SELECT COUNT(*) FROM mem_tree_chunks", [], |r| r.get(0))?;
|
let n: i64 = conn.query_row("SELECT COUNT(*) FROM mem_tree_chunks", [], |r| r.get(0))?;
|
||||||
Ok(n)
|
Ok(n)
|
||||||
}) {
|
}) {
|
||||||
|
|||||||
@@ -89,7 +89,7 @@ fn check_memory_tree_db_ok_when_accessible() {
|
|||||||
let cfg = test_config_in(&tmp);
|
let cfg = test_config_in(&tmp);
|
||||||
|
|
||||||
// Trigger DB creation.
|
// Trigger DB creation.
|
||||||
crate::openhuman::memory_tree::store::with_connection(&cfg, |_conn| Ok(()))
|
crate::openhuman::memory_store::chunks::store::with_connection(&cfg, |_conn| Ok(()))
|
||||||
.expect("DB init must succeed");
|
.expect("DB init must succeed");
|
||||||
|
|
||||||
let mut items = vec![];
|
let mut items = vec![];
|
||||||
|
|||||||
@@ -18,8 +18,13 @@ pub mod ollama;
|
|||||||
pub mod openai;
|
pub mod openai;
|
||||||
mod provider_trait;
|
mod provider_trait;
|
||||||
pub mod rate_limit;
|
pub mod rate_limit;
|
||||||
pub mod store;
|
|
||||||
|
|
||||||
|
// VectorStore has moved to memory_store::vectors; re-exported for callers.
|
||||||
|
pub use crate::openhuman::memory_store::vectors::store;
|
||||||
|
|
||||||
|
pub use crate::openhuman::memory_store::vectors::{
|
||||||
|
bytes_to_vec, cosine_similarity, vec_to_bytes, SearchResult, VectorStore,
|
||||||
|
};
|
||||||
pub use cloud::{
|
pub use cloud::{
|
||||||
OpenHumanCloudEmbedding, DEFAULT_CLOUD_EMBEDDING_DIMENSIONS, DEFAULT_CLOUD_EMBEDDING_MODEL,
|
OpenHumanCloudEmbedding, DEFAULT_CLOUD_EMBEDDING_DIMENSIONS, DEFAULT_CLOUD_EMBEDDING_MODEL,
|
||||||
};
|
};
|
||||||
@@ -30,7 +35,6 @@ pub use noop::NoopEmbedding;
|
|||||||
pub use ollama::{OllamaEmbedding, DEFAULT_OLLAMA_DIMENSIONS, DEFAULT_OLLAMA_MODEL};
|
pub use ollama::{OllamaEmbedding, DEFAULT_OLLAMA_DIMENSIONS, DEFAULT_OLLAMA_MODEL};
|
||||||
pub use openai::OpenAiEmbedding;
|
pub use openai::OpenAiEmbedding;
|
||||||
pub use provider_trait::{format_embedding_signature, EmbeddingProvider};
|
pub use provider_trait::{format_embedding_signature, EmbeddingProvider};
|
||||||
pub use store::{bytes_to_vec, cosine_similarity, vec_to_bytes, SearchResult, VectorStore};
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
//!
|
//!
|
||||||
//! The memory tree's embedder (`bge-m3`) is requested with
|
//! The memory tree's embedder (`bge-m3`) is requested with
|
||||||
//! `num_ctx = 8192` (see
|
//! `num_ctx = 8192` (see
|
||||||
//! [`crate::openhuman::memory_tree::score::embed::ollama::EMBED_NUM_CTX`])
|
//! [`crate::openhuman::memory::score::embed::ollama::EMBED_NUM_CTX`])
|
||||||
//! and the summariser hard-caps its output to fit that 8192-token embed
|
//! and the summariser hard-caps its output to fit that 8192-token embed
|
||||||
//! ceiling. A local model whose native context window is below this floor
|
//! ceiling. A local model whose native context window is below this floor
|
||||||
//! silently truncates chunks/summaries and corrupts recall, so we refuse
|
//! silently truncates chunks/summaries and corrupts recall, so we refuse
|
||||||
@@ -21,7 +21,7 @@ use serde::Serialize;
|
|||||||
/// time. Changing the embedder's context request automatically moves the
|
/// time. Changing the embedder's context request automatically moves the
|
||||||
/// acceptance floor with it.
|
/// acceptance floor with it.
|
||||||
pub const MIN_CONTEXT_TOKENS: u64 =
|
pub const MIN_CONTEXT_TOKENS: u64 =
|
||||||
crate::openhuman::memory_tree::score::embed::ollama::EMBED_NUM_CTX as u64;
|
crate::openhuman::memory::score::embed::ollama::EMBED_NUM_CTX as u64;
|
||||||
|
|
||||||
/// Verdict for a single model's context window against
|
/// Verdict for a single model's context window against
|
||||||
/// [`MIN_CONTEXT_TOKENS`]. Serialized into the diagnostics payload so the
|
/// [`MIN_CONTEXT_TOKENS`]. Serialized into the diagnostics payload so the
|
||||||
@@ -79,7 +79,7 @@ mod tests {
|
|||||||
// requests; this guards against the two drifting apart.
|
// requests; this guards against the two drifting apart.
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
MIN_CONTEXT_TOKENS,
|
MIN_CONTEXT_TOKENS,
|
||||||
crate::openhuman::memory_tree::score::embed::ollama::EMBED_NUM_CTX as u64
|
crate::openhuman::memory::score::embed::ollama::EMBED_NUM_CTX as u64
|
||||||
);
|
);
|
||||||
assert_eq!(MIN_CONTEXT_TOKENS, 8_192);
|
assert_eq!(MIN_CONTEXT_TOKENS, 8_192);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ use rusqlite::Connection;
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use crate::openhuman::learning::candidate::FacetClass;
|
use crate::openhuman::learning::candidate::FacetClass;
|
||||||
use crate::openhuman::memory::store::profile::{self, ProfileFacet, UserState};
|
use crate::openhuman::memory_store::profile::{self, ProfileFacet, UserState};
|
||||||
|
|
||||||
/// Thin wrapper around the `user_profile` table.
|
/// Thin wrapper around the `user_profile` table.
|
||||||
///
|
///
|
||||||
@@ -115,7 +115,7 @@ pub fn class_prefix(class: FacetClass) -> &'static str {
|
|||||||
|
|
||||||
// ── Facet state enum re-export (convenience for callers of this module) ───────
|
// ── Facet state enum re-export (convenience for callers of this module) ───────
|
||||||
|
|
||||||
pub use crate::openhuman::memory::store::profile::{
|
pub use crate::openhuman::memory_store::profile::{
|
||||||
FacetState as CacheFacetState, UserState as CacheUserState,
|
FacetState as CacheFacetState, UserState as CacheUserState,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ use std::sync::Arc;
|
|||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::openhuman::learning::candidate::{EvidenceRef, FacetClass};
|
use crate::openhuman::learning::candidate::{EvidenceRef, FacetClass};
|
||||||
use crate::openhuman::memory::store::profile::{
|
use crate::openhuman::memory_store::profile::{
|
||||||
FacetState, FacetType, ProfileFacet, UserState, PROFILE_INIT_SQL,
|
FacetState, FacetType, ProfileFacet, UserState, PROFILE_INIT_SQL,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -680,15 +680,15 @@ pub async fn scrape_linkedin_profile(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Build a local memory client for profile persistence.
|
/// Build a local memory client for profile persistence.
|
||||||
fn build_memory_client() -> anyhow::Result<crate::openhuman::memory::store::MemoryClient> {
|
fn build_memory_client() -> anyhow::Result<crate::openhuman::memory_store::MemoryClient> {
|
||||||
crate::openhuman::memory::store::MemoryClient::new_local()
|
crate::openhuman::memory_store::MemoryClient::new_local()
|
||||||
.map_err(|e| anyhow::anyhow!("memory client unavailable: {e}"))
|
.map_err(|e| anyhow::anyhow!("memory client unavailable: {e}"))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Persist the full scraped LinkedIn profile to the user-profile memory
|
/// Persist the full scraped LinkedIn profile to the user-profile memory
|
||||||
/// namespace so the agent has rich context about the user.
|
/// namespace so the agent has rich context about the user.
|
||||||
async fn persist_linkedin_profile(
|
async fn persist_linkedin_profile(
|
||||||
memory: &crate::openhuman::memory::store::MemoryClient,
|
memory: &crate::openhuman::memory_store::MemoryClient,
|
||||||
url: &str,
|
url: &str,
|
||||||
data: &serde_json::Value,
|
data: &serde_json::Value,
|
||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
@@ -720,7 +720,7 @@ async fn persist_linkedin_profile(
|
|||||||
|
|
||||||
/// Fallback: persist just the LinkedIn URL when the full scrape fails.
|
/// Fallback: persist just the LinkedIn URL when the full scrape fails.
|
||||||
async fn persist_linkedin_url_only(
|
async fn persist_linkedin_url_only(
|
||||||
memory: &crate::openhuman::memory::store::MemoryClient,
|
memory: &crate::openhuman::memory_store::MemoryClient,
|
||||||
url: &str,
|
url: &str,
|
||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
memory
|
memory
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ use async_trait::async_trait;
|
|||||||
use crate::core::event_bus::{subscribe_global, DomainEvent, EventHandler, SubscriptionHandle};
|
use crate::core::event_bus::{subscribe_global, DomainEvent, EventHandler, SubscriptionHandle};
|
||||||
use crate::openhuman::composio::providers::profile_md::replace_managed_block;
|
use crate::openhuman::composio::providers::profile_md::replace_managed_block;
|
||||||
use crate::openhuman::learning::cache::FacetCache;
|
use crate::openhuman::learning::cache::FacetCache;
|
||||||
use crate::openhuman::memory::store::profile::UserState;
|
use crate::openhuman::memory_store::profile::UserState;
|
||||||
|
|
||||||
// ── Class → block metadata ────────────────────────────────────────────────────
|
// ── Class → block metadata ────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -215,7 +215,7 @@ impl EventHandler for RendererSubscriber {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::openhuman::composio::providers::profile_md::{block_end, block_start};
|
use crate::openhuman::composio::providers::profile_md::{block_end, block_start};
|
||||||
use crate::openhuman::memory::store::profile::{
|
use crate::openhuman::memory_store::profile::{
|
||||||
FacetState, FacetType, ProfileFacet, UserState, PROFILE_INIT_SQL,
|
FacetState, FacetType, ProfileFacet, UserState, PROFILE_INIT_SQL,
|
||||||
};
|
};
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
|
|||||||
@@ -163,7 +163,7 @@ pub fn load_learned_from_cache(
|
|||||||
|
|
||||||
// Group by class prefix (portion before the first '/'), then sort within
|
// Group by class prefix (portion before the first '/'), then sort within
|
||||||
// each class by stability descending, then by key alphabetically.
|
// each class by stability descending, then by key alphabetically.
|
||||||
use crate::openhuman::memory::store::profile::ProfileFacet;
|
use crate::openhuman::memory_store::profile::ProfileFacet;
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
let mut by_class: BTreeMap<String, Vec<usize>> = BTreeMap::new();
|
let mut by_class: BTreeMap<String, Vec<usize>> = BTreeMap::new();
|
||||||
|
|
||||||
@@ -197,7 +197,7 @@ pub fn load_learned_from_cache(
|
|||||||
// agent can parse the source. Goal class keeps value-only (full
|
// agent can parse the source. Goal class keeps value-only (full
|
||||||
// sentence, no key prefix). Pinned entries get a trailing suffix.
|
// sentence, no key prefix). Pinned entries get a trailing suffix.
|
||||||
let pinned =
|
let pinned =
|
||||||
if f.user_state == crate::openhuman::memory::store::profile::UserState::Pinned {
|
if f.user_state == crate::openhuman::memory_store::profile::UserState::Pinned {
|
||||||
" *(pinned)*"
|
" *(pinned)*"
|
||||||
} else {
|
} else {
|
||||||
""
|
""
|
||||||
@@ -376,7 +376,7 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn load_learned_from_cache_formats_active_facets() {
|
fn load_learned_from_cache_formats_active_facets() {
|
||||||
use crate::openhuman::learning::cache::FacetCache;
|
use crate::openhuman::learning::cache::FacetCache;
|
||||||
use crate::openhuman::memory::store::profile::{
|
use crate::openhuman::memory_store::profile::{
|
||||||
FacetState, FacetType, ProfileFacet, UserState, PROFILE_INIT_SQL,
|
FacetState, FacetType, ProfileFacet, UserState, PROFILE_INIT_SQL,
|
||||||
};
|
};
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
@@ -456,7 +456,7 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn load_learned_from_cache_empty_when_no_active_facets() {
|
fn load_learned_from_cache_empty_when_no_active_facets() {
|
||||||
use crate::openhuman::learning::cache::FacetCache;
|
use crate::openhuman::learning::cache::FacetCache;
|
||||||
use crate::openhuman::memory::store::profile::PROFILE_INIT_SQL;
|
use crate::openhuman::memory_store::profile::PROFILE_INIT_SQL;
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
use rusqlite::Connection;
|
use rusqlite::Connection;
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ use std::sync::Arc;
|
|||||||
|
|
||||||
use super::load_learned_from_cache;
|
use super::load_learned_from_cache;
|
||||||
use crate::openhuman::learning::cache::FacetCache;
|
use crate::openhuman::learning::cache::FacetCache;
|
||||||
use crate::openhuman::memory::store::profile::{
|
use crate::openhuman::memory_store::profile::{
|
||||||
FacetState, FacetType, ProfileFacet, UserState, PROFILE_INIT_SQL,
|
FacetState, FacetType, ProfileFacet, UserState, PROFILE_INIT_SQL,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -644,7 +644,7 @@ fn handle_rebuild_cache(_params: Map<String, Value>) -> ControllerFuture {
|
|||||||
fn handle_cache_stats(_params: Map<String, Value>) -> ControllerFuture {
|
fn handle_cache_stats(_params: Map<String, Value>) -> ControllerFuture {
|
||||||
Box::pin(async move {
|
Box::pin(async move {
|
||||||
use crate::openhuman::learning::cache::FacetCache;
|
use crate::openhuman::learning::cache::FacetCache;
|
||||||
use crate::openhuman::memory::store::profile::FacetState;
|
use crate::openhuman::memory_store::profile::FacetState;
|
||||||
|
|
||||||
tracing::debug!("[learning.cache_stats] cache stats requested via RPC");
|
tracing::debug!("[learning.cache_stats] cache stats requested via RPC");
|
||||||
|
|
||||||
@@ -723,7 +723,7 @@ fn full_key(class_str: &str, key_suffix: &str) -> String {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Serialize a [`ProfileFacet`] to a serde_json [`Value`] for RPC output.
|
/// Serialize a [`ProfileFacet`] to a serde_json [`Value`] for RPC output.
|
||||||
fn facet_to_json(f: &crate::openhuman::memory::store::profile::ProfileFacet) -> serde_json::Value {
|
fn facet_to_json(f: &crate::openhuman::memory_store::profile::ProfileFacet) -> serde_json::Value {
|
||||||
serde_json::json!({
|
serde_json::json!({
|
||||||
"key": f.key,
|
"key": f.key,
|
||||||
"value": f.value,
|
"value": f.value,
|
||||||
@@ -742,7 +742,7 @@ fn facet_to_json(f: &crate::openhuman::memory::store::profile::ProfileFacet) ->
|
|||||||
|
|
||||||
fn handle_list_facets(params: Map<String, Value>) -> ControllerFuture {
|
fn handle_list_facets(params: Map<String, Value>) -> ControllerFuture {
|
||||||
Box::pin(async move {
|
Box::pin(async move {
|
||||||
use crate::openhuman::memory::store::profile::FacetState;
|
use crate::openhuman::memory_store::profile::FacetState;
|
||||||
|
|
||||||
tracing::debug!("[learning.list_facets] called");
|
tracing::debug!("[learning.list_facets] called");
|
||||||
|
|
||||||
@@ -822,7 +822,7 @@ fn handle_get_facet(params: Map<String, Value>) -> ControllerFuture {
|
|||||||
|
|
||||||
fn handle_update_facet(params: Map<String, Value>) -> ControllerFuture {
|
fn handle_update_facet(params: Map<String, Value>) -> ControllerFuture {
|
||||||
Box::pin(async move {
|
Box::pin(async move {
|
||||||
use crate::openhuman::memory::store::profile::UserState;
|
use crate::openhuman::memory_store::profile::UserState;
|
||||||
|
|
||||||
let class_str = params
|
let class_str = params
|
||||||
.get("class")
|
.get("class")
|
||||||
@@ -875,7 +875,7 @@ fn handle_update_facet(params: Map<String, Value>) -> ControllerFuture {
|
|||||||
|
|
||||||
fn handle_pin_facet(params: Map<String, Value>) -> ControllerFuture {
|
fn handle_pin_facet(params: Map<String, Value>) -> ControllerFuture {
|
||||||
Box::pin(async move {
|
Box::pin(async move {
|
||||||
use crate::openhuman::memory::store::profile::UserState;
|
use crate::openhuman::memory_store::profile::UserState;
|
||||||
|
|
||||||
let class_str = params
|
let class_str = params
|
||||||
.get("class")
|
.get("class")
|
||||||
@@ -915,7 +915,7 @@ fn handle_pin_facet(params: Map<String, Value>) -> ControllerFuture {
|
|||||||
|
|
||||||
fn handle_unpin_facet(params: Map<String, Value>) -> ControllerFuture {
|
fn handle_unpin_facet(params: Map<String, Value>) -> ControllerFuture {
|
||||||
Box::pin(async move {
|
Box::pin(async move {
|
||||||
use crate::openhuman::memory::store::profile::UserState;
|
use crate::openhuman::memory_store::profile::UserState;
|
||||||
|
|
||||||
let class_str = params
|
let class_str = params
|
||||||
.get("class")
|
.get("class")
|
||||||
@@ -955,7 +955,7 @@ fn handle_unpin_facet(params: Map<String, Value>) -> ControllerFuture {
|
|||||||
|
|
||||||
fn handle_forget_facet(params: Map<String, Value>) -> ControllerFuture {
|
fn handle_forget_facet(params: Map<String, Value>) -> ControllerFuture {
|
||||||
Box::pin(async move {
|
Box::pin(async move {
|
||||||
use crate::openhuman::memory::store::profile::{FacetState, UserState};
|
use crate::openhuman::memory_store::profile::{FacetState, UserState};
|
||||||
|
|
||||||
let class_str = params
|
let class_str = params
|
||||||
.get("class")
|
.get("class")
|
||||||
@@ -1003,7 +1003,7 @@ fn handle_forget_facet(params: Map<String, Value>) -> ControllerFuture {
|
|||||||
|
|
||||||
fn handle_reset_cache(_params: Map<String, Value>) -> ControllerFuture {
|
fn handle_reset_cache(_params: Map<String, Value>) -> ControllerFuture {
|
||||||
Box::pin(async move {
|
Box::pin(async move {
|
||||||
use crate::openhuman::memory::store::profile::UserState;
|
use crate::openhuman::memory_store::profile::UserState;
|
||||||
|
|
||||||
tracing::debug!("[learning.reset_cache] called");
|
tracing::debug!("[learning.reset_cache] called");
|
||||||
|
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ use crate::core::event_bus;
|
|||||||
use crate::core::event_bus::DomainEvent;
|
use crate::core::event_bus::DomainEvent;
|
||||||
use crate::openhuman::learning::cache::FacetCache;
|
use crate::openhuman::learning::cache::FacetCache;
|
||||||
use crate::openhuman::learning::candidate::{self, CueFamily, FacetClass, LearningCandidate};
|
use crate::openhuman::learning::candidate::{self, CueFamily, FacetClass, LearningCandidate};
|
||||||
use crate::openhuman::memory::store::profile::{FacetState, FacetType, ProfileFacet, UserState};
|
use crate::openhuman::memory_store::profile::{FacetState, FacetType, ProfileFacet, UserState};
|
||||||
|
|
||||||
// ── Thresholds ────────────────────────────────────────────────────────────────
|
// ── Thresholds ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -554,7 +554,7 @@ mod tests {
|
|||||||
use crate::openhuman::learning::candidate::{
|
use crate::openhuman::learning::candidate::{
|
||||||
Buffer, EvidenceRef, FacetClass, LearningCandidate,
|
Buffer, EvidenceRef, FacetClass, LearningCandidate,
|
||||||
};
|
};
|
||||||
use crate::openhuman::memory::store::profile::PROFILE_INIT_SQL;
|
use crate::openhuman::memory_store::profile::PROFILE_INIT_SQL;
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
use rusqlite::Connection;
|
use rusqlite::Connection;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
@@ -788,7 +788,7 @@ mod tests {
|
|||||||
let now = 1_000_000.0;
|
let now = 1_000_000.0;
|
||||||
|
|
||||||
// Manually insert a Pinned row.
|
// Manually insert a Pinned row.
|
||||||
use crate::openhuman::memory::store::profile::{FacetState, FacetType, UserState};
|
use crate::openhuman::memory_store::profile::{FacetState, FacetType, UserState};
|
||||||
let pinned = ProfileFacet {
|
let pinned = ProfileFacet {
|
||||||
facet_id: "f-pinned".into(),
|
facet_id: "f-pinned".into(),
|
||||||
facet_type: FacetType::Preference,
|
facet_type: FacetType::Preference,
|
||||||
|
|||||||
@@ -1,82 +1,75 @@
|
|||||||
# Memory
|
# memory
|
||||||
|
|
||||||
Persistent knowledge layer. Owns the unified store (SQLite + FTS5 + vector embeddings + graph relations), document ingestion pipelines, namespace + KV operations, conversation history, and retrieval scoring. Does NOT own raw provider embedding APIs (`local_ai/`), agent prompt assembly (`agent/memory_loader.rs`), or per-channel ingestion adapters beyond the bundled Slack importer.
|
Orchestration layer over the memory stack. Owns:
|
||||||
|
|
||||||
## Architecture
|
- **Ingest pipeline** — orchestrates source → canonicalise → chunk →
|
||||||
|
score → persist → enqueue extract jobs.
|
||||||
|
- **Job handlers** — background workers that drain the queue (admit,
|
||||||
|
extract, seal, digest, topic-curate).
|
||||||
|
- **Scoring** — fast scorers, signal aggregation, score persistence.
|
||||||
|
- **Tree policy** — `tree_global` and `tree_topic` building rules.
|
||||||
|
- **RPC surface** — `read_rpc`, `tree_rpc`, controller schemas for the
|
||||||
|
memory_* RPC namespace.
|
||||||
|
- **Recall** — `stm_recall`, `retrieval` ranking, query orchestration.
|
||||||
|
|
||||||
The module is organised in concentric layers — the contract on the
|
Does **not** own any storage primitives — those live in
|
||||||
inside, the persistent backend around it, the ingestion + retrieval
|
[`memory_store`](../memory_store/). See that module for raw md, chunks,
|
||||||
pipelines on top, and the per-domain glue at the edge:
|
entities, trees, vectors, kv, and contacts.
|
||||||
|
|
||||||
```text
|
## Sibling memory_* modules
|
||||||
┌──────────────────────────────────────┐
|
|
||||||
│ conversations/ slack_ingestion/ │ per-domain plumbing
|
|
||||||
├──────────────────────────────────────┤
|
|
||||||
│ tree/ (bucket-seal LLD pipeline) │ new retrieval architecture
|
|
||||||
├──────────────────────────────────────┤
|
|
||||||
│ ingestion/ (extract chunks) │ document ingestion
|
|
||||||
├──────────────────────────────────────┤
|
|
||||||
│ store/ (UnifiedMemory backend) │ SQLite + FTS5 + vectors
|
|
||||||
├──────────────────────────────────────┤
|
|
||||||
│ traits.rs (Memory trait) │ contract
|
|
||||||
└──────────────────────────────────────┘
|
|
||||||
```
|
|
||||||
|
|
||||||
- **`traits.rs`** — `Memory`, `MemoryEntry`, `MemoryCategory`,
|
The memory stack is split across several top-level modules so each has
|
||||||
`RecallOpts`. The backend-agnostic contract every store implements.
|
one job. memory orchestrates and routes between them.
|
||||||
- **`store/`** — `UnifiedMemory` is the production backend (SQLite
|
|
||||||
with FTS5 for keyword search, vector tables for embeddings, and
|
|
||||||
graph tables for entity/relation triples) plus the `MemoryClient`
|
|
||||||
handle used by the rest of the process.
|
|
||||||
- **`ingestion/`** — chunking + extraction pipeline (entities,
|
|
||||||
relations, embeddings) and the background `IngestionQueue` worker.
|
|
||||||
- **`tree/`** — the new bucket-seal retrieval architecture from
|
|
||||||
`docs/MEMORY_ARCHITECTURE_LLD.md`: `canonicalize` (normalise
|
|
||||||
inputs), `chunker` and `content_store` (durable chunks),
|
|
||||||
`score`/`retrieval` (ranking surface),
|
|
||||||
`tree_source`/`tree_topic`/`tree_global` (the three concentric
|
|
||||||
trees the LLD calls for), and `jobs` (background seals/summaries).
|
|
||||||
- **`conversations/`** — workspace-backed JSONL chat thread/message
|
|
||||||
history. See `conversations/README.md`.
|
|
||||||
- **`slack_ingestion/`** — Slack provider plumbing (bucketer +
|
|
||||||
ingest wrapper + RPC). See `slack_ingestion/README.md`.
|
|
||||||
|
|
||||||
The legacy memory store (`store/` + `ingestion/`) and the new
|
| Module | Role |
|
||||||
`tree/` pipeline coexist for now — `tree/` is replacing the older
|
| --- | --- |
|
||||||
retrieval surface incrementally and both must remain wired into RPC
|
| [`memory_store`](../memory_store/) | Storage primitives: raw / chunks / entities / trees / vectors / kv / contacts. SQLite + on-disk md. |
|
||||||
until the migration completes.
|
| [`memory_tree`](../memory_tree/) | Generic tree mechanics: bucket-seal, flush, summarise, walk. Kind-agnostic. |
|
||||||
|
| [`memory_archivist`](../memory_archivist/) | Chat conversation → clip tool-calls → push to tree. |
|
||||||
|
| [`memory_entities`](../memory_entities/) | Md-backed entity registry (people + orgs + topics + …). Replacing `people/`. |
|
||||||
|
| [`memory_graph`](../memory_graph/) | Derived co-occurrence edges over the entity index. |
|
||||||
|
| [`memory_tools`](../memory_tools/) | Tool-scoped rules + agent read/write tools. |
|
||||||
|
| [`memory_sync`](../memory_sync/) | Composio + workspace + MCP sync pipelines. |
|
||||||
|
|
||||||
## Public surface
|
## What lives here
|
||||||
|
|
||||||
- `pub trait Memory` / `pub struct MemoryEntry` / `pub enum MemoryCategory` / `pub struct RecallOpts` — `traits.rs:11-100` — backend contract for any memory store.
|
| Path | Role |
|
||||||
- `pub struct UnifiedMemory` — `store/unified/` (re-exported `store/mod.rs:40`) — primary SQLite + FTS5 + vector implementation.
|
| --- | --- |
|
||||||
- `pub struct MemoryClient` / `pub struct MemoryClientRef` / `pub enum MemoryState` — `store/client.rs` — async client handle used by RPC handlers.
|
| [`mod.rs`](mod.rs) | Module root + re-exports. |
|
||||||
- `pub fn create_memory` / `pub fn create_memory_with_storage` / `pub fn create_memory_with_storage_and_routes` / `pub fn create_memory_for_migration` — `store/factories.rs` — bootstrap a memory instance.
|
| [`ingest_pipeline.rs`](ingest_pipeline.rs) | Source-agnostic ingest orchestration. Called by sync pipelines and tree_rpc. |
|
||||||
- `pub struct MemoryIngestionRequest` / `pub struct MemoryIngestionResult` / `pub struct MemoryIngestionConfig` / `pub enum ExtractionMode` / `pub struct ExtractedEntity` / `pub struct ExtractedRelation` / `const DEFAULT_MEMORY_EXTRACTION_MODEL` — `ingestion.rs` (re-exported `mod.rs:22`).
|
| [`jobs/`](../memory_queue/) | Background workers: extract, admit, seal, digest, topic curate. Re-exported here from `memory_queue`. |
|
||||||
- `pub struct IngestionQueue` / `pub struct IngestionJob` — `ingestion_queue.rs` — async background ingestion worker.
|
| [`score/`](score/) | Fast scorer, signals, embeddings, entity extraction, entity-index persistence. `store.rs` will eventually split — entity-index pieces move to `memory_store::entities`. |
|
||||||
- `pub struct NamespaceDocumentInput` / `pub struct NamespaceMemoryHit` / `pub struct NamespaceQueryResult` / `pub struct NamespaceRetrievalContext` / `pub struct RetrievalScoreBreakdown` / `pub enum MemoryItemKind` — `store/types.rs`.
|
| [`retrieval/`](retrieval/) | Drill-down, fetch, query_source, query_global, query_topic, search; scoring + ranking on top of memory_store. |
|
||||||
- RPC `memory.{init, list_documents, list_namespaces, delete_document, query_namespace, recall_context, recall_memories, list_files, read_file, write_file, namespace_list, doc_put, doc_ingest, doc_list, doc_delete, context_query, context_recall, kv_set, kv_get, kv_delete, kv_list_namespace, graph_upsert, graph_query, clear_namespace}` — `schemas.rs:29-55`.
|
| [`tree_global/`](../memory_tree/global/) | Global digest tree building policy: seal, digest, recap. Implemented in `memory_tree/global` and routed from here. |
|
||||||
- RPC tree `memory.tree.*` and retrieval — `tree/` (re-exported via `all_memory_tree_*` / `all_retrieval_*`).
|
| [`tree_topic/`](../memory_tree/topic/) | Topic tree building policy: hotness gating, routing, curator, backfill. Implemented in `memory_tree/topic` and routed from here. |
|
||||||
- RPC slack ingestion — `slack_ingestion/` (re-exported via `all_slack_ingestion_*`).
|
| [`summarizer/`](../memory_tree/summarise.rs) | LLM summarisation pipeline for sealed buckets. Implemented in `memory_tree/summarise.rs`. |
|
||||||
|
| [`stm_recall/`](stm_recall/) | Short-term recall: cross-session FTS5 lookup + ranking. |
|
||||||
|
| [`ingestion/`](ingestion/) | Document ingestion queue + extraction (entities, relations, embeddings) — feeds UnifiedMemory documents. |
|
||||||
|
| [`canonicalize/`](../memory_sync/canonicalize/) | Source → canonical markdown (chat / email / document). Implemented in `memory_sync/canonicalize` and used at ingest time. |
|
||||||
|
| [`chat/`](chat.rs) | Chat-source canonicalisation helpers. |
|
||||||
|
| [`conversations/`](../memory_conversations/) | Workspace-backed JSONL chat thread/message history. Re-exported here from `memory_conversations`. |
|
||||||
|
| [`read_rpc.rs`](read_rpc.rs) | RPC handlers for memory reads. |
|
||||||
|
| [`tree_rpc.rs`](tree_rpc.rs) | RPC handlers for tree ingest + introspection. |
|
||||||
|
| [`schemas/`](schemas/) + [`schema.rs`](schema.rs) | Controller schema definitions for the memory + memory_tree RPC namespaces. |
|
||||||
|
| [`sync_status/`](../memory_sync/sync_status/) | Sync freshness tracking + RPC. Re-exported here from `memory_sync::sync_status`. |
|
||||||
|
| [`ops/`](ops/) | RPC operation handlers + the shared `active_memory_client` helper. |
|
||||||
|
| [`preferences.rs`](preferences.rs) | User preference read/write helpers. |
|
||||||
|
| [`rpc_models.rs`](rpc_models.rs) | Shared RPC request/response shapes. |
|
||||||
|
| [`traits.rs`](traits.rs) | `Memory`, `MemoryEntry`, `MemoryCategory`, `NamespaceSummary`, `RecallOpts`. The backend-agnostic contract every store implements. |
|
||||||
|
| [`util/`](util/) | Small helpers (redact for log PII). |
|
||||||
|
| [`global.rs`](global.rs) | Global-namespace helpers. |
|
||||||
|
|
||||||
## Calls into
|
## Layer rules
|
||||||
|
|
||||||
- `src/openhuman/local_ai/` — embedding model, sentiment scoring, extraction LLM.
|
- **No storage in this module.** All persistence goes through
|
||||||
- `src/openhuman/embeddings/` — vector backend selection.
|
`memory_store::*`. If you're tempted to open a SQLite connection
|
||||||
- `src/openhuman/config/` — memory backend choice + filesystem paths.
|
here, the connection helper belongs one layer down.
|
||||||
- `src/openhuman/encryption/` — at-rest secrets for KV namespaces.
|
- **No upward dependencies.** memory may import from memory_store /
|
||||||
- `src/core/event_bus/` — emits `DomainEvent::Memory(*)` on ingestion / mutation.
|
memory_tree / memory_entities / memory_archivist / memory_graph /
|
||||||
|
memory_tools, but the inverse is a layer violation. (The two
|
||||||
## Called by
|
documented exceptions today — `memory_store::retrieval::tree_walk`
|
||||||
|
calling `memory::retrieval::drill_down`, and `memory_store::trees::registry`
|
||||||
- `src/openhuman/agent/` (`memory_loader.rs`, `harness/memory_context.rs`, `harness/archivist*.rs`, `harness/fork_context.rs`) — context injection and episodic indexing.
|
pulling `GLOBAL_SCOPE` from `memory::tree_global` — are tracked in
|
||||||
- `src/openhuman/learning/{reflection,tool_tracker,user_profile,prompt_sections}.rs` — long-term insight storage.
|
their respective READMEs.)
|
||||||
- `src/openhuman/screen_intelligence/{helpers,tests}.rs` — recall surfaces for visual context.
|
- **Surface high-level tool calls** that route to the right submodule;
|
||||||
- `src/openhuman/autocomplete/history.rs` — query-history recall.
|
don't expose internals at the call site.
|
||||||
- `src/openhuman/tools/ops.rs` and `tools/impl/system/tool_stats.rs` — memory-backed tool stats.
|
|
||||||
- `src/core/all.rs` — registers `all_memory_*` controllers.
|
|
||||||
|
|
||||||
## Tests
|
|
||||||
|
|
||||||
- Unit: `ops_tests.rs`, `schemas_tests.rs`, `rpc_models_tests.rs`, `ingestion_tests.rs`, plus `*_tests.rs` files inside `store/`, `tree/`, `conversations/`, `slack_ingestion/`.
|
|
||||||
- Integration: `tests/autocomplete_memory_e2e.rs`, `tests/memory_graph_sync_e2e.rs`.
|
|
||||||
|
|||||||
@@ -0,0 +1,257 @@
|
|||||||
|
//! Memory LLM adapter backed by the unified inference provider stack.
|
||||||
|
//!
|
||||||
|
//! Memory callers still want a tiny prompt surface: one system message, one
|
||||||
|
//! user message, and a string response. This module keeps that narrow contract
|
||||||
|
//! for the rest of the memory layer, but routes every production call through
|
||||||
|
//! `openhuman::inference::provider` so memory uses the same workload routing as
|
||||||
|
//! the rest of the app.
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use anyhow::Result;
|
||||||
|
use async_trait::async_trait;
|
||||||
|
|
||||||
|
use crate::openhuman::config::{Config, DEFAULT_CLOUD_LLM_MODEL};
|
||||||
|
use crate::openhuman::inference::provider::{
|
||||||
|
create_chat_provider, provider_for_role, ChatMessage, Provider,
|
||||||
|
};
|
||||||
|
|
||||||
|
/// One pair of prompt messages handed to the memory LLM backend.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ChatPrompt {
|
||||||
|
pub system: String,
|
||||||
|
pub user: String,
|
||||||
|
pub temperature: f64,
|
||||||
|
pub kind: &'static str,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pluggable LLM surface used by the memory layer.
|
||||||
|
#[async_trait]
|
||||||
|
pub trait ChatProvider: Send + Sync {
|
||||||
|
fn name(&self) -> &str;
|
||||||
|
|
||||||
|
async fn chat_for_json(&self, prompt: &ChatPrompt) -> Result<String>;
|
||||||
|
|
||||||
|
async fn chat_for_text(&self, prompt: &ChatPrompt) -> Result<String> {
|
||||||
|
self.chat_for_json(prompt).await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct InferenceChatProvider {
|
||||||
|
inner: Box<dyn Provider>,
|
||||||
|
model: String,
|
||||||
|
display: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl InferenceChatProvider {
|
||||||
|
fn new(inner: Box<dyn Provider>, model: String) -> Self {
|
||||||
|
let display = format!("inference:{model}");
|
||||||
|
Self {
|
||||||
|
inner,
|
||||||
|
model,
|
||||||
|
display,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run(&self, prompt: &ChatPrompt) -> Result<String> {
|
||||||
|
log::debug!(
|
||||||
|
"[memory::chat] provider={} kind={} model={} sys_chars={} user_chars={}",
|
||||||
|
self.display,
|
||||||
|
prompt.kind,
|
||||||
|
self.model,
|
||||||
|
prompt.system.len(),
|
||||||
|
prompt.user.len()
|
||||||
|
);
|
||||||
|
|
||||||
|
let messages = vec![
|
||||||
|
ChatMessage::system(prompt.system.clone()),
|
||||||
|
ChatMessage::user(prompt.user.clone()),
|
||||||
|
];
|
||||||
|
|
||||||
|
let text = self
|
||||||
|
.inner
|
||||||
|
.chat_with_history(&messages, &self.model, prompt.temperature)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
log::debug!(
|
||||||
|
"[memory::chat] provider={} kind={} response_chars={}",
|
||||||
|
self.display,
|
||||||
|
prompt.kind,
|
||||||
|
text.len()
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl ChatProvider for InferenceChatProvider {
|
||||||
|
fn name(&self) -> &str {
|
||||||
|
&self.display
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn chat_for_json(&self, prompt: &ChatPrompt) -> Result<String> {
|
||||||
|
self.run(prompt).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn chat_for_text(&self, prompt: &ChatPrompt) -> Result<String> {
|
||||||
|
self.run(prompt).await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn routed_memory_config(config: &Config) -> Config {
|
||||||
|
let mut routed = config.clone();
|
||||||
|
if !config.workload_uses_local("memory") {
|
||||||
|
routed.default_model = Some(
|
||||||
|
config
|
||||||
|
.memory_tree
|
||||||
|
.cloud_llm_model
|
||||||
|
.clone()
|
||||||
|
.unwrap_or_else(|| DEFAULT_CLOUD_LLM_MODEL.to_string()),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
routed
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
fn test_override_runtime() -> Option<(Arc<dyn ChatProvider>, String)> {
|
||||||
|
test_override::current().map(|provider| (provider, "test:override".to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(test))]
|
||||||
|
fn test_override_runtime() -> Option<(Arc<dyn ChatProvider>, String)> {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build the memory LLM provider and return the resolved model id.
|
||||||
|
pub fn build_chat_runtime(config: &Config) -> Result<(Arc<dyn ChatProvider>, String)> {
|
||||||
|
if let Some(runtime) = test_override_runtime() {
|
||||||
|
return Ok(runtime);
|
||||||
|
}
|
||||||
|
|
||||||
|
let routed = routed_memory_config(config);
|
||||||
|
let resolved_provider = provider_for_role("summarization", &routed);
|
||||||
|
let (provider, model) = create_chat_provider("summarization", &routed)?;
|
||||||
|
|
||||||
|
log::debug!(
|
||||||
|
"[memory::chat] built provider route={} model={}",
|
||||||
|
resolved_provider,
|
||||||
|
model
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok((
|
||||||
|
Arc::new(InferenceChatProvider::new(provider, model.clone())),
|
||||||
|
model,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build the memory LLM provider dictated by the inference workload routing.
|
||||||
|
pub fn build_chat_provider(config: &Config) -> Result<Arc<dyn ChatProvider>> {
|
||||||
|
Ok(build_chat_runtime(config)?.0)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub struct StaticChatProvider {
|
||||||
|
pub response: String,
|
||||||
|
pub calls: std::sync::atomic::AtomicUsize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
impl StaticChatProvider {
|
||||||
|
pub fn new(response: impl Into<String>) -> Self {
|
||||||
|
Self {
|
||||||
|
response: response.into(),
|
||||||
|
calls: std::sync::atomic::AtomicUsize::new(0),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[async_trait]
|
||||||
|
impl ChatProvider for StaticChatProvider {
|
||||||
|
fn name(&self) -> &str {
|
||||||
|
"test:static"
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn chat_for_json(&self, _prompt: &ChatPrompt) -> Result<String> {
|
||||||
|
self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||||
|
Ok(self.response.clone())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub mod test_override {
|
||||||
|
use super::ChatProvider;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
tokio::task_local! {
|
||||||
|
static OVERRIDE: Arc<dyn ChatProvider>;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn current() -> Option<Arc<dyn ChatProvider>> {
|
||||||
|
OVERRIDE.try_with(Arc::clone).ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn with_provider<F, T>(provider: Arc<dyn ChatProvider>, fut: F) -> T
|
||||||
|
where
|
||||||
|
F: std::future::Future<Output = T>,
|
||||||
|
{
|
||||||
|
OVERRIDE.scope(provider, fut).await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn build_provider_returns_inference_wrapper_when_default() {
|
||||||
|
let cfg = Config::default();
|
||||||
|
let provider = build_chat_provider(&cfg).unwrap();
|
||||||
|
assert!(provider.name().contains("inference:"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn build_chat_runtime_defaults_to_openhuman_resolved_model() {
|
||||||
|
let cfg = Config::default();
|
||||||
|
let (_provider, model) = build_chat_runtime(&cfg).unwrap();
|
||||||
|
assert_eq!(model, "reasoning-v1");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn build_chat_runtime_still_builds_when_cloud_memory_model_is_overridden() {
|
||||||
|
let mut cfg = Config::default();
|
||||||
|
cfg.memory_tree.cloud_llm_model = Some("custom-summary-model".into());
|
||||||
|
let (_provider, model) = build_chat_runtime(&cfg).unwrap();
|
||||||
|
assert_eq!(model, "reasoning-v1");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn build_provider_returns_inference_wrapper_when_local_memory_is_configured() {
|
||||||
|
let mut cfg = Config::default();
|
||||||
|
cfg.memory_provider = Some("ollama:qwen2.5:0.5b".into());
|
||||||
|
let provider = build_chat_provider(&cfg).unwrap();
|
||||||
|
assert!(provider.name().contains("qwen2.5:0.5b"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn build_chat_runtime_preserves_local_memory_model() {
|
||||||
|
let mut cfg = Config::default();
|
||||||
|
cfg.memory_provider = Some("ollama:qwen2.5:0.5b".into());
|
||||||
|
let (_provider, model) = build_chat_runtime(&cfg).unwrap();
|
||||||
|
assert_eq!(model, "qwen2.5:0.5b");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn static_chat_provider_returns_response_and_counts() {
|
||||||
|
let p = StaticChatProvider::new("hello");
|
||||||
|
let prompt = ChatPrompt {
|
||||||
|
system: "sys".into(),
|
||||||
|
user: "u".into(),
|
||||||
|
temperature: 0.0,
|
||||||
|
kind: "test",
|
||||||
|
};
|
||||||
|
assert_eq!(p.chat_for_json(&prompt).await.unwrap(), "hello");
|
||||||
|
assert_eq!(p.calls.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,72 +0,0 @@
|
|||||||
//! Wire/storage types for the workspace-backed conversation store: threads,
|
|
||||||
//! messages, create requests, and partial-update patches.
|
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
use serde_json::Value;
|
|
||||||
|
|
||||||
/// A persisted conversation thread, mirroring one entry in `threads.jsonl`.
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct ConversationThread {
|
|
||||||
pub id: String,
|
|
||||||
pub title: String,
|
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
||||||
pub chat_id: Option<i64>,
|
|
||||||
pub is_active: bool,
|
|
||||||
pub message_count: usize,
|
|
||||||
pub last_message_at: String,
|
|
||||||
pub created_at: String,
|
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
||||||
pub parent_thread_id: Option<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
pub labels: Vec<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A single message appended to a thread's JSONL log.
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct ConversationMessage {
|
|
||||||
pub id: String,
|
|
||||||
pub content: String,
|
|
||||||
#[serde(rename = "type")]
|
|
||||||
pub message_type: String,
|
|
||||||
#[serde(default)]
|
|
||||||
pub extra_metadata: Value,
|
|
||||||
pub sender: String,
|
|
||||||
pub created_at: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Input payload to create-or-update a thread via [`super::ensure_thread`].
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct CreateConversationThread {
|
|
||||||
pub id: String,
|
|
||||||
pub title: String,
|
|
||||||
pub created_at: String,
|
|
||||||
#[serde(default)]
|
|
||||||
pub parent_thread_id: Option<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
pub labels: Option<Vec<String>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Partial update to apply to a stored message (e.g. rewriting `extraMetadata`).
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct ConversationMessagePatch {
|
|
||||||
#[serde(default)]
|
|
||||||
pub extra_metadata: Option<Value>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A single match returned by
|
|
||||||
/// [`super::store::ConversationStore::search_cross_thread_messages`]. Carries
|
|
||||||
/// the source `thread_id` so the caller can render provenance into the
|
|
||||||
/// `[Cross-chat context]` block (issue #1505).
|
|
||||||
#[derive(Debug, Clone, PartialEq)]
|
|
||||||
pub struct CrossThreadHit {
|
|
||||||
pub thread_id: String,
|
|
||||||
pub message_id: String,
|
|
||||||
pub role: String,
|
|
||||||
pub content: String,
|
|
||||||
pub created_at: String,
|
|
||||||
pub score: f64,
|
|
||||||
}
|
|
||||||
@@ -11,19 +11,19 @@ use serde::{Deserialize, Serialize};
|
|||||||
|
|
||||||
use crate::core::event_bus::{publish_global, DomainEvent};
|
use crate::core::event_bus::{publish_global, DomainEvent};
|
||||||
use crate::openhuman::config::Config;
|
use crate::openhuman::config::Config;
|
||||||
use crate::openhuman::memory_tree::canonicalize::{
|
use crate::openhuman::memory::jobs::{self, ExtractChunkPayload, NewJob};
|
||||||
|
use crate::openhuman::memory::score::{self, ScoreResult, ScoringConfig};
|
||||||
|
use crate::openhuman::memory::util::redact::redact;
|
||||||
|
use crate::openhuman::memory_store::chunks::store as chunk_store;
|
||||||
|
use crate::openhuman::memory_store::chunks::types::SourceKind;
|
||||||
|
use crate::openhuman::memory_store::chunks::{chunk_markdown, ChunkerInput, ChunkerOptions};
|
||||||
|
use crate::openhuman::memory_store::content as content_store;
|
||||||
|
use crate::openhuman::memory_sync::canonicalize::{
|
||||||
chat::{self, ChatBatch},
|
chat::{self, ChatBatch},
|
||||||
document::{self, DocumentInput},
|
document::{self, DocumentInput},
|
||||||
email::{self, EmailThread},
|
email::{self, EmailThread},
|
||||||
CanonicalisedSource,
|
CanonicalisedSource,
|
||||||
};
|
};
|
||||||
use crate::openhuman::memory_tree::chunker::{chunk_markdown, ChunkerInput, ChunkerOptions};
|
|
||||||
use crate::openhuman::memory_tree::content_store;
|
|
||||||
use crate::openhuman::memory_tree::jobs::{self, ExtractChunkPayload, NewJob};
|
|
||||||
use crate::openhuman::memory_tree::score::{self, ScoreResult, ScoringConfig};
|
|
||||||
use crate::openhuman::memory_tree::store;
|
|
||||||
use crate::openhuman::memory_tree::types::SourceKind;
|
|
||||||
use crate::openhuman::memory_tree::util::redact::redact;
|
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
const BODY_PREVIEW_MAX_BYTES: usize = 2048;
|
const BODY_PREVIEW_MAX_BYTES: usize = 2048;
|
||||||
@@ -124,7 +124,7 @@ pub async fn ingest_document(
|
|||||||
) -> Result<IngestResult> {
|
) -> Result<IngestResult> {
|
||||||
if already_ingested(config, SourceKind::Document, source_id).await? {
|
if already_ingested(config, SourceKind::Document, source_id).await? {
|
||||||
log::debug!(
|
log::debug!(
|
||||||
"[memory_tree::ingest] skip ingest_document — source_id_hash={} already ingested",
|
"[memory::ingest_pipeline] skip ingest_document — source_id_hash={} already ingested",
|
||||||
redact(source_id)
|
redact(source_id)
|
||||||
);
|
);
|
||||||
return Ok(IngestResult::already_ingested(source_id));
|
return Ok(IngestResult::already_ingested(source_id));
|
||||||
@@ -147,7 +147,7 @@ async fn already_ingested(
|
|||||||
) -> Result<bool> {
|
) -> Result<bool> {
|
||||||
let cfg = config.clone();
|
let cfg = config.clone();
|
||||||
let sid = source_id.to_string();
|
let sid = source_id.to_string();
|
||||||
tokio::task::spawn_blocking(move || store::is_source_ingested(&cfg, source_kind, &sid))
|
tokio::task::spawn_blocking(move || chunk_store::is_source_ingested(&cfg, source_kind, &sid))
|
||||||
.await
|
.await
|
||||||
.map_err(|e| anyhow::anyhow!("already_ingested join error: {e}"))?
|
.map_err(|e| anyhow::anyhow!("already_ingested join error: {e}"))?
|
||||||
}
|
}
|
||||||
@@ -175,8 +175,8 @@ async fn persist(
|
|||||||
Ok(preview) => Some(preview),
|
Ok(preview) => Some(preview),
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
log::error!(
|
log::error!(
|
||||||
"[memory_tree::ingest] markdown_body_preview panicked for source_id_hash={}; falling back to no preview",
|
"[memory::ingest_pipeline] markdown_body_preview panicked for source_id_hash={}; falling back to no preview",
|
||||||
crate::openhuman::memory_tree::util::redact::redact(source_id)
|
crate::openhuman::memory::util::redact::redact(source_id)
|
||||||
);
|
);
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
@@ -201,13 +201,13 @@ async fn persist(
|
|||||||
// spawn_blocking so errors surface before the DB transaction opens.
|
// spawn_blocking so errors surface before the DB transaction opens.
|
||||||
let content_root = config.memory_tree_content_root();
|
let content_root = config.memory_tree_content_root();
|
||||||
let staged = content_store::stage_chunks(&content_root, &chunks)
|
let staged = content_store::stage_chunks(&content_root, &chunks)
|
||||||
.map_err(|e| anyhow::anyhow!("[memory_tree::ingest] stage_chunks failed: {e}"))?;
|
.map_err(|e| anyhow::anyhow!("[memory::ingest_pipeline] stage_chunks failed: {e}"))?;
|
||||||
|
|
||||||
let scoring_cfg = ScoringConfig::from_config(config);
|
let scoring_cfg = ScoringConfig::from_config(config);
|
||||||
let scores = score::score_chunks_fast(&chunks, &scoring_cfg).await?;
|
let scores = score::score_chunks_fast(&chunks, &scoring_cfg).await?;
|
||||||
if scores.len() != chunks.len() {
|
if scores.len() != chunks.len() {
|
||||||
anyhow::bail!(
|
anyhow::bail!(
|
||||||
"[memory_tree::ingest] scorer length mismatch: chunks={} scores={}",
|
"[memory::ingest_pipeline] scorer length mismatch: chunks={} scores={}",
|
||||||
chunks.len(),
|
chunks.len(),
|
||||||
scores.len()
|
scores.len()
|
||||||
);
|
);
|
||||||
@@ -226,7 +226,7 @@ async fn persist(
|
|||||||
let source_id_for_store = source_id.to_string();
|
let source_id_for_store = source_id.to_string();
|
||||||
let written = tokio::task::spawn_blocking(move || -> Result<Option<usize>> {
|
let written = tokio::task::spawn_blocking(move || -> Result<Option<usize>> {
|
||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{HashMap, HashSet};
|
||||||
store::with_connection(&config_owned, |conn| {
|
chunk_store::with_connection(&config_owned, |conn| {
|
||||||
// IMMEDIATE, not the default DEFERRED: this transaction reads
|
// IMMEDIATE, not the default DEFERRED: this transaction reads
|
||||||
// (get_chunk_lifecycle_status_tx) before it writes
|
// (get_chunk_lifecycle_status_tx) before it writes
|
||||||
// (upsert_staged_chunks_tx). A DEFERRED tx takes only a read
|
// (upsert_staged_chunks_tx). A DEFERRED tx takes only a read
|
||||||
@@ -262,7 +262,7 @@ async fn persist(
|
|||||||
// genuine replays without blocking legitimate appends.
|
// genuine replays without blocking legitimate appends.
|
||||||
if source_kind_for_store == SourceKind::Document {
|
if source_kind_for_store == SourceKind::Document {
|
||||||
let now_ms = chrono::Utc::now().timestamp_millis();
|
let now_ms = chrono::Utc::now().timestamp_millis();
|
||||||
let claimed = store::claim_source_ingest_tx(
|
let claimed = chunk_store::claim_source_ingest_tx(
|
||||||
&tx,
|
&tx,
|
||||||
source_kind_for_store,
|
source_kind_for_store,
|
||||||
&source_id_for_store,
|
&source_id_for_store,
|
||||||
@@ -270,7 +270,7 @@ async fn persist(
|
|||||||
)?;
|
)?;
|
||||||
if !claimed {
|
if !claimed {
|
||||||
log::debug!(
|
log::debug!(
|
||||||
"[memory_tree::ingest] persist gate: document already ingested source_id_hash={}",
|
"[memory::ingest_pipeline] persist gate: document already ingested source_id_hash={}",
|
||||||
redact(&source_id_for_store)
|
redact(&source_id_for_store)
|
||||||
);
|
);
|
||||||
// Drop the (empty) transaction implicitly; nothing to commit.
|
// Drop the (empty) transaction implicitly; nothing to commit.
|
||||||
@@ -287,11 +287,11 @@ async fn persist(
|
|||||||
// "already-admitted-from-prior-ingest".
|
// "already-admitted-from-prior-ingest".
|
||||||
let mut prior: HashMap<String, Option<String>> = HashMap::new();
|
let mut prior: HashMap<String, Option<String>> = HashMap::new();
|
||||||
for s in &staged_for_store {
|
for s in &staged_for_store {
|
||||||
let status = store::get_chunk_lifecycle_status_tx(&tx, &s.chunk.id)?;
|
let status = chunk_store::get_chunk_lifecycle_status_tx(&tx, &s.chunk.id)?;
|
||||||
prior.insert(s.chunk.id.clone(), status);
|
prior.insert(s.chunk.id.clone(), status);
|
||||||
}
|
}
|
||||||
|
|
||||||
let n = store::upsert_staged_chunks_tx(&tx, &staged_for_store)?;
|
let n = chunk_store::upsert_staged_chunks_tx(&tx, &staged_for_store)?;
|
||||||
|
|
||||||
// Re-ingest of identical content (same chunk_id) must NOT
|
// Re-ingest of identical content (same chunk_id) must NOT
|
||||||
// downgrade chunks that have already progressed through the
|
// downgrade chunks that have already progressed through the
|
||||||
@@ -312,13 +312,13 @@ async fn persist(
|
|||||||
let pre = prior.get(&s.chunk.id).cloned().flatten();
|
let pre = prior.get(&s.chunk.id).cloned().flatten();
|
||||||
let needs_processing = matches!(
|
let needs_processing = matches!(
|
||||||
pre.as_deref(),
|
pre.as_deref(),
|
||||||
None | Some(store::CHUNK_STATUS_PENDING_EXTRACTION),
|
None | Some(chunk_store::CHUNK_STATUS_PENDING_EXTRACTION),
|
||||||
);
|
);
|
||||||
if needs_processing {
|
if needs_processing {
|
||||||
store::set_chunk_lifecycle_status_tx(
|
chunk_store::set_chunk_lifecycle_status_tx(
|
||||||
&tx,
|
&tx,
|
||||||
&s.chunk.id,
|
&s.chunk.id,
|
||||||
store::CHUNK_STATUS_PENDING_EXTRACTION,
|
chunk_store::CHUNK_STATUS_PENDING_EXTRACTION,
|
||||||
)?;
|
)?;
|
||||||
to_schedule.insert(s.chunk.id.clone());
|
to_schedule.insert(s.chunk.id.clone());
|
||||||
}
|
}
|
||||||
@@ -407,7 +407,7 @@ fn markdown_body_preview(md: &str) -> String {
|
|||||||
// continuation byte; fall back to the full string rather than panicking.
|
// continuation byte; fall back to the full string rather than panicking.
|
||||||
if start > len || !md.is_char_boundary(start) {
|
if start > len || !md.is_char_boundary(start) {
|
||||||
log::error!(
|
log::error!(
|
||||||
"[memory_tree::ingest] ceil_char_boundary returned invalid boundary start={start} len={len}; returning full markdown"
|
"[memory::ingest_pipeline] ceil_char_boundary returned invalid boundary start={start} len={len}; returning full markdown"
|
||||||
);
|
);
|
||||||
md.to_string()
|
md.to_string()
|
||||||
} else {
|
} else {
|
||||||
@@ -419,14 +419,14 @@ fn markdown_body_preview(md: &str) -> String {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::openhuman::memory_tree::canonicalize::chat::ChatMessage;
|
use crate::openhuman::memory::jobs::drain_until_idle;
|
||||||
use crate::openhuman::memory_tree::jobs::drain_until_idle;
|
use crate::openhuman::memory::score::store::{count_scores, lookup_entity};
|
||||||
use crate::openhuman::memory_tree::score::store::{count_scores, lookup_entity};
|
use crate::openhuman::memory_store::chunks::store::{
|
||||||
use crate::openhuman::memory_tree::store::{
|
|
||||||
count_chunks, count_chunks_by_lifecycle_status, get_chunk_embedding, list_chunks,
|
count_chunks, count_chunks_by_lifecycle_status, get_chunk_embedding, list_chunks,
|
||||||
ListChunksQuery, CHUNK_STATUS_BUFFERED, CHUNK_STATUS_DROPPED,
|
ListChunksQuery, CHUNK_STATUS_BUFFERED, CHUNK_STATUS_DROPPED,
|
||||||
};
|
};
|
||||||
use crate::openhuman::memory_tree::types::SourceKind;
|
use crate::openhuman::memory_store::chunks::types::SourceKind;
|
||||||
|
use crate::openhuman::memory_sync::canonicalize::chat::ChatMessage;
|
||||||
use chrono::{TimeZone, Utc};
|
use chrono::{TimeZone, Utc};
|
||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
|
|
||||||
@@ -30,8 +30,8 @@ use parse::{enrich_document_metadata, parse_document};
|
|||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
use types::ParsedIngestion;
|
use types::ParsedIngestion;
|
||||||
|
|
||||||
use crate::openhuman::memory::store::types::NamespaceDocumentInput;
|
|
||||||
use crate::openhuman::memory::UnifiedMemory;
|
use crate::openhuman::memory::UnifiedMemory;
|
||||||
|
use crate::openhuman::memory_store::types::NamespaceDocumentInput;
|
||||||
|
|
||||||
impl UnifiedMemory {
|
impl UnifiedMemory {
|
||||||
/// Run the full ingestion pipeline for a document: parse + chunk + extract
|
/// Run the full ingestion pipeline for a document: parse + chunk + extract
|
||||||
|
|||||||
@@ -14,8 +14,8 @@ use super::types::{
|
|||||||
ExtractedEntity, ExtractedRelation, ExtractionAccumulator, ExtractionMode, ExtractionUnit,
|
ExtractedEntity, ExtractedRelation, ExtractionAccumulator, ExtractionMode, ExtractionUnit,
|
||||||
MemoryIngestionConfig, ParsedIngestion, RawEntity, RawRelation, DEFAULT_CHUNK_TOKENS,
|
MemoryIngestionConfig, ParsedIngestion, RawEntity, RawRelation, DEFAULT_CHUNK_TOKENS,
|
||||||
};
|
};
|
||||||
use crate::openhuman::memory::store::types::NamespaceDocumentInput;
|
|
||||||
use crate::openhuman::memory::UnifiedMemory;
|
use crate::openhuman::memory::UnifiedMemory;
|
||||||
|
use crate::openhuman::memory_store::types::NamespaceDocumentInput;
|
||||||
|
|
||||||
// ── Chunking helpers ──────────────────────────────────────────────────────────
|
// ── Chunking helpers ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -933,3 +933,143 @@ pub(super) async fn parse_document(
|
|||||||
decision_count: accumulator.decisions.len(),
|
decision_count: accumulator.decisions.len(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::openhuman::memory_store::types::NamespaceDocumentInput;
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
fn sample_input() -> NamespaceDocumentInput {
|
||||||
|
NamespaceDocumentInput {
|
||||||
|
namespace: "global".into(),
|
||||||
|
key: "doc-1".into(),
|
||||||
|
title: "OpenHuman roadmap".into(),
|
||||||
|
content: "Alice owns roadmap".into(),
|
||||||
|
source_type: "manual".into(),
|
||||||
|
priority: "normal".into(),
|
||||||
|
tags: vec!["existing".into()],
|
||||||
|
metadata: json!({"seed": true}),
|
||||||
|
category: "core".into(),
|
||||||
|
session_id: Some("session-1".into()),
|
||||||
|
document_id: Some("doc-id-1".into()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn split_sentences_breaks_on_punctuation_and_merges_tiny_fragments() {
|
||||||
|
let parts = split_sentences("Hello world. Ok.\nNext line?");
|
||||||
|
assert_eq!(parts.len(), 2);
|
||||||
|
assert_eq!(parts[0], "Hello world Ok");
|
||||||
|
assert_eq!(parts[1], "Next line?");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn build_units_respects_extraction_mode() {
|
||||||
|
let chunks = vec!["One. Two.".to_string(), "Three".to_string()];
|
||||||
|
let sentence_units = build_units(&chunks, ExtractionMode::Sentence);
|
||||||
|
let chunk_units = build_units(&chunks, ExtractionMode::Chunk);
|
||||||
|
|
||||||
|
assert_eq!(sentence_units.len(), 2);
|
||||||
|
assert_eq!(sentence_units[0].chunk_index, 0);
|
||||||
|
assert_eq!(sentence_units[0].text, "One Two");
|
||||||
|
assert_eq!(sentence_units[1].chunk_index, 1);
|
||||||
|
assert_eq!(sentence_units[1].text, "Three");
|
||||||
|
|
||||||
|
assert_eq!(chunk_units.len(), 2);
|
||||||
|
assert_eq!(chunk_units[0].text, "One. Two");
|
||||||
|
assert_eq!(chunk_units[1].text, "Three");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn find_chunk_index_prefers_hint_then_wraps() {
|
||||||
|
let chunks = vec![
|
||||||
|
"alpha content".to_string(),
|
||||||
|
"beta needle".to_string(),
|
||||||
|
"gamma trailing".to_string(),
|
||||||
|
];
|
||||||
|
assert_eq!(find_chunk_index(&chunks, "needle", 1), 1);
|
||||||
|
assert_eq!(find_chunk_index(&chunks, "alpha", 2), 0);
|
||||||
|
assert_eq!(find_chunk_index(&chunks, "missing", 2), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn alias_map_builds_reverse_lookup() {
|
||||||
|
let mut entities = HashMap::new();
|
||||||
|
entities.insert(
|
||||||
|
"ALICE".into(),
|
||||||
|
RawEntity {
|
||||||
|
name: "ALICE".into(),
|
||||||
|
entity_type: "PERSON".into(),
|
||||||
|
confidence: 0.8,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
entities.insert(
|
||||||
|
"ALICE SMITH".into(),
|
||||||
|
RawEntity {
|
||||||
|
name: "ALICE SMITH".into(),
|
||||||
|
entity_type: "PERSON".into(),
|
||||||
|
confidence: 0.9,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let aliases = build_alias_map(&entities);
|
||||||
|
assert_eq!(
|
||||||
|
aliases.get("ALICE").map(String::as_str),
|
||||||
|
Some("ALICE SMITH")
|
||||||
|
);
|
||||||
|
assert_eq!(resolve_alias("ALICE", &aliases), "ALICE SMITH");
|
||||||
|
|
||||||
|
let reverse = reverse_aliases(&aliases);
|
||||||
|
assert_eq!(reverse.get("ALICE SMITH"), Some(&vec!["ALICE".to_string()]));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn enrich_document_metadata_merges_tags_and_ingestion_details() {
|
||||||
|
let input = sample_input();
|
||||||
|
let parsed = ParsedIngestion {
|
||||||
|
tags: vec!["decision".into(), "existing".into()],
|
||||||
|
metadata: json!({"kind": "profile", "extra": 1}),
|
||||||
|
entities: vec![],
|
||||||
|
relations: vec![],
|
||||||
|
chunk_count: 3,
|
||||||
|
preference_count: 1,
|
||||||
|
decision_count: 2,
|
||||||
|
};
|
||||||
|
let config = MemoryIngestionConfig::default();
|
||||||
|
let (enriched, tags) = enrich_document_metadata(&input, &parsed, &config);
|
||||||
|
|
||||||
|
assert_eq!(tags, vec!["decision".to_string(), "existing".to_string()]);
|
||||||
|
assert_eq!(enriched.tags, tags);
|
||||||
|
assert_eq!(enriched.metadata["seed"], json!(true));
|
||||||
|
assert_eq!(enriched.metadata["extra"], json!(1));
|
||||||
|
assert_eq!(
|
||||||
|
enriched.metadata["ingestion"]["model_name"],
|
||||||
|
config.model_name
|
||||||
|
);
|
||||||
|
assert_eq!(enriched.metadata["ingestion"]["chunk_count"], json!(3));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extract_people_from_header_collects_named_people() {
|
||||||
|
let mut acc = ExtractionAccumulator::default();
|
||||||
|
let people = extract_people_from_header(
|
||||||
|
"Alice Smith <alice@example.com>, Bob Jones <bob@example.com>",
|
||||||
|
&mut acc,
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
people,
|
||||||
|
vec!["ALICE SMITH".to_string(), "BOB JONES".to_string()]
|
||||||
|
);
|
||||||
|
assert!(acc.entities.contains_key("ALICE SMITH"));
|
||||||
|
assert!(acc.entities.contains_key("BOB JONES"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn detect_primary_subject_only_matches_openhuman() {
|
||||||
|
assert_eq!(
|
||||||
|
detect_primary_subject("OpenHuman desktop roadmap"),
|
||||||
|
Some("OPENHUMAN".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(detect_primary_subject("General roadmap"), None);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ use tokio::sync::mpsc;
|
|||||||
use super::state::IngestionState;
|
use super::state::IngestionState;
|
||||||
use super::MemoryIngestionConfig;
|
use super::MemoryIngestionConfig;
|
||||||
use crate::core::event_bus::{publish_global, DomainEvent};
|
use crate::core::event_bus::{publish_global, DomainEvent};
|
||||||
use crate::openhuman::memory::store::{NamespaceDocumentInput, UnifiedMemory};
|
use crate::openhuman::memory_store::{NamespaceDocumentInput, UnifiedMemory};
|
||||||
|
|
||||||
/// Default capacity of the ingestion job channel.
|
/// Default capacity of the ingestion job channel.
|
||||||
///
|
///
|
||||||
|
|||||||
@@ -187,3 +187,33 @@ pub(super) fn classify_entity(name: &str, known_people: &HashMap<String, String>
|
|||||||
}
|
}
|
||||||
"TOPIC"
|
"TOPIC"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sanitize_entity_name_trims_punctuation_and_uppercases() {
|
||||||
|
assert_eq!(sanitize_entity_name(" Alice Smith. "), "ALICE SMITH");
|
||||||
|
assert_eq!(sanitize_entity_name("\"openhuman\""), "OPENHUMAN");
|
||||||
|
assert_eq!(sanitize_entity_name(""), "");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sanitize_fact_text_collapses_whitespace_and_strips_edges() {
|
||||||
|
assert_eq!(sanitize_fact_text(" - Hello world. "), "Hello world");
|
||||||
|
assert_eq!(sanitize_fact_text(":: spaced\ttext ;;"), "spaced text");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn classify_entity_detects_dates_people_and_products() {
|
||||||
|
let mut known_people = HashMap::new();
|
||||||
|
known_people.insert("ALICE SMITH".to_string(), "ALICE SMITH".to_string());
|
||||||
|
|
||||||
|
assert_eq!(classify_entity("Jan 5, 2026", &known_people), "DATE");
|
||||||
|
assert_eq!(classify_entity("Alice Smith", &known_people), "PERSON");
|
||||||
|
assert_eq!(classify_entity("OpenHuman", &known_people), "PRODUCT");
|
||||||
|
assert_eq!(classify_entity("Kitchen", &known_people), "ROOM");
|
||||||
|
assert_eq!(classify_entity("phoenix-project", &known_people), "PROJECT");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -220,3 +220,106 @@ impl ExtractionAccumulator {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn relation_rule_normalizes_supported_predicates() {
|
||||||
|
let owns = relation_rule("works_on").unwrap();
|
||||||
|
assert_eq!(owns.canonical, "OWNS");
|
||||||
|
assert_eq!(owns.allowed_head, PERSON_TYPES);
|
||||||
|
|
||||||
|
let prefers = relation_rule("prefers").unwrap();
|
||||||
|
assert_eq!(prefers.canonical, "PREFERS");
|
||||||
|
|
||||||
|
let deadline = relation_rule("due_on").unwrap();
|
||||||
|
assert_eq!(deadline.canonical, "HAS_DEADLINE");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn type_allowed_honors_allowlist_and_empty_list() {
|
||||||
|
assert!(type_allowed("PERSON", PERSON_TYPES));
|
||||||
|
assert!(!type_allowed("PROJECT", PERSON_TYPES));
|
||||||
|
assert!(type_allowed("ANYTHING", &[]));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resolve_person_alias_uses_known_people_map() {
|
||||||
|
let mut known = std::collections::HashMap::new();
|
||||||
|
known.insert("ALICE".to_string(), "ALICE SMITH".to_string());
|
||||||
|
assert_eq!(resolve_person_alias("ALICE", &known), "ALICE SMITH");
|
||||||
|
assert_eq!(resolve_person_alias("BOB", &known), "BOB");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn add_entity_tracks_highest_confidence_and_person_aliases() {
|
||||||
|
let mut acc = ExtractionAccumulator::default();
|
||||||
|
let first = acc.add_entity("Alice Smith", "PERSON", 0.6).unwrap();
|
||||||
|
let second = acc.add_entity("Alice Smith", "PERSON", 0.9).unwrap();
|
||||||
|
assert_eq!(first, "ALICE SMITH");
|
||||||
|
assert_eq!(second, "ALICE SMITH");
|
||||||
|
assert_eq!(acc.entities["ALICE SMITH"].confidence, 0.9);
|
||||||
|
assert_eq!(
|
||||||
|
acc.known_people.get("ALICE").map(String::as_str),
|
||||||
|
Some("ALICE SMITH")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn add_relation_rejects_invalid_or_self_relations() {
|
||||||
|
let mut acc = ExtractionAccumulator::default();
|
||||||
|
acc.add_relation(
|
||||||
|
"Alice",
|
||||||
|
"PERSON",
|
||||||
|
"owns",
|
||||||
|
"Alice",
|
||||||
|
"PERSON",
|
||||||
|
0.8,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
Map::new(),
|
||||||
|
);
|
||||||
|
assert!(acc.relations.is_empty(), "self relation should be dropped");
|
||||||
|
|
||||||
|
acc.add_relation(
|
||||||
|
"Alice",
|
||||||
|
"PERSON",
|
||||||
|
"unknown_predicate",
|
||||||
|
"Project X",
|
||||||
|
"PROJECT",
|
||||||
|
0.8,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
Map::new(),
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
acc.relations.is_empty(),
|
||||||
|
"unknown predicate should be ignored"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn add_relation_canonicalizes_predicate_and_collects_chunk_index() {
|
||||||
|
let mut acc = ExtractionAccumulator::default();
|
||||||
|
acc.add_relation(
|
||||||
|
"Alice",
|
||||||
|
"PERSON",
|
||||||
|
"works_on",
|
||||||
|
"Phoenix",
|
||||||
|
"PROJECT",
|
||||||
|
0.8,
|
||||||
|
3,
|
||||||
|
11,
|
||||||
|
Map::new(),
|
||||||
|
);
|
||||||
|
assert_eq!(acc.relations.len(), 1);
|
||||||
|
let relation = &acc.relations[0];
|
||||||
|
assert_eq!(relation.predicate, "OWNS");
|
||||||
|
assert_eq!(relation.subject, "ALICE");
|
||||||
|
assert_eq!(relation.object, "PHOENIX");
|
||||||
|
assert!(relation.chunk_indexes.contains(&3));
|
||||||
|
assert_eq!(relation.order_index, 11);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ use std::collections::{BTreeSet, HashMap};
|
|||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::{Map, Value};
|
use serde_json::{Map, Value};
|
||||||
|
|
||||||
use crate::openhuman::memory::store::types::NamespaceDocumentInput;
|
use crate::openhuman::memory_store::types::NamespaceDocumentInput;
|
||||||
|
|
||||||
/// Default extraction backend label reported in ingestion metadata.
|
/// Default extraction backend label reported in ingestion metadata.
|
||||||
pub const DEFAULT_MEMORY_EXTRACTION_MODEL: &str = "heuristic-only";
|
pub const DEFAULT_MEMORY_EXTRACTION_MODEL: &str = "heuristic-only";
|
||||||
@@ -243,3 +243,45 @@ pub(super) struct ParsedIngestion {
|
|||||||
pub(super) preference_count: usize,
|
pub(super) preference_count: usize,
|
||||||
pub(super) decision_count: usize,
|
pub(super) decision_count: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extraction_mode_default_is_sentence() {
|
||||||
|
assert_eq!(ExtractionMode::default(), ExtractionMode::Sentence);
|
||||||
|
assert_eq!(ExtractionMode::Sentence.as_str(), "sentence");
|
||||||
|
assert_eq!(ExtractionMode::Chunk.as_str(), "chunk");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn memory_ingestion_config_default_matches_expected_thresholds() {
|
||||||
|
let cfg = MemoryIngestionConfig::default();
|
||||||
|
assert_eq!(cfg.model_name, DEFAULT_MEMORY_EXTRACTION_MODEL);
|
||||||
|
assert_eq!(cfg.extraction_mode, ExtractionMode::Sentence);
|
||||||
|
assert_eq!(cfg.entity_threshold, 0.45);
|
||||||
|
assert_eq!(cfg.relation_threshold, 0.30);
|
||||||
|
assert_eq!(cfg.adjacency_threshold, 0.50);
|
||||||
|
assert_eq!(cfg.batch_size, 16);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn memory_ingestion_request_defaults_config_when_absent() {
|
||||||
|
let request: MemoryIngestionRequest = serde_json::from_value(json!({
|
||||||
|
"document": {
|
||||||
|
"namespace": "global",
|
||||||
|
"key": "doc-1",
|
||||||
|
"title": "Doc",
|
||||||
|
"content": "Body",
|
||||||
|
"source_type": "manual",
|
||||||
|
"priority": "normal",
|
||||||
|
"category": "core"
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(request.config.model_name, DEFAULT_MEMORY_EXTRACTION_MODEL);
|
||||||
|
assert_eq!(request.config.extraction_mode, ExtractionMode::Sentence);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+43
-23
@@ -1,24 +1,56 @@
|
|||||||
//! Memory system for OpenHuman.
|
//! Memory orchestration.
|
||||||
//!
|
//!
|
||||||
//! This module provides the core abstractions and implementations for the memory system,
|
//! This module is the high-level routing + policy layer over the memory
|
||||||
//! including semantic search, ingestion pipelines, document management, and knowledge graph
|
//! stack. Owns the ingest pipeline, background job handlers, scoring,
|
||||||
//! operations. It integrates vector search, keyword search, and relational data to provide
|
//! tree-building policy (tree_global / tree_topic), recall ranking, and
|
||||||
//! a unified memory interface for AI agents.
|
//! the RPC surface. Storage primitives all live in sibling memory_*
|
||||||
|
//! modules — see [`README.md`](README.md) for the full map.
|
||||||
|
//!
|
||||||
|
//! No SQLite, no on-disk md, no vector tables here — those belong one
|
||||||
|
//! layer down in [`memory_store`](crate::openhuman::memory_store).
|
||||||
|
|
||||||
pub mod chunker;
|
// Legacy memory modules
|
||||||
pub mod conversations;
|
|
||||||
pub mod global;
|
pub mod global;
|
||||||
pub mod ingestion;
|
pub mod ingestion;
|
||||||
pub mod ops;
|
pub mod ops;
|
||||||
pub mod preferences;
|
pub mod preferences;
|
||||||
pub mod rpc_models;
|
pub mod rpc_models;
|
||||||
pub mod safety;
|
|
||||||
pub mod schemas;
|
pub mod schemas;
|
||||||
pub mod stm_recall;
|
pub mod stm_recall;
|
||||||
pub mod store;
|
|
||||||
pub mod sync_status;
|
|
||||||
pub mod tool_memory;
|
|
||||||
pub mod traits;
|
pub mod traits;
|
||||||
|
|
||||||
|
// Tool-scoped memory moved to top-level `memory_tools`. Re-exported here so
|
||||||
|
// existing `memory::tool_memory::*` paths still resolve during the migration.
|
||||||
|
pub use crate::openhuman::memory_tools as tool_memory;
|
||||||
|
|
||||||
|
// Modules moved from memory_tree (Phase 3)
|
||||||
|
pub mod chat;
|
||||||
|
pub mod ingest_pipeline;
|
||||||
|
pub mod read_rpc;
|
||||||
|
pub mod retrieval;
|
||||||
|
pub mod schema;
|
||||||
|
pub mod score;
|
||||||
|
pub mod tree_rpc;
|
||||||
|
pub mod util;
|
||||||
|
|
||||||
|
// Conversation storage moved to top-level `memory_conversations`. Re-exported
|
||||||
|
// here so existing `memory::conversations::*` paths still resolve during the
|
||||||
|
// migration.
|
||||||
|
pub use crate::openhuman::memory_conversations as conversations;
|
||||||
|
// Async memory job queue moved to top-level `memory_queue`. Re-exported here
|
||||||
|
// so existing `memory::jobs::*` paths still resolve during the migration.
|
||||||
|
pub use crate::openhuman::memory_queue as jobs;
|
||||||
|
|
||||||
|
pub use crate::openhuman::memory_store::{
|
||||||
|
create_memory, create_memory_for_migration, create_memory_with_local_ai,
|
||||||
|
effective_embedding_settings, effective_memory_backend_name, MemoryClient, MemoryClientRef,
|
||||||
|
MemoryItemKind, MemoryState, NamespaceDocumentInput, NamespaceMemoryHit, NamespaceQueryResult,
|
||||||
|
NamespaceRetrievalContext, RetrievalScoreBreakdown, UnifiedMemory,
|
||||||
|
};
|
||||||
|
pub use crate::openhuman::memory_sync::sync_status::{
|
||||||
|
all_memory_sync_status_controller_schemas, all_memory_sync_status_registered_controllers,
|
||||||
|
FreshnessLabel, MemorySyncStatus,
|
||||||
|
};
|
||||||
pub use ingestion::{
|
pub use ingestion::{
|
||||||
ExtractedEntity, ExtractedRelation, ExtractionMode, IngestionJob, IngestionQueue,
|
ExtractedEntity, ExtractedRelation, ExtractionMode, IngestionJob, IngestionQueue,
|
||||||
IngestionState, IngestionStatusSnapshot, MemoryIngestionConfig, MemoryIngestionRequest,
|
IngestionState, IngestionStatusSnapshot, MemoryIngestionConfig, MemoryIngestionRequest,
|
||||||
@@ -31,18 +63,6 @@ pub use schemas::{
|
|||||||
all_controller_schemas as all_memory_controller_schemas,
|
all_controller_schemas as all_memory_controller_schemas,
|
||||||
all_registered_controllers as all_memory_registered_controllers,
|
all_registered_controllers as all_memory_registered_controllers,
|
||||||
};
|
};
|
||||||
pub use store::{
|
|
||||||
create_memory, create_memory_for_migration, create_memory_with_local_ai,
|
|
||||||
create_memory_with_storage, create_memory_with_storage_and_routes,
|
|
||||||
effective_embedding_settings, effective_embedding_settings_probed,
|
|
||||||
effective_memory_backend_name, MemoryClient, MemoryClientRef, MemoryItemKind, MemoryState,
|
|
||||||
NamespaceDocumentInput, NamespaceMemoryHit, NamespaceQueryResult, NamespaceRetrievalContext,
|
|
||||||
RetrievalScoreBreakdown, UnifiedMemory,
|
|
||||||
};
|
|
||||||
pub use sync_status::{
|
|
||||||
all_memory_sync_status_controller_schemas, all_memory_sync_status_registered_controllers,
|
|
||||||
FreshnessLabel, MemorySyncStatus,
|
|
||||||
};
|
|
||||||
pub use tool_memory::{
|
pub use tool_memory::{
|
||||||
render_tool_memory_rules, tool_memory_namespace, ToolMemoryCaptureHook, ToolMemoryPriority,
|
render_tool_memory_rules, tool_memory_namespace, ToolMemoryCaptureHook, ToolMemoryPriority,
|
||||||
ToolMemoryRule, ToolMemoryRulesSection, ToolMemorySource, ToolMemoryStore, TOOL_MEMORY_HEADING,
|
ToolMemoryRule, ToolMemoryRulesSection, ToolMemorySource, ToolMemoryStore, TOOL_MEMORY_HEADING,
|
||||||
|
|||||||
@@ -489,3 +489,222 @@ pub async fn memory_recall_memories(
|
|||||||
Err(message) => Ok(error_envelope("memory.recall_memories_failed", message)),
|
Err(message) => Ok(error_envelope("memory.recall_memories_failed", message)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::sync::OnceLock;
|
||||||
|
|
||||||
|
use serde_json::json;
|
||||||
|
use tempfile::TempDir;
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn ensure_memory_client() {
|
||||||
|
static WORKSPACE: OnceLock<PathBuf> = OnceLock::new();
|
||||||
|
let workspace = WORKSPACE.get_or_init(|| {
|
||||||
|
let tmp = TempDir::new().expect("tempdir");
|
||||||
|
let path = tmp.path().join("workspace");
|
||||||
|
std::fs::create_dir_all(&path).expect("workspace dir");
|
||||||
|
std::mem::forget(tmp);
|
||||||
|
path
|
||||||
|
});
|
||||||
|
let _ = crate::openhuman::memory::global::init(workspace.clone());
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unique_namespace(prefix: &str) -> String {
|
||||||
|
format!("{prefix}-{}", uuid::Uuid::new_v4())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sample_put(namespace: String, key: String, title: &str, content: &str) -> PutDocParams {
|
||||||
|
PutDocParams {
|
||||||
|
namespace,
|
||||||
|
key,
|
||||||
|
title: title.into(),
|
||||||
|
content: content.into(),
|
||||||
|
source_type: default_source_type(),
|
||||||
|
priority: default_priority(),
|
||||||
|
tags: vec!["test".into()],
|
||||||
|
metadata: json!({"source": "test"}),
|
||||||
|
category: default_category(),
|
||||||
|
session_id: Some("session-docs".into()),
|
||||||
|
document_id: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn direct_document_handlers_roundtrip_through_namespace() {
|
||||||
|
ensure_memory_client();
|
||||||
|
let namespace = unique_namespace("memory-docs-direct");
|
||||||
|
let key = format!("note-{}", uuid::Uuid::new_v4());
|
||||||
|
|
||||||
|
let put = doc_put(sample_put(
|
||||||
|
namespace.clone(),
|
||||||
|
key.clone(),
|
||||||
|
"Rust ownership",
|
||||||
|
"Ownership and borrowing let Rust enforce memory safety.",
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.expect("doc_put");
|
||||||
|
let document_id = put.value.document_id.clone();
|
||||||
|
assert!(!document_id.is_empty());
|
||||||
|
|
||||||
|
let listed = doc_list(Some(NamespaceOnlyParams {
|
||||||
|
namespace: namespace.clone(),
|
||||||
|
}))
|
||||||
|
.await
|
||||||
|
.expect("doc_list");
|
||||||
|
let docs = listed
|
||||||
|
.value
|
||||||
|
.get("documents")
|
||||||
|
.and_then(|v| v.as_array())
|
||||||
|
.expect("documents array");
|
||||||
|
assert!(docs.iter().any(|doc| doc["key"] == key));
|
||||||
|
|
||||||
|
let queried = context_query(QueryNamespaceParams {
|
||||||
|
namespace: namespace.clone(),
|
||||||
|
query: "ownership".into(),
|
||||||
|
limit: Some(5),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("context_query");
|
||||||
|
assert!(
|
||||||
|
queried.value.to_lowercase().contains("ownership"),
|
||||||
|
"query result should mention the stored concept"
|
||||||
|
);
|
||||||
|
|
||||||
|
let recalled = context_recall(RecallNamespaceParams {
|
||||||
|
namespace: namespace.clone(),
|
||||||
|
limit: Some(5),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("context_recall");
|
||||||
|
assert!(recalled.value.is_some());
|
||||||
|
|
||||||
|
let deleted = doc_delete(DeleteDocParams {
|
||||||
|
namespace: namespace.clone(),
|
||||||
|
document_id: document_id.clone(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("doc_delete");
|
||||||
|
assert_eq!(deleted.logs, vec!["memory document deleted".to_string()]);
|
||||||
|
|
||||||
|
let deleted_flag = deleted
|
||||||
|
.value
|
||||||
|
.get("deleted")
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false);
|
||||||
|
assert!(deleted_flag, "delete result should report success");
|
||||||
|
|
||||||
|
let after = doc_list(Some(NamespaceOnlyParams { namespace }))
|
||||||
|
.await
|
||||||
|
.expect("doc_list after delete");
|
||||||
|
let after_docs = after
|
||||||
|
.value
|
||||||
|
.get("documents")
|
||||||
|
.and_then(|v| v.as_array())
|
||||||
|
.expect("documents array after delete");
|
||||||
|
assert!(after_docs.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn envelope_memory_handlers_report_counts_and_statuses() {
|
||||||
|
ensure_memory_client();
|
||||||
|
let namespace = unique_namespace("memory-docs-envelope");
|
||||||
|
let key = format!("env-{}", uuid::Uuid::new_v4());
|
||||||
|
|
||||||
|
let _ = memory_init(MemoryInitRequest { jwt_token: None })
|
||||||
|
.await
|
||||||
|
.expect("memory_init");
|
||||||
|
|
||||||
|
let direct = doc_put(sample_put(
|
||||||
|
namespace.clone(),
|
||||||
|
key.clone(),
|
||||||
|
"Borrow checker",
|
||||||
|
"The borrow checker enforces aliasing and mutation rules.",
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.expect("seed document");
|
||||||
|
let document_id = direct.value.document_id;
|
||||||
|
|
||||||
|
let listed = memory_list_documents(ListDocumentsRequest {
|
||||||
|
namespace: Some(namespace.clone()),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("memory_list_documents");
|
||||||
|
let listed_data = listed.value.data.expect("list envelope data");
|
||||||
|
assert_eq!(listed_data.count, 1);
|
||||||
|
assert_eq!(listed_data.documents[0].key, key);
|
||||||
|
assert_eq!(
|
||||||
|
listed
|
||||||
|
.value
|
||||||
|
.meta
|
||||||
|
.counts
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|m| m.get("num_documents")),
|
||||||
|
Some(&1)
|
||||||
|
);
|
||||||
|
|
||||||
|
let namespaces = memory_list_namespaces(EmptyRequest {})
|
||||||
|
.await
|
||||||
|
.expect("memory_list_namespaces");
|
||||||
|
let namespace_data = namespaces.value.data.expect("namespace data");
|
||||||
|
assert!(
|
||||||
|
namespace_data.namespaces.iter().any(|ns| ns == &namespace),
|
||||||
|
"expected namespace list to include the seeded namespace"
|
||||||
|
);
|
||||||
|
|
||||||
|
let queried = memory_query_namespace(QueryNamespaceRequest {
|
||||||
|
namespace: namespace.clone(),
|
||||||
|
query: "borrow checker".into(),
|
||||||
|
limit: Some(5),
|
||||||
|
max_chunks: None,
|
||||||
|
include_references: Some(true),
|
||||||
|
document_ids: None,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("memory_query_namespace");
|
||||||
|
let query_data = queried.value.data.expect("query data");
|
||||||
|
assert!(query_data.llm_context_message.is_some());
|
||||||
|
assert!(query_data.context.is_some());
|
||||||
|
|
||||||
|
let recalled = memory_recall_memories(RecallMemoriesRequest {
|
||||||
|
namespace: namespace.clone(),
|
||||||
|
min_retention: None,
|
||||||
|
as_of: None,
|
||||||
|
limit: Some(5),
|
||||||
|
max_chunks: None,
|
||||||
|
top_k: None,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("memory_recall_memories");
|
||||||
|
let recall_data = recalled.value.data.expect("recall data");
|
||||||
|
assert_eq!(recall_data.memories.len(), 1);
|
||||||
|
assert_eq!(recall_data.memories[0].kind, "document");
|
||||||
|
|
||||||
|
let deleted = memory_delete_document(DeleteDocumentRequest {
|
||||||
|
namespace: namespace.clone(),
|
||||||
|
document_id,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("memory_delete_document");
|
||||||
|
let deleted_data = deleted.value.data.expect("delete envelope data");
|
||||||
|
assert_eq!(deleted_data.status, "completed");
|
||||||
|
assert!(deleted_data.deleted);
|
||||||
|
|
||||||
|
let cleared = clear_namespace(ClearNamespaceParams {
|
||||||
|
namespace: namespace.clone(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("clear_namespace");
|
||||||
|
assert!(cleared.value.cleared);
|
||||||
|
|
||||||
|
let listed_after = memory_list_documents(ListDocumentsRequest {
|
||||||
|
namespace: Some(namespace),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("memory_list_documents after clear");
|
||||||
|
let after_data = listed_after.value.data.expect("after clear data");
|
||||||
|
assert_eq!(after_data.count, 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -80,3 +80,34 @@ pub(crate) fn error_envelope<T: Serialize>(
|
|||||||
vec![],
|
vec![],
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn memory_counts_collects_named_entries() {
|
||||||
|
let counts = memory_counts([("documents", 2), ("entities", 5)]);
|
||||||
|
assert_eq!(counts.get("documents"), Some(&2));
|
||||||
|
assert_eq!(counts.get("entities"), Some(&5));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn envelope_wraps_data_with_meta() {
|
||||||
|
let outcome = envelope(serde_json::json!({"ok": true}), None, None);
|
||||||
|
let value = outcome.into_cli_compatible_json().unwrap();
|
||||||
|
assert_eq!(value["data"]["ok"], true);
|
||||||
|
assert!(value["meta"]["request_id"].as_str().is_some());
|
||||||
|
assert!(value["error"].is_null());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn error_envelope_wraps_code_and_message() {
|
||||||
|
let outcome: RpcOutcome<ApiEnvelope<serde_json::Value>> =
|
||||||
|
error_envelope("bad_request", "boom".to_string());
|
||||||
|
let value = outcome.into_cli_compatible_json().unwrap();
|
||||||
|
assert_eq!(value["error"]["code"], "bad_request");
|
||||||
|
assert_eq!(value["error"]["message"], "boom");
|
||||||
|
assert!(value["data"].is_null());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -102,3 +102,280 @@ pub async fn ai_write_memory_file(
|
|||||||
None,
|
None,
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::ffi::OsString;
|
||||||
|
use std::sync::{Mutex, OnceLock};
|
||||||
|
|
||||||
|
use tempfile::TempDir;
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn env_mutex() -> &'static Mutex<()> {
|
||||||
|
static ENV_MUTEX: OnceLock<Mutex<()>> = OnceLock::new();
|
||||||
|
ENV_MUTEX.get_or_init(|| Mutex::new(()))
|
||||||
|
}
|
||||||
|
|
||||||
|
struct WorkspaceEnvGuard {
|
||||||
|
previous: Option<OsString>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WorkspaceEnvGuard {
|
||||||
|
fn set(path: &std::path::Path) -> Self {
|
||||||
|
let previous = std::env::var_os("OPENHUMAN_WORKSPACE");
|
||||||
|
unsafe {
|
||||||
|
std::env::set_var("OPENHUMAN_WORKSPACE", path);
|
||||||
|
}
|
||||||
|
Self { previous }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for WorkspaceEnvGuard {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
unsafe {
|
||||||
|
if let Some(previous) = self.previous.take() {
|
||||||
|
std::env::set_var("OPENHUMAN_WORKSPACE", previous);
|
||||||
|
} else {
|
||||||
|
std::env::remove_var("OPENHUMAN_WORKSPACE");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn write_read_and_list_memory_files_roundtrip() {
|
||||||
|
let _guard = env_mutex()
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||||
|
let tmp = TempDir::new().expect("tempdir");
|
||||||
|
let _workspace = WorkspaceEnvGuard::set(tmp.path());
|
||||||
|
|
||||||
|
let write = ai_write_memory_file(WriteMemoryFileRequest {
|
||||||
|
relative_path: "notes/today.md".to_string(),
|
||||||
|
content: "remember this".to_string(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("write should succeed");
|
||||||
|
let write_data = write.value.data.expect("write data");
|
||||||
|
assert_eq!(write_data.relative_path, "notes/today.md");
|
||||||
|
assert!(write_data.written);
|
||||||
|
assert_eq!(write_data.bytes_written, "remember this".len());
|
||||||
|
|
||||||
|
let read = ai_read_memory_file(ReadMemoryFileRequest {
|
||||||
|
relative_path: "notes/today.md".to_string(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("read should succeed");
|
||||||
|
let read_data = read.value.data.expect("read data");
|
||||||
|
assert_eq!(read_data.relative_path, "notes/today.md");
|
||||||
|
assert_eq!(read_data.content, "remember this");
|
||||||
|
|
||||||
|
let memory_root = super::super::helpers::resolve_existing_memory_path("")
|
||||||
|
.await
|
||||||
|
.expect("resolve memory root");
|
||||||
|
tokio::fs::write(memory_root.join("b.md"), "b")
|
||||||
|
.await
|
||||||
|
.expect("write b");
|
||||||
|
tokio::fs::write(memory_root.join("a.md"), "a")
|
||||||
|
.await
|
||||||
|
.expect("write a");
|
||||||
|
tokio::fs::create_dir_all(memory_root.join("nested"))
|
||||||
|
.await
|
||||||
|
.expect("create nested dir");
|
||||||
|
tokio::fs::write(memory_root.join("nested").join("hidden.md"), "hidden")
|
||||||
|
.await
|
||||||
|
.expect("write nested file");
|
||||||
|
|
||||||
|
let listed = ai_list_memory_files(ListMemoryFilesRequest {
|
||||||
|
relative_dir: String::new(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("list should succeed");
|
||||||
|
let listed_data = listed.value.data.expect("list data");
|
||||||
|
assert_eq!(listed_data.relative_dir, "");
|
||||||
|
assert_eq!(listed_data.files, vec!["a.md", "b.md"]);
|
||||||
|
assert_eq!(listed_data.count, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn list_memory_files_rejects_non_directory_target() {
|
||||||
|
let _guard = env_mutex()
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||||
|
let tmp = TempDir::new().expect("tempdir");
|
||||||
|
let _workspace = WorkspaceEnvGuard::set(tmp.path());
|
||||||
|
|
||||||
|
tokio::fs::create_dir_all(tmp.path().join("memory"))
|
||||||
|
.await
|
||||||
|
.expect("create memory root");
|
||||||
|
tokio::fs::write(tmp.path().join("memory").join("single.md"), "hello")
|
||||||
|
.await
|
||||||
|
.expect("write file");
|
||||||
|
|
||||||
|
let err = ai_list_memory_files(ListMemoryFilesRequest {
|
||||||
|
relative_dir: "single.md".to_string(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect_err("listing a file path should fail");
|
||||||
|
assert!(
|
||||||
|
err.contains("memory directory not found") || err.contains("resolve memory path"),
|
||||||
|
"unexpected error: {err}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn read_and_write_memory_files_reject_path_traversal() {
|
||||||
|
let _guard = env_mutex()
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||||
|
let tmp = TempDir::new().expect("tempdir");
|
||||||
|
let _workspace = WorkspaceEnvGuard::set(tmp.path());
|
||||||
|
|
||||||
|
let write_err = ai_write_memory_file(WriteMemoryFileRequest {
|
||||||
|
relative_path: "../secrets.txt".to_string(),
|
||||||
|
content: "nope".to_string(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect_err("path traversal should fail for writes");
|
||||||
|
assert!(write_err.contains("path traversal is not allowed"));
|
||||||
|
|
||||||
|
let read_err = ai_read_memory_file(ReadMemoryFileRequest {
|
||||||
|
relative_path: "../secrets.txt".to_string(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect_err("path traversal should fail for reads");
|
||||||
|
assert!(read_err.contains("path traversal is not allowed"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn list_and_read_memory_files_reject_absolute_paths() {
|
||||||
|
let _guard = env_mutex()
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||||
|
let tmp = TempDir::new().expect("tempdir");
|
||||||
|
let _workspace = WorkspaceEnvGuard::set(tmp.path());
|
||||||
|
|
||||||
|
let list_err = ai_list_memory_files(ListMemoryFilesRequest {
|
||||||
|
relative_dir: "/tmp".to_string(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect_err("absolute list path should fail");
|
||||||
|
assert!(list_err.contains("absolute paths are not allowed"));
|
||||||
|
|
||||||
|
let read_err = ai_read_memory_file(ReadMemoryFileRequest {
|
||||||
|
relative_path: "/tmp/secret.txt".to_string(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect_err("absolute read path should fail");
|
||||||
|
assert!(read_err.contains("absolute paths are not allowed"));
|
||||||
|
|
||||||
|
let write_err = ai_write_memory_file(WriteMemoryFileRequest {
|
||||||
|
relative_path: "/tmp/secret.txt".to_string(),
|
||||||
|
content: "nope".to_string(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect_err("absolute write path should fail");
|
||||||
|
assert!(write_err.contains("absolute paths are not allowed"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn read_memory_file_surfaces_missing_file_error() {
|
||||||
|
let _guard = env_mutex()
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||||
|
let tmp = TempDir::new().expect("tempdir");
|
||||||
|
let _workspace = WorkspaceEnvGuard::set(tmp.path());
|
||||||
|
|
||||||
|
let err = ai_read_memory_file(ReadMemoryFileRequest {
|
||||||
|
relative_path: "missing.md".to_string(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect_err("missing file should fail");
|
||||||
|
assert!(
|
||||||
|
err.contains("resolve memory path") || err.contains("read memory file"),
|
||||||
|
"unexpected error: {err}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn read_memory_file_surfaces_invalid_utf8_error() {
|
||||||
|
let _guard = env_mutex()
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||||
|
let tmp = TempDir::new().expect("tempdir");
|
||||||
|
let _workspace = WorkspaceEnvGuard::set(tmp.path());
|
||||||
|
|
||||||
|
let memory_root = super::super::helpers::resolve_existing_memory_path("")
|
||||||
|
.await
|
||||||
|
.expect("resolve memory root");
|
||||||
|
tokio::fs::write(memory_root.join("binary.bin"), [0xff, 0xfe, 0xfd])
|
||||||
|
.await
|
||||||
|
.expect("write invalid utf8 file");
|
||||||
|
|
||||||
|
let err = ai_read_memory_file(ReadMemoryFileRequest {
|
||||||
|
relative_path: "binary.bin".to_string(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect_err("invalid utf8 file should fail");
|
||||||
|
assert!(err.contains("read memory file"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn write_memory_file_rejects_symlink_targets() {
|
||||||
|
use std::os::unix::fs::symlink;
|
||||||
|
|
||||||
|
let _guard = env_mutex()
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||||
|
let tmp = TempDir::new().expect("tempdir");
|
||||||
|
let _workspace = WorkspaceEnvGuard::set(tmp.path());
|
||||||
|
|
||||||
|
let memory_root = super::super::helpers::resolve_existing_memory_path("")
|
||||||
|
.await
|
||||||
|
.expect("resolve memory root");
|
||||||
|
let real = memory_root.join("real.md");
|
||||||
|
tokio::fs::write(&real, "hello")
|
||||||
|
.await
|
||||||
|
.expect("write real file");
|
||||||
|
symlink(&real, memory_root.join("alias.md")).expect("create symlink");
|
||||||
|
|
||||||
|
let err = ai_write_memory_file(WriteMemoryFileRequest {
|
||||||
|
relative_path: "alias.md".to_string(),
|
||||||
|
content: "mutate".to_string(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect_err("writing through symlink should fail");
|
||||||
|
assert!(err.contains("refusing to write through symlink"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn list_memory_files_skips_symlink_entries() {
|
||||||
|
use std::os::unix::fs::symlink;
|
||||||
|
|
||||||
|
let _guard = env_mutex()
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||||
|
let tmp = TempDir::new().expect("tempdir");
|
||||||
|
let _workspace = WorkspaceEnvGuard::set(tmp.path());
|
||||||
|
|
||||||
|
let memory_root = super::super::helpers::resolve_existing_memory_path("")
|
||||||
|
.await
|
||||||
|
.expect("resolve memory root");
|
||||||
|
let real = memory_root.join("real.md");
|
||||||
|
tokio::fs::write(&real, "hello")
|
||||||
|
.await
|
||||||
|
.expect("write real file");
|
||||||
|
symlink(&real, memory_root.join("alias.md")).expect("create symlink");
|
||||||
|
|
||||||
|
let listed = ai_list_memory_files(ListMemoryFilesRequest {
|
||||||
|
relative_dir: String::new(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("list should succeed");
|
||||||
|
let listed_data = listed.value.data.expect("list data");
|
||||||
|
assert_eq!(listed_data.files, vec!["real.md"]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -9,12 +9,12 @@ use serde::Deserialize;
|
|||||||
use serde_json::{json, Value};
|
use serde_json::{json, Value};
|
||||||
|
|
||||||
use crate::openhuman::config::Config;
|
use crate::openhuman::config::Config;
|
||||||
use crate::openhuman::memory::store::GraphRelationRecord;
|
|
||||||
use crate::openhuman::memory::{
|
use crate::openhuman::memory::{
|
||||||
MemoryClient, MemoryClientRef, MemoryDocumentSummary, MemoryItemKind, MemoryRetrievalChunk,
|
MemoryClient, MemoryClientRef, MemoryDocumentSummary, MemoryItemKind, MemoryRetrievalChunk,
|
||||||
MemoryRetrievalContext, MemoryRetrievalEntity, MemoryRetrievalRelation, NamespaceMemoryHit,
|
MemoryRetrievalContext, MemoryRetrievalEntity, MemoryRetrievalRelation, NamespaceMemoryHit,
|
||||||
QueryNamespaceRequest,
|
QueryNamespaceRequest,
|
||||||
};
|
};
|
||||||
|
use crate::openhuman::memory_store::GraphRelationRecord;
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Formatting helpers
|
// Formatting helpers
|
||||||
@@ -225,6 +225,80 @@ pub(crate) fn format_llm_context_message(
|
|||||||
Some(parts.join("\n\n"))
|
Some(parts.join("\n\n"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::openhuman::memory::RetrievalScoreBreakdown;
|
||||||
|
|
||||||
|
fn sample_hit(kind: MemoryItemKind) -> NamespaceMemoryHit {
|
||||||
|
NamespaceMemoryHit {
|
||||||
|
id: "hit-1".into(),
|
||||||
|
kind,
|
||||||
|
namespace: "global".into(),
|
||||||
|
key: "note-1".into(),
|
||||||
|
title: Some("Title".into()),
|
||||||
|
content: "Body text".into(),
|
||||||
|
category: "core".into(),
|
||||||
|
source_type: Some("manual".into()),
|
||||||
|
updated_at: 1.5,
|
||||||
|
score: 0.7,
|
||||||
|
score_breakdown: RetrievalScoreBreakdown::default(),
|
||||||
|
document_id: Some("doc-1".into()),
|
||||||
|
chunk_id: Some("chunk-1".into()),
|
||||||
|
supporting_relations: vec![GraphRelationRecord {
|
||||||
|
namespace: Some("global".into()),
|
||||||
|
subject: "Alice".into(),
|
||||||
|
predicate: "OWNS".into(),
|
||||||
|
object: "OpenHuman".into(),
|
||||||
|
attrs: json!({"entity_types": {"subject": "PERSON", "object": "PRODUCT"}}),
|
||||||
|
updated_at: 2.0,
|
||||||
|
evidence_count: 1,
|
||||||
|
order_index: Some(0),
|
||||||
|
document_ids: vec!["doc-1".into()],
|
||||||
|
chunk_ids: vec!["chunk-1".into()],
|
||||||
|
}],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn timestamp_to_rfc3339_rejects_invalid_values() {
|
||||||
|
assert!(timestamp_to_rfc3339(f64::NAN).is_none());
|
||||||
|
assert!(timestamp_to_rfc3339(f64::INFINITY).is_none());
|
||||||
|
assert!(timestamp_to_rfc3339(-1.0).is_none());
|
||||||
|
assert!(timestamp_to_rfc3339(1.5).is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn relation_identity_and_metadata_include_namespace_and_attrs() {
|
||||||
|
let relation = sample_hit(MemoryItemKind::Document)
|
||||||
|
.supporting_relations
|
||||||
|
.remove(0);
|
||||||
|
assert_eq!(relation_identity(&relation), "global|Alice|OWNS|OpenHuman");
|
||||||
|
let meta = relation_metadata(&relation);
|
||||||
|
assert_eq!(meta["namespace"], "global");
|
||||||
|
assert_eq!(meta["attrs"]["entity_types"]["subject"], "PERSON");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn build_retrieval_context_deduplicates_relations_and_entities() {
|
||||||
|
let hit = sample_hit(MemoryItemKind::Document);
|
||||||
|
let ctx = build_retrieval_context(&[hit.clone(), hit]);
|
||||||
|
assert_eq!(ctx.chunks.len(), 2);
|
||||||
|
assert_eq!(ctx.relations.len(), 1);
|
||||||
|
assert!(ctx.entities.iter().any(|e| e.name == "Alice"));
|
||||||
|
assert!(ctx.entities.iter().any(|e| e.name == "OpenHuman"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn format_llm_context_message_includes_query_and_relation_text() {
|
||||||
|
let hit = sample_hit(MemoryItemKind::Document);
|
||||||
|
let text = format_llm_context_message(Some("who owns it"), &[hit]).unwrap();
|
||||||
|
assert!(text.contains("Query: who owns it"));
|
||||||
|
assert!(text.contains("Title: Body text"));
|
||||||
|
assert!(text.contains("Alice (PERSON) -[OWNS]-> OpenHuman (PRODUCT)"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Filters memory hits to only include those matching specific document IDs.
|
/// Filters memory hits to only include those matching specific document IDs.
|
||||||
pub(crate) fn filter_hits_by_document_ids(
|
pub(crate) fn filter_hits_by_document_ids(
|
||||||
hits: Vec<NamespaceMemoryHit>,
|
hits: Vec<NamespaceMemoryHit>,
|
||||||
|
|||||||
@@ -134,3 +134,111 @@ pub async fn graph_query(
|
|||||||
.await?;
|
.await?;
|
||||||
Ok(RpcOutcome::single_log(rows, "memory graph queried"))
|
Ok(RpcOutcome::single_log(rows, "memory graph queried"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::sync::OnceLock;
|
||||||
|
|
||||||
|
use tempfile::TempDir;
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn ensure_memory_client() {
|
||||||
|
static WORKSPACE: OnceLock<PathBuf> = OnceLock::new();
|
||||||
|
let workspace = WORKSPACE.get_or_init(|| {
|
||||||
|
let tmp = TempDir::new().expect("tempdir");
|
||||||
|
let path = tmp.path().join("workspace");
|
||||||
|
std::fs::create_dir_all(&path).expect("workspace dir");
|
||||||
|
std::mem::forget(tmp);
|
||||||
|
path
|
||||||
|
});
|
||||||
|
let _ = crate::openhuman::memory::global::init(workspace.clone());
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unique_namespace(prefix: &str) -> String {
|
||||||
|
format!("{prefix}-{}", uuid::Uuid::new_v4())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn kv_handlers_roundtrip_scoped_values() {
|
||||||
|
ensure_memory_client();
|
||||||
|
let namespace = unique_namespace("kv-graph-kv");
|
||||||
|
let key = format!("state-{}", uuid::Uuid::new_v4());
|
||||||
|
|
||||||
|
let set = kv_set(KvSetParams {
|
||||||
|
namespace: Some(namespace.clone()),
|
||||||
|
key: key.clone(),
|
||||||
|
value: serde_json::json!({"open": true}),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("kv set");
|
||||||
|
assert!(set.value);
|
||||||
|
|
||||||
|
let get = kv_get(KvGetDeleteParams {
|
||||||
|
namespace: Some(namespace.clone()),
|
||||||
|
key: key.clone(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("kv get");
|
||||||
|
assert_eq!(get.value, Some(serde_json::json!({"open": true})));
|
||||||
|
|
||||||
|
let listed = kv_list_namespace(super::super::documents::NamespaceOnlyParams {
|
||||||
|
namespace: namespace.clone(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("kv list namespace");
|
||||||
|
assert!(listed
|
||||||
|
.value
|
||||||
|
.iter()
|
||||||
|
.any(|row| row["key"] == key && row["value"] == serde_json::json!({"open": true})));
|
||||||
|
|
||||||
|
let deleted = kv_delete(KvGetDeleteParams {
|
||||||
|
namespace: Some(namespace.clone()),
|
||||||
|
key: key.clone(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("kv delete");
|
||||||
|
assert!(deleted.value);
|
||||||
|
|
||||||
|
let after = kv_get(KvGetDeleteParams {
|
||||||
|
namespace: Some(namespace),
|
||||||
|
key,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("kv get after delete");
|
||||||
|
assert!(after.value.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn graph_handlers_roundtrip_relation_rows() {
|
||||||
|
ensure_memory_client();
|
||||||
|
let namespace = unique_namespace("kv-graph-rel");
|
||||||
|
let subject = format!("alice-{}", uuid::Uuid::new_v4());
|
||||||
|
|
||||||
|
let upsert = graph_upsert(GraphUpsertParams {
|
||||||
|
namespace: Some(namespace.clone()),
|
||||||
|
subject: subject.clone(),
|
||||||
|
predicate: "OWNS".into(),
|
||||||
|
object: "Atlas".into(),
|
||||||
|
attrs: serde_json::json!({"source": "test", "confidence": 0.9}),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("graph upsert");
|
||||||
|
assert!(upsert.value);
|
||||||
|
|
||||||
|
let queried = graph_query(GraphQueryParams {
|
||||||
|
namespace: Some(namespace),
|
||||||
|
subject: Some(subject.clone()),
|
||||||
|
predicate: Some("OWNS".into()),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("graph query");
|
||||||
|
|
||||||
|
assert_eq!(queried.logs, vec!["memory graph queried".to_string()]);
|
||||||
|
assert_eq!(queried.value.len(), 1);
|
||||||
|
assert_eq!(queried.value[0]["subject"], subject.to_uppercase());
|
||||||
|
assert_eq!(queried.value[0]["predicate"], "OWNS");
|
||||||
|
assert_eq!(queried.value[0]["object"], "ATLAS");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -111,9 +111,10 @@ pub async fn memory_learn_all(
|
|||||||
"[memory.learn] running summarization for namespace='{}'",
|
"[memory.learn] running summarization for namespace='{}'",
|
||||||
namespace
|
namespace
|
||||||
);
|
);
|
||||||
let outcome =
|
let outcome = crate::openhuman::memory_tree::tree_runtime::ops::tree_summarizer_run(
|
||||||
crate::openhuman::memory_tree::summarizer::ops::tree_summarizer_run(&config, namespace)
|
&config, namespace,
|
||||||
.await;
|
)
|
||||||
|
.await;
|
||||||
match outcome {
|
match outcome {
|
||||||
Ok(_) => {
|
Ok(_) => {
|
||||||
tracing::info!("[memory.learn] namespace='{}' ok", namespace);
|
tracing::info!("[memory.learn] namespace='{}' ok", namespace);
|
||||||
@@ -151,3 +152,192 @@ pub async fn memory_learn_all(
|
|||||||
vec![],
|
vec![],
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::ffi::OsString;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::sync::OnceLock;
|
||||||
|
|
||||||
|
use serde_json::json;
|
||||||
|
use tempfile::TempDir;
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::openhuman::memory_store::NamespaceDocumentInput;
|
||||||
|
|
||||||
|
fn ensure_memory_client() {
|
||||||
|
static WORKSPACE: OnceLock<PathBuf> = OnceLock::new();
|
||||||
|
let workspace = WORKSPACE.get_or_init(|| {
|
||||||
|
let tmp = TempDir::new().expect("tempdir");
|
||||||
|
let path = tmp.path().join("workspace");
|
||||||
|
std::fs::create_dir_all(&path).expect("workspace dir");
|
||||||
|
std::mem::forget(tmp);
|
||||||
|
path
|
||||||
|
});
|
||||||
|
let _ = crate::openhuman::memory::global::init(workspace.clone());
|
||||||
|
}
|
||||||
|
|
||||||
|
struct WorkspaceEnvGuard {
|
||||||
|
_lock: std::sync::MutexGuard<'static, ()>,
|
||||||
|
previous: Option<OsString>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WorkspaceEnvGuard {
|
||||||
|
fn set(path: &std::path::Path) -> Self {
|
||||||
|
let lock = crate::openhuman::config::TEST_ENV_LOCK.lock().unwrap();
|
||||||
|
let previous = std::env::var_os("OPENHUMAN_WORKSPACE");
|
||||||
|
std::env::set_var("OPENHUMAN_WORKSPACE", path);
|
||||||
|
Self {
|
||||||
|
_lock: lock,
|
||||||
|
previous,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for WorkspaceEnvGuard {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if let Some(previous) = self.previous.as_ref() {
|
||||||
|
std::env::set_var("OPENHUMAN_WORKSPACE", previous);
|
||||||
|
} else {
|
||||||
|
std::env::remove_var("OPENHUMAN_WORKSPACE");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn seed_namespace(prefix: &str) -> String {
|
||||||
|
ensure_memory_client();
|
||||||
|
let namespace = format!("{prefix}-{}", uuid::Uuid::new_v4());
|
||||||
|
let client = crate::openhuman::memory::global::client().expect("memory client");
|
||||||
|
client
|
||||||
|
.put_doc_light(NamespaceDocumentInput {
|
||||||
|
namespace: namespace.clone(),
|
||||||
|
key: format!("key-{}", uuid::Uuid::new_v4()),
|
||||||
|
title: "Test".into(),
|
||||||
|
content: "Seed content".into(),
|
||||||
|
source_type: "doc".into(),
|
||||||
|
priority: "normal".into(),
|
||||||
|
tags: vec!["test".into()],
|
||||||
|
metadata: json!({"source": "test"}),
|
||||||
|
category: "core".into(),
|
||||||
|
session_id: None,
|
||||||
|
document_id: None,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("seed namespace doc");
|
||||||
|
namespace
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn write_config_with_runtime_enabled(
|
||||||
|
workspace_root: &std::path::Path,
|
||||||
|
runtime_enabled: bool,
|
||||||
|
) {
|
||||||
|
let _guard = WorkspaceEnvGuard::set(workspace_root);
|
||||||
|
let mut config = crate::openhuman::config::Config::load_or_init()
|
||||||
|
.await
|
||||||
|
.expect("load config");
|
||||||
|
config.local_ai.runtime_enabled = runtime_enabled;
|
||||||
|
config.save().await.expect("save config");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn memory_learn_all_is_noop_for_explicit_empty_namespace_list() {
|
||||||
|
ensure_memory_client();
|
||||||
|
let outcome = memory_learn_all(LearnAllParams {
|
||||||
|
namespaces: Some(vec![]),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("empty list should early-return");
|
||||||
|
assert_eq!(outcome.value.namespaces_processed, 0);
|
||||||
|
assert!(outcome.value.results.is_empty());
|
||||||
|
assert!(outcome.logs.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn memory_learn_all_is_noop_when_requested_namespaces_do_not_exist() {
|
||||||
|
ensure_memory_client();
|
||||||
|
let missing = format!("missing-{}", uuid::Uuid::new_v4());
|
||||||
|
let outcome = memory_learn_all(LearnAllParams {
|
||||||
|
namespaces: Some(vec![missing]),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("unknown namespaces should filter to no-op");
|
||||||
|
assert_eq!(outcome.value.namespaces_processed, 0);
|
||||||
|
assert!(outcome.value.results.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn memory_learn_all_filters_missing_namespaces_and_dedupes_requested_order() {
|
||||||
|
let namespace_a = seed_namespace("memory-learn-a").await;
|
||||||
|
let namespace_b = seed_namespace("memory-learn-b").await;
|
||||||
|
let missing = format!("missing-{}", uuid::Uuid::new_v4());
|
||||||
|
let tmp = TempDir::new().expect("tempdir");
|
||||||
|
write_config_with_runtime_enabled(tmp.path(), true).await;
|
||||||
|
let _workspace = WorkspaceEnvGuard::set(tmp.path());
|
||||||
|
|
||||||
|
let outcome = memory_learn_all(LearnAllParams {
|
||||||
|
namespaces: Some(vec![
|
||||||
|
missing,
|
||||||
|
namespace_b.clone(),
|
||||||
|
namespace_a.clone(),
|
||||||
|
namespace_b.clone(),
|
||||||
|
]),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("existing namespaces with runtime enabled should run");
|
||||||
|
|
||||||
|
assert_eq!(outcome.value.namespaces_processed, 2);
|
||||||
|
assert_eq!(outcome.value.results.len(), 2);
|
||||||
|
assert_eq!(outcome.value.results[0].namespace, namespace_b);
|
||||||
|
assert_eq!(outcome.value.results[1].namespace, namespace_a);
|
||||||
|
assert!(outcome.value.results.iter().all(|r| r.status == "ok"));
|
||||||
|
assert!(outcome.value.results.iter().all(|r| r.error.is_none()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn memory_learn_all_requires_local_ai_once_existing_namespace_is_selected() {
|
||||||
|
let namespace = seed_namespace("memory-learn-runtime").await;
|
||||||
|
let tmp = TempDir::new().expect("tempdir");
|
||||||
|
write_config_with_runtime_enabled(tmp.path(), false).await;
|
||||||
|
let _workspace = WorkspaceEnvGuard::set(tmp.path());
|
||||||
|
|
||||||
|
let err = memory_learn_all(LearnAllParams {
|
||||||
|
namespaces: Some(vec![namespace]),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect_err("runtime-disabled config should hard-fail");
|
||||||
|
|
||||||
|
assert!(err.contains("memory_learn_all requires local_ai.runtime_enabled=true"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn memory_learn_all_uses_all_namespaces_when_none_is_requested() {
|
||||||
|
let namespace_a = seed_namespace("memory-learn-all-a").await;
|
||||||
|
let namespace_b = seed_namespace("memory-learn-all-b").await;
|
||||||
|
let tmp = TempDir::new().expect("tempdir");
|
||||||
|
write_config_with_runtime_enabled(tmp.path(), true).await;
|
||||||
|
let _workspace = WorkspaceEnvGuard::set(tmp.path());
|
||||||
|
|
||||||
|
let outcome = memory_learn_all(LearnAllParams { namespaces: None })
|
||||||
|
.await
|
||||||
|
.expect("runtime-enabled config should process all namespaces");
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
outcome.value.namespaces_processed >= 2,
|
||||||
|
"expected at least the two seeded namespaces to be processed"
|
||||||
|
);
|
||||||
|
let namespaces: std::collections::BTreeSet<_> = outcome
|
||||||
|
.value
|
||||||
|
.results
|
||||||
|
.iter()
|
||||||
|
.map(|r| r.namespace.as_str())
|
||||||
|
.collect();
|
||||||
|
assert!(namespaces.contains(namespace_a.as_str()));
|
||||||
|
assert!(namespaces.contains(namespace_b.as_str()));
|
||||||
|
assert!(outcome
|
||||||
|
.value
|
||||||
|
.results
|
||||||
|
.iter()
|
||||||
|
.filter(|r| r.namespace == namespace_a || r.namespace == namespace_b)
|
||||||
|
.all(|r| r.status == "ok" && r.error.is_none()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -110,3 +110,163 @@ pub async fn memory_ingestion_status() -> Result<RpcOutcome<IngestionStatusResul
|
|||||||
vec![],
|
vec![],
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::sync::{Arc, OnceLock};
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use serde_json::json;
|
||||||
|
use tempfile::TempDir;
|
||||||
|
use tokio::sync::mpsc;
|
||||||
|
use tokio::time::{timeout, Duration};
|
||||||
|
|
||||||
|
use crate::core::event_bus::{self, DomainEvent, EventHandler};
|
||||||
|
|
||||||
|
fn test_mutex() -> &'static std::sync::Mutex<()> {
|
||||||
|
static LOCK: OnceLock<std::sync::Mutex<()>> = OnceLock::new();
|
||||||
|
LOCK.get_or_init(|| std::sync::Mutex::new(()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ensure_memory_client() -> crate::openhuman::memory::MemoryClientRef {
|
||||||
|
static WORKSPACE: OnceLock<PathBuf> = OnceLock::new();
|
||||||
|
let workspace = WORKSPACE.get_or_init(|| {
|
||||||
|
let tmp = TempDir::new().expect("tempdir");
|
||||||
|
let path = tmp.path().join("workspace");
|
||||||
|
std::fs::create_dir_all(&path).expect("workspace dir");
|
||||||
|
std::mem::forget(tmp);
|
||||||
|
path
|
||||||
|
});
|
||||||
|
crate::openhuman::memory::global::init(workspace.clone()).expect("init memory client")
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ChannelCapture {
|
||||||
|
tx: mpsc::UnboundedSender<Option<String>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl EventHandler for ChannelCapture {
|
||||||
|
fn name(&self) -> &str {
|
||||||
|
"memory::ops::sync::tests::capture"
|
||||||
|
}
|
||||||
|
|
||||||
|
fn domains(&self) -> Option<&[&str]> {
|
||||||
|
Some(&["memory"])
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle(&self, event: &DomainEvent) {
|
||||||
|
if let DomainEvent::MemorySyncRequested { channel_id } = event {
|
||||||
|
let _ = self.tx.send(channel_id.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sync_channel_params_deserialize_channel_id() {
|
||||||
|
let params: SyncChannelParams =
|
||||||
|
serde_json::from_value(json!({"channel_id": "channel-1"})).unwrap();
|
||||||
|
assert_eq!(params.channel_id, "channel-1");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ingestion_status_result_default_is_idle() {
|
||||||
|
let status = IngestionStatusResult::default();
|
||||||
|
assert!(!status.running);
|
||||||
|
assert!(status.current_document_id.is_none());
|
||||||
|
assert!(status.current_title.is_none());
|
||||||
|
assert!(status.current_namespace.is_none());
|
||||||
|
assert_eq!(status.queue_depth, 0);
|
||||||
|
assert!(status.last_completed_at.is_none());
|
||||||
|
assert!(status.last_document_id.is_none());
|
||||||
|
assert!(status.last_success.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sync_result_structs_serialize_expected_fields() {
|
||||||
|
let one = serde_json::to_value(SyncChannelResult {
|
||||||
|
requested: true,
|
||||||
|
channel_id: "abc".into(),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(one, json!({"requested": true, "channel_id": "abc"}));
|
||||||
|
|
||||||
|
let all = serde_json::to_value(SyncAllResult { requested: true }).unwrap();
|
||||||
|
assert_eq!(all, json!({"requested": true}));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn memory_sync_channel_publishes_targeted_event() {
|
||||||
|
let _guard = test_mutex()
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||||
|
event_bus::init_global(event_bus::DEFAULT_CAPACITY);
|
||||||
|
let (tx, mut rx) = mpsc::unbounded_channel();
|
||||||
|
let _subscription = event_bus::subscribe_global(Arc::new(ChannelCapture { tx }))
|
||||||
|
.expect("global bus should be initialized");
|
||||||
|
|
||||||
|
let outcome = memory_sync_channel(SyncChannelParams {
|
||||||
|
channel_id: "channel-123".into(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("memory_sync_channel");
|
||||||
|
assert!(outcome.value.requested);
|
||||||
|
assert_eq!(outcome.value.channel_id, "channel-123");
|
||||||
|
|
||||||
|
let received = timeout(Duration::from_secs(1), rx.recv())
|
||||||
|
.await
|
||||||
|
.expect("event should arrive before timeout")
|
||||||
|
.expect("sender should still be connected");
|
||||||
|
assert_eq!(received.as_deref(), Some("channel-123"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn memory_sync_all_publishes_broadcast_event() {
|
||||||
|
let _guard = test_mutex()
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||||
|
event_bus::init_global(event_bus::DEFAULT_CAPACITY);
|
||||||
|
let (tx, mut rx) = mpsc::unbounded_channel();
|
||||||
|
let _subscription = event_bus::subscribe_global(Arc::new(ChannelCapture { tx }))
|
||||||
|
.expect("global bus should be initialized");
|
||||||
|
|
||||||
|
let outcome = memory_sync_all().await.expect("memory_sync_all");
|
||||||
|
assert!(outcome.value.requested);
|
||||||
|
|
||||||
|
let received = timeout(Duration::from_secs(1), rx.recv())
|
||||||
|
.await
|
||||||
|
.expect("event should arrive before timeout")
|
||||||
|
.expect("sender should still be connected");
|
||||||
|
assert!(
|
||||||
|
received.is_none(),
|
||||||
|
"sync-all should publish channel_id=None"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn memory_ingestion_status_reflects_initialized_client_snapshot() {
|
||||||
|
let _guard = test_mutex()
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||||
|
let client = ensure_memory_client();
|
||||||
|
let state = client.ingestion_state();
|
||||||
|
|
||||||
|
state.enqueue();
|
||||||
|
state.mark_running("doc-sync", "Sync Title", "sync-test");
|
||||||
|
|
||||||
|
let status = memory_ingestion_status()
|
||||||
|
.await
|
||||||
|
.expect("memory_ingestion_status")
|
||||||
|
.value;
|
||||||
|
|
||||||
|
assert!(status.running);
|
||||||
|
assert_eq!(status.current_document_id.as_deref(), Some("doc-sync"));
|
||||||
|
assert_eq!(status.current_title.as_deref(), Some("Sync Title"));
|
||||||
|
assert_eq!(status.current_namespace.as_deref(), Some("sync-test"));
|
||||||
|
assert_eq!(status.queue_depth, 1);
|
||||||
|
|
||||||
|
state.dequeue();
|
||||||
|
state.mark_completed("doc-sync", true, 12345);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -172,3 +172,162 @@ pub async fn tool_rules_json(params: ToolRuleListParams) -> Result<RpcOutcome<Va
|
|||||||
let value = store.list_rules_json(¶ms.tool_name).await?;
|
let value = store.list_rules_json(¶ms.tool_name).await?;
|
||||||
Ok(RpcOutcome::single_log(value, "tool memory rules json"))
|
Ok(RpcOutcome::single_log(value, "tool memory rules json"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::sync::OnceLock;
|
||||||
|
|
||||||
|
use tempfile::TempDir;
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::openhuman::memory::ToolMemoryPriority;
|
||||||
|
|
||||||
|
fn ensure_memory_client() {
|
||||||
|
static WORKSPACE: OnceLock<PathBuf> = OnceLock::new();
|
||||||
|
let workspace = WORKSPACE.get_or_init(|| {
|
||||||
|
let tmp = TempDir::new().expect("tempdir");
|
||||||
|
let path = tmp.path().join("workspace");
|
||||||
|
std::fs::create_dir_all(&path).expect("workspace dir");
|
||||||
|
std::mem::forget(tmp);
|
||||||
|
path
|
||||||
|
});
|
||||||
|
let _ = crate::openhuman::memory::global::init(workspace.clone());
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unique_tool_name() -> String {
|
||||||
|
format!("tool-memory-{}", uuid::Uuid::new_v4())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn tool_rule_put_get_list_and_delete_roundtrip() {
|
||||||
|
ensure_memory_client();
|
||||||
|
let tool_name = unique_tool_name();
|
||||||
|
|
||||||
|
let stored = tool_rule_put(ToolRulePutParams {
|
||||||
|
tool_name: tool_name.clone(),
|
||||||
|
rule: "Always ask before sending emails".into(),
|
||||||
|
priority: None,
|
||||||
|
source: None,
|
||||||
|
tags: vec!["safety".into()],
|
||||||
|
id: Some(" ".into()),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("tool rule put")
|
||||||
|
.value;
|
||||||
|
|
||||||
|
assert_eq!(stored.tool_name, tool_name);
|
||||||
|
assert_eq!(stored.priority, ToolMemoryPriority::Normal);
|
||||||
|
assert_eq!(
|
||||||
|
stored.source,
|
||||||
|
crate::openhuman::memory::ToolMemorySource::Programmatic
|
||||||
|
);
|
||||||
|
assert_eq!(stored.tags, vec!["safety".to_string()]);
|
||||||
|
assert!(
|
||||||
|
!stored.id.trim().is_empty(),
|
||||||
|
"blank id should be regenerated"
|
||||||
|
);
|
||||||
|
|
||||||
|
let fetched = tool_rule_get(ToolRuleRefParams {
|
||||||
|
tool_name: stored.tool_name.clone(),
|
||||||
|
id: stored.id.clone(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("tool rule get")
|
||||||
|
.value
|
||||||
|
.expect("stored rule should exist");
|
||||||
|
assert_eq!(fetched.rule, "Always ask before sending emails");
|
||||||
|
|
||||||
|
let listed = tool_rule_list(ToolRuleListParams {
|
||||||
|
tool_name: stored.tool_name.clone(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("tool rule list")
|
||||||
|
.value;
|
||||||
|
assert_eq!(listed.len(), 1);
|
||||||
|
assert_eq!(listed[0].id, stored.id);
|
||||||
|
|
||||||
|
let deleted = tool_rule_delete(ToolRuleRefParams {
|
||||||
|
tool_name: stored.tool_name.clone(),
|
||||||
|
id: stored.id.clone(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("tool rule delete")
|
||||||
|
.value;
|
||||||
|
assert!(deleted);
|
||||||
|
|
||||||
|
let after = tool_rule_get(ToolRuleRefParams {
|
||||||
|
tool_name: stored.tool_name,
|
||||||
|
id: stored.id,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("tool rule get after delete");
|
||||||
|
assert!(after.value.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn tool_rules_for_prompt_sorts_by_priority_and_tool_name() {
|
||||||
|
ensure_memory_client();
|
||||||
|
let primary_tool = unique_tool_name();
|
||||||
|
let secondary_tool = unique_tool_name();
|
||||||
|
|
||||||
|
let high = tool_rule_put(ToolRulePutParams {
|
||||||
|
tool_name: primary_tool.clone(),
|
||||||
|
rule: "Use the dry-run mode first".into(),
|
||||||
|
priority: Some(ToolMemoryPriority::High),
|
||||||
|
source: None,
|
||||||
|
tags: vec![],
|
||||||
|
id: None,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("put high")
|
||||||
|
.value;
|
||||||
|
let normal = tool_rule_put(ToolRulePutParams {
|
||||||
|
tool_name: secondary_tool.clone(),
|
||||||
|
rule: "Log the final command".into(),
|
||||||
|
priority: Some(ToolMemoryPriority::Normal),
|
||||||
|
source: None,
|
||||||
|
tags: vec![],
|
||||||
|
id: None,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("put normal")
|
||||||
|
.value;
|
||||||
|
|
||||||
|
let prompt = tool_rules_for_prompt(ToolRulesForPromptParams {
|
||||||
|
tools: vec![secondary_tool.clone(), primary_tool.clone()],
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("rules for prompt")
|
||||||
|
.value;
|
||||||
|
|
||||||
|
assert_eq!(prompt.rules.len(), 1, "only eager rules should be included");
|
||||||
|
assert_eq!(prompt.rules[0].id, high.id);
|
||||||
|
assert!(prompt.rendered.contains(&primary_tool));
|
||||||
|
assert!(prompt.rendered.contains("Use the dry-run mode first"));
|
||||||
|
|
||||||
|
let json_rules = tool_rules_json(ToolRuleListParams {
|
||||||
|
tool_name: secondary_tool.clone(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("tool rules json")
|
||||||
|
.value;
|
||||||
|
assert!(json_rules.is_array(), "tool rules json should be an array");
|
||||||
|
assert!(json_rules
|
||||||
|
.as_array()
|
||||||
|
.expect("array")
|
||||||
|
.iter()
|
||||||
|
.any(|row| row["rule"] == "Log the final command"));
|
||||||
|
|
||||||
|
let _ = tool_rule_delete(ToolRuleRefParams {
|
||||||
|
tool_name: primary_tool,
|
||||||
|
id: high.id,
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
let _ = tool_rule_delete(ToolRuleRefParams {
|
||||||
|
tool_name: secondary_tool,
|
||||||
|
id: normal.id,
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,8 +4,8 @@
|
|||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
|
||||||
use super::{build_retrieval_context, filter_hits_by_document_ids, format_llm_context_message};
|
use super::{build_retrieval_context, filter_hits_by_document_ids, format_llm_context_message};
|
||||||
use crate::openhuman::memory::store::GraphRelationRecord;
|
|
||||||
use crate::openhuman::memory::{MemoryItemKind, NamespaceMemoryHit, RetrievalScoreBreakdown};
|
use crate::openhuman::memory::{MemoryItemKind, NamespaceMemoryHit, RetrievalScoreBreakdown};
|
||||||
|
use crate::openhuman::memory_store::GraphRelationRecord;
|
||||||
|
|
||||||
fn sample_hit() -> NamespaceMemoryHit {
|
fn sample_hit() -> NamespaceMemoryHit {
|
||||||
NamespaceMemoryHit {
|
NamespaceMemoryHit {
|
||||||
|
|||||||
@@ -31,11 +31,11 @@ use schemars::JsonSchema;
|
|||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use crate::openhuman::config::Config;
|
use crate::openhuman::config::Config;
|
||||||
use crate::openhuman::memory_tree::content_store::read as content_read;
|
use crate::openhuman::memory::retrieval::types::NodeKind;
|
||||||
use crate::openhuman::memory_tree::retrieval::types::NodeKind;
|
use crate::openhuman::memory::score::store as score_store;
|
||||||
use crate::openhuman::memory_tree::score::store as score_store;
|
use crate::openhuman::memory_store::chunks::store::{self as chunk_store, with_connection};
|
||||||
use crate::openhuman::memory_tree::store::{self as chunk_store, with_connection};
|
use crate::openhuman::memory_store::chunks::types::SourceKind;
|
||||||
use crate::openhuman::memory_tree::types::SourceKind;
|
use crate::openhuman::memory_store::content::read as content_read;
|
||||||
use crate::rpc::RpcOutcome;
|
use crate::rpc::RpcOutcome;
|
||||||
|
|
||||||
const PREVIEW_MAX_CHARS: usize = 500;
|
const PREVIEW_MAX_CHARS: usize = 500;
|
||||||
@@ -46,7 +46,7 @@ const MAX_LIST_LIMIT: u32 = 1_000;
|
|||||||
|
|
||||||
/// Wire-shape chunk returned by the read RPCs.
|
/// Wire-shape chunk returned by the read RPCs.
|
||||||
///
|
///
|
||||||
/// Distinct from [`crate::openhuman::memory_tree::types::Chunk`] in two
|
/// Distinct from [`crate::openhuman::memory_store::chunks::types::Chunk`] in two
|
||||||
/// ways: serialised timestamps are ms-since-epoch (matches the rest of the
|
/// ways: serialised timestamps are ms-since-epoch (matches the rest of the
|
||||||
/// JSON-RPC surface) and the body is replaced with a `≤500-char preview`
|
/// JSON-RPC surface) and the body is replaced with a `≤500-char preview`
|
||||||
/// + a flag indicating whether the row has an embedding. UIs needing the
|
/// + a flag indicating whether the row has an embedding. UIs needing the
|
||||||
@@ -462,7 +462,7 @@ pub async fn recall_rpc(
|
|||||||
// Reuse the source-tree retrieval path which already does cosine
|
// Reuse the source-tree retrieval path which already does cosine
|
||||||
// rerank against query embeddings. We pull more summaries than `k`
|
// rerank against query embeddings. We pull more summaries than `k`
|
||||||
// because each summary expands into multiple leaves.
|
// because each summary expands into multiple leaves.
|
||||||
let resp = crate::openhuman::memory_tree::retrieval::query_source(
|
let resp = crate::openhuman::memory::retrieval::query_source(
|
||||||
config,
|
config,
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
@@ -780,7 +780,7 @@ pub async fn chunk_score_rpc(
|
|||||||
ScoreBreakdown {
|
ScoreBreakdown {
|
||||||
signals,
|
signals,
|
||||||
total: r.total,
|
total: r.total,
|
||||||
threshold: crate::openhuman::memory_tree::score::DEFAULT_DROP_THRESHOLD,
|
threshold: crate::openhuman::memory::score::DEFAULT_DROP_THRESHOLD,
|
||||||
kept: !r.dropped,
|
kept: !r.dropped,
|
||||||
llm_consulted,
|
llm_consulted,
|
||||||
}
|
}
|
||||||
@@ -843,7 +843,7 @@ pub async fn delete_chunk_rpc(
|
|||||||
if e.kind() != std::io::ErrorKind::NotFound {
|
if e.kind() != std::io::ErrorKind::NotFound {
|
||||||
log::warn!(
|
log::warn!(
|
||||||
"[memory_tree::read::delete] failed to remove chunk file path_hash={}: {e}",
|
"[memory_tree::read::delete] failed to remove chunk file path_hash={}: {e}",
|
||||||
crate::openhuman::memory_tree::util::redact::redact(&rel),
|
crate::openhuman::memory::util::redact::redact(&rel),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1001,7 +1001,7 @@ pub async fn graph_export_rpc(
|
|||||||
mode,
|
mode,
|
||||||
resp.nodes.len(),
|
resp.nodes.len(),
|
||||||
resp.edges.len(),
|
resp.edges.len(),
|
||||||
crate::openhuman::memory_tree::util::redact::redact(&resp.content_root_abs),
|
crate::openhuman::memory::util::redact::redact(&resp.content_root_abs),
|
||||||
);
|
);
|
||||||
Ok(RpcOutcome::single_log(resp, log))
|
Ok(RpcOutcome::single_log(resp, log))
|
||||||
}
|
}
|
||||||
@@ -1371,7 +1371,7 @@ pub async fn wipe_all_rpc(config: &Config) -> Result<RpcOutcome<WipeAllResponse>
|
|||||||
/// keyed under [`crate::openhuman::composio::providers::sync_state::KV_NAMESPACE`].
|
/// keyed under [`crate::openhuman::composio::providers::sync_state::KV_NAMESPACE`].
|
||||||
///
|
///
|
||||||
/// We open the SQLite file directly rather than going through
|
/// We open the SQLite file directly rather than going through
|
||||||
/// [`crate::openhuman::memory::store::client::MemoryClientRef`] so
|
/// [`crate::openhuman::memory_store::client::MemoryClientRef`] so
|
||||||
/// `wipe_all` stays a pure synchronous operation runnable from
|
/// `wipe_all` stays a pure synchronous operation runnable from
|
||||||
/// `spawn_blocking` without dragging in the full memory-store init
|
/// `spawn_blocking` without dragging in the full memory-store init
|
||||||
/// path. The `kv_namespace` table is created up-front by
|
/// path. The `kv_namespace` table is created up-front by
|
||||||
@@ -1430,8 +1430,8 @@ pub struct ResetTreeResponse {
|
|||||||
/// outside `spawn_blocking`) so the on-disk removal can use
|
/// outside `spawn_blocking`) so the on-disk removal can use
|
||||||
/// async retry without blocking the worker thread.
|
/// async retry without blocking the worker thread.
|
||||||
pub async fn reset_tree_rpc(config: &Config) -> Result<RpcOutcome<ResetTreeResponse>, String> {
|
pub async fn reset_tree_rpc(config: &Config) -> Result<RpcOutcome<ResetTreeResponse>, String> {
|
||||||
use crate::openhuman::memory_tree::jobs::store as jobs_store;
|
use crate::openhuman::memory::jobs::store as jobs_store;
|
||||||
use crate::openhuman::memory_tree::jobs::types::{ExtractChunkPayload, NewJob};
|
use crate::openhuman::memory::jobs::types::{ExtractChunkPayload, NewJob};
|
||||||
|
|
||||||
let cfg = config.clone();
|
let cfg = config.clone();
|
||||||
let (tree_rows_deleted, chunks_requeued, jobs_enqueued) =
|
let (tree_rows_deleted, chunks_requeued, jobs_enqueued) =
|
||||||
@@ -1535,7 +1535,7 @@ pub async fn reset_tree_rpc(config: &Config) -> Result<RpcOutcome<ResetTreeRespo
|
|||||||
// Wake the worker pool. Done after the on-disk cleanup so jobs don't
|
// Wake the worker pool. Done after the on-disk cleanup so jobs don't
|
||||||
// start racing against an in-progress directory removal; the small
|
// start racing against an in-progress directory removal; the small
|
||||||
// delay (at most the retry window on Windows) is acceptable.
|
// delay (at most the retry window on Windows) is acceptable.
|
||||||
crate::openhuman::memory_tree::jobs::wake_workers();
|
crate::openhuman::memory::jobs::wake_workers();
|
||||||
|
|
||||||
let resp = ResetTreeResponse {
|
let resp = ResetTreeResponse {
|
||||||
tree_rows_deleted,
|
tree_rows_deleted,
|
||||||
@@ -1579,9 +1579,9 @@ pub struct FlushNowResponse {
|
|||||||
/// where `<block>` is the current 3-hour UTC block (0..=7), so
|
/// where `<block>` is the current 3-hour UTC block (0..=7), so
|
||||||
/// spamming the button within the same window doesn't queue duplicates.
|
/// spamming the button within the same window doesn't queue duplicates.
|
||||||
pub async fn flush_now_rpc(config: &Config) -> Result<RpcOutcome<FlushNowResponse>, String> {
|
pub async fn flush_now_rpc(config: &Config) -> Result<RpcOutcome<FlushNowResponse>, String> {
|
||||||
use crate::openhuman::memory_tree::jobs::store as jobs_store;
|
use crate::openhuman::memory::jobs::store as jobs_store;
|
||||||
use crate::openhuman::memory_tree::jobs::types::{FlushStalePayload, NewJob};
|
use crate::openhuman::memory::jobs::types::{FlushStalePayload, NewJob};
|
||||||
use crate::openhuman::memory_tree::tree_source::store as tree_store;
|
use crate::openhuman::memory_tree::tree::store as tree_store;
|
||||||
|
|
||||||
let cfg = config.clone();
|
let cfg = config.clone();
|
||||||
let resp = tokio::task::spawn_blocking(move || -> Result<FlushNowResponse> {
|
let resp = tokio::task::spawn_blocking(move || -> Result<FlushNowResponse> {
|
||||||
@@ -1827,9 +1827,14 @@ fn parse_source_kind_str(s: &str) -> Option<SourceKind> {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage};
|
use crate::openhuman::composio::providers::sync_state::KV_NAMESPACE;
|
||||||
use crate::openhuman::memory_tree::ingest::ingest_chat;
|
use crate::openhuman::embeddings::NoopEmbedding;
|
||||||
|
use crate::openhuman::memory::ingest_pipeline::ingest_chat;
|
||||||
|
use crate::openhuman::memory_store::unified::UnifiedMemory;
|
||||||
|
use crate::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage};
|
||||||
use chrono::{TimeZone, Utc};
|
use chrono::{TimeZone, Utc};
|
||||||
|
use rusqlite::params;
|
||||||
|
use std::sync::Arc;
|
||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
|
|
||||||
fn test_config() -> (TempDir, Config) {
|
fn test_config() -> (TempDir, Config) {
|
||||||
@@ -1866,6 +1871,45 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn update_chunk_timestamp(cfg: &Config, chunk_id: &str, timestamp_ms: i64) {
|
||||||
|
with_connection(cfg, |conn| {
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE mem_tree_chunks
|
||||||
|
SET timestamp_ms = ?1,
|
||||||
|
time_range_start_ms = ?1,
|
||||||
|
time_range_end_ms = ?1
|
||||||
|
WHERE id = ?2",
|
||||||
|
params![timestamp_ms, chunk_id],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn insert_raw_chunk(
|
||||||
|
cfg: &Config,
|
||||||
|
id: &str,
|
||||||
|
source_kind: &str,
|
||||||
|
source_id: &str,
|
||||||
|
timestamp_ms: i64,
|
||||||
|
tags_json: &str,
|
||||||
|
content: &str,
|
||||||
|
token_count: i64,
|
||||||
|
) {
|
||||||
|
with_connection(cfg, |conn| {
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO mem_tree_chunks (
|
||||||
|
id, source_kind, source_id, source_ref, owner, timestamp_ms,
|
||||||
|
time_range_start_ms, time_range_end_ms, tags_json, content,
|
||||||
|
token_count, seq_in_source, created_at_ms, lifecycle_status, content_path
|
||||||
|
) VALUES (?1, ?2, ?3, NULL, 'tester', ?4, ?4, ?4, ?5, ?6, ?7, 0, ?4, 'seeded', NULL)",
|
||||||
|
params![id, source_kind, source_id, timestamp_ms, tags_json, content, token_count],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn list_chunks_returns_seeded_chunk() {
|
async fn list_chunks_returns_seeded_chunk() {
|
||||||
let (_tmp, cfg) = test_config();
|
let (_tmp, cfg) = test_config();
|
||||||
@@ -1912,11 +1956,143 @@ mod tests {
|
|||||||
.await
|
.await
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.value;
|
.value;
|
||||||
assert!(resp.chunks.iter().any(|c| c
|
assert!(resp.chunks.iter().any(|c| {
|
||||||
.content_preview
|
c.content_preview
|
||||||
.as_deref()
|
.as_deref()
|
||||||
.unwrap_or("")
|
.unwrap_or("")
|
||||||
.contains("phoenix")));
|
.contains("phoenix")
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn list_chunks_filters_by_source_kind_and_applies_limit_offset() {
|
||||||
|
let (_tmp, cfg) = test_config();
|
||||||
|
seed_chat_chunk(&cfg, "slack:#a", "first chat").await;
|
||||||
|
seed_chat_chunk(&cfg, "slack:#b", "second chat").await;
|
||||||
|
|
||||||
|
let filtered = list_chunks_rpc(
|
||||||
|
&cfg,
|
||||||
|
ChunkFilter {
|
||||||
|
source_kinds: Some(vec!["chat".into()]),
|
||||||
|
limit: Some(1),
|
||||||
|
offset: Some(1),
|
||||||
|
..ChunkFilter::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.value;
|
||||||
|
assert_eq!(filtered.chunks.len(), 1);
|
||||||
|
assert_eq!(filtered.total, 2);
|
||||||
|
assert!(filtered.chunks.iter().all(|c| c.source_kind == "chat"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn list_chunks_filters_by_entity_id_and_time_window() {
|
||||||
|
let (_tmp, cfg) = test_config();
|
||||||
|
seed_chat_chunk(&cfg, "slack:#eng", "alice@example.com handles phoenix").await;
|
||||||
|
seed_chat_chunk(&cfg, "slack:#eng", "bob@example.com handles atlas").await;
|
||||||
|
|
||||||
|
let seeded = list_chunks_rpc(&cfg, ChunkFilter::default())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.value
|
||||||
|
.chunks;
|
||||||
|
let alice = seeded
|
||||||
|
.iter()
|
||||||
|
.find(|chunk| {
|
||||||
|
chunk
|
||||||
|
.content_preview
|
||||||
|
.as_deref()
|
||||||
|
.unwrap_or("")
|
||||||
|
.contains("alice@example.com")
|
||||||
|
})
|
||||||
|
.expect("alice chunk present");
|
||||||
|
let bob = seeded
|
||||||
|
.iter()
|
||||||
|
.find(|chunk| {
|
||||||
|
chunk
|
||||||
|
.content_preview
|
||||||
|
.as_deref()
|
||||||
|
.unwrap_or("")
|
||||||
|
.contains("bob@example.com")
|
||||||
|
})
|
||||||
|
.expect("bob chunk present");
|
||||||
|
|
||||||
|
update_chunk_timestamp(&cfg, &alice.id, 1_700_000_000_100);
|
||||||
|
update_chunk_timestamp(&cfg, &bob.id, 1_700_000_000_900);
|
||||||
|
|
||||||
|
let filtered = list_chunks_rpc(
|
||||||
|
&cfg,
|
||||||
|
ChunkFilter {
|
||||||
|
entity_ids: Some(vec!["email:alice@example.com".into()]),
|
||||||
|
since_ms: Some(1_700_000_000_000),
|
||||||
|
until_ms: Some(1_700_000_000_500),
|
||||||
|
..ChunkFilter::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.value;
|
||||||
|
|
||||||
|
assert_eq!(filtered.total, 1);
|
||||||
|
assert_eq!(filtered.chunks.len(), 1);
|
||||||
|
assert_eq!(filtered.chunks[0].id, alice.id);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn list_chunks_ignores_empty_filter_lists_and_blank_query() {
|
||||||
|
let (_tmp, cfg) = test_config();
|
||||||
|
seed_chat_chunk(&cfg, "slack:#a", "alpha").await;
|
||||||
|
seed_chat_chunk(&cfg, "slack:#b", "beta").await;
|
||||||
|
|
||||||
|
let resp = list_chunks_rpc(
|
||||||
|
&cfg,
|
||||||
|
ChunkFilter {
|
||||||
|
source_kinds: Some(vec![]),
|
||||||
|
source_ids: Some(vec![]),
|
||||||
|
entity_ids: Some(vec![]),
|
||||||
|
query: Some(" ".into()),
|
||||||
|
limit: Some(10),
|
||||||
|
..ChunkFilter::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.value;
|
||||||
|
|
||||||
|
assert_eq!(resp.total, 2);
|
||||||
|
assert_eq!(resp.chunks.len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn list_chunks_normalizes_invalid_tags_negative_tokens_and_empty_content() {
|
||||||
|
let (_tmp, cfg) = test_config();
|
||||||
|
insert_raw_chunk(
|
||||||
|
&cfg,
|
||||||
|
"raw-empty",
|
||||||
|
"document",
|
||||||
|
"notion:page-1",
|
||||||
|
1_700_000_000_123,
|
||||||
|
"not-json",
|
||||||
|
"",
|
||||||
|
-7,
|
||||||
|
);
|
||||||
|
|
||||||
|
let resp = list_chunks_rpc(&cfg, ChunkFilter::default())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.value;
|
||||||
|
let row = resp
|
||||||
|
.chunks
|
||||||
|
.into_iter()
|
||||||
|
.find(|chunk| chunk.id == "raw-empty")
|
||||||
|
.expect("raw chunk listed");
|
||||||
|
|
||||||
|
assert_eq!(row.token_count, 0);
|
||||||
|
assert_eq!(row.tags, Vec::<String>::new());
|
||||||
|
assert_eq!(row.content_preview, None);
|
||||||
|
assert!(!row.has_embedding);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -1938,6 +2114,33 @@ mod tests {
|
|||||||
assert_eq!(b.chunk_count, 1);
|
assert_eq!(b.chunk_count, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn list_sources_formats_email_threads_with_trimmed_user_hint() {
|
||||||
|
let (_tmp, cfg) = test_config();
|
||||||
|
insert_raw_chunk(
|
||||||
|
&cfg,
|
||||||
|
"email-thread",
|
||||||
|
"email",
|
||||||
|
"gmail:Alice@Example.com|bob@example.com|carol@example.com",
|
||||||
|
1_700_000_000_123,
|
||||||
|
"[]",
|
||||||
|
"thread body",
|
||||||
|
12,
|
||||||
|
);
|
||||||
|
|
||||||
|
let sources = list_sources_rpc(&cfg, Some(" alice@example.com ".into()))
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.value;
|
||||||
|
let source = sources
|
||||||
|
.iter()
|
||||||
|
.find(|row| {
|
||||||
|
row.source_id == "gmail:Alice@Example.com|bob@example.com|carol@example.com"
|
||||||
|
})
|
||||||
|
.expect("email thread source present");
|
||||||
|
assert_eq!(source.display_name, "bob@example.com, carol@example.com");
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn entity_index_for_returns_extracted_entities() {
|
async fn entity_index_for_returns_extracted_entities() {
|
||||||
let (_tmp, cfg) = test_config();
|
let (_tmp, cfg) = test_config();
|
||||||
@@ -1956,6 +2159,25 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn chunks_for_entity_returns_leaf_chunk_ids_only() {
|
||||||
|
let (_tmp, cfg) = test_config();
|
||||||
|
seed_chat_chunk(&cfg, "slack:#eng", "alice@example.com owns it").await;
|
||||||
|
let chunk_id = list_chunks_rpc(&cfg, ChunkFilter::default())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.value
|
||||||
|
.chunks[0]
|
||||||
|
.id
|
||||||
|
.clone();
|
||||||
|
|
||||||
|
let rows = chunks_for_entity_rpc(&cfg, "email:alice@example.com".into())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.value;
|
||||||
|
assert_eq!(rows, vec![chunk_id]);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn top_entities_returns_most_frequent() {
|
async fn top_entities_returns_most_frequent() {
|
||||||
let (_tmp, cfg) = test_config();
|
let (_tmp, cfg) = test_config();
|
||||||
@@ -2030,11 +2252,74 @@ mod tests {
|
|||||||
seed_chat_chunk(&cfg, "slack:#eng", "phoenix migration scheduled friday").await;
|
seed_chat_chunk(&cfg, "slack:#eng", "phoenix migration scheduled friday").await;
|
||||||
seed_chat_chunk(&cfg, "slack:#eng", "different unrelated text").await;
|
seed_chat_chunk(&cfg, "slack:#eng", "different unrelated text").await;
|
||||||
let hits = search_rpc(&cfg, "phoenix".into(), 10).await.unwrap().value;
|
let hits = search_rpc(&cfg, "phoenix".into(), 10).await.unwrap().value;
|
||||||
assert!(hits.iter().any(|c| c
|
assert!(hits.iter().any(|c| {
|
||||||
|
c.content_preview
|
||||||
|
.as_deref()
|
||||||
|
.unwrap_or("")
|
||||||
|
.contains("phoenix")
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn read_chunk_row_returns_preview_and_metadata() {
|
||||||
|
let (_tmp, cfg) = test_config();
|
||||||
|
seed_chat_chunk(
|
||||||
|
&cfg,
|
||||||
|
"slack:#eng",
|
||||||
|
"phoenix migration scheduled friday with context and source refs",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let chunk = list_chunks_rpc(&cfg, ChunkFilter::default())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.value
|
||||||
|
.chunks
|
||||||
|
.into_iter()
|
||||||
|
.next()
|
||||||
|
.expect("seeded chunk");
|
||||||
|
|
||||||
|
let row = read_chunk_row(&cfg, &chunk.id).unwrap().expect("chunk row");
|
||||||
|
assert_eq!(row.id, chunk.id);
|
||||||
|
assert_eq!(row.source_kind, "chat");
|
||||||
|
assert_eq!(row.source_id, "slack:#eng");
|
||||||
|
assert_eq!(row.source_ref.as_deref(), Some("slack://x"));
|
||||||
|
assert_eq!(row.owner, "alice");
|
||||||
|
assert_eq!(row.lifecycle_status, "pending_extraction");
|
||||||
|
assert!(row.content_path.is_some());
|
||||||
|
assert!(row
|
||||||
.content_preview
|
.content_preview
|
||||||
.as_deref()
|
.as_deref()
|
||||||
.unwrap_or("")
|
.unwrap_or("")
|
||||||
.contains("phoenix")));
|
.contains("phoenix migration scheduled friday"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn read_chunk_row_falls_back_to_sqlite_preview_when_file_missing() {
|
||||||
|
let (_tmp, cfg) = test_config();
|
||||||
|
let body = "sqlite preview survives missing file";
|
||||||
|
seed_chat_chunk(&cfg, "slack:#eng", body).await;
|
||||||
|
let chunk = list_chunks_rpc(&cfg, ChunkFilter::default())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.value
|
||||||
|
.chunks
|
||||||
|
.into_iter()
|
||||||
|
.next()
|
||||||
|
.expect("seeded chunk");
|
||||||
|
|
||||||
|
let rel_path = chunk.content_path.clone().expect("content path present");
|
||||||
|
let abs_path = cfg.memory_tree_content_root().join(rel_path);
|
||||||
|
std::fs::remove_file(&abs_path).expect("remove chunk file");
|
||||||
|
|
||||||
|
let row = read_chunk_row(&cfg, &chunk.id).unwrap().expect("chunk row");
|
||||||
|
assert_eq!(row.content_path, chunk.content_path);
|
||||||
|
assert!(row.content_preview.as_deref().unwrap_or("").contains(body));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn read_chunk_row_returns_none_for_missing_chunk() {
|
||||||
|
let (_tmp, cfg) = test_config();
|
||||||
|
assert!(read_chunk_row(&cfg, "missing-chunk").unwrap().is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -2338,8 +2623,92 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn display_name_handles_multiple_participants_and_trimmed_hint() {
|
||||||
|
let name = display_name_for_source(
|
||||||
|
"gmail:Alice@Example.com|bob@example.com|carol@example.com",
|
||||||
|
Some(" alice@example.com "),
|
||||||
|
);
|
||||||
|
assert_eq!(name, "bob@example.com, carol@example.com");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn display_name_handles_no_prefix() {
|
fn display_name_handles_no_prefix() {
|
||||||
assert_eq!(display_name_for_source("loose-id", None), "loose-id");
|
assert_eq!(display_name_for_source("loose-id", None), "loose-id");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sanitize_basename_replaces_windows_illegal_characters() {
|
||||||
|
assert_eq!(
|
||||||
|
sanitize_basename(r#"chat:slack/#eng\name*?"<>|"#),
|
||||||
|
"chat-slack-#eng-name------"
|
||||||
|
);
|
||||||
|
assert_eq!(sanitize_basename("safe-name.md"), "safe-name.md");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_source_kind_str_accepts_known_values_only() {
|
||||||
|
assert_eq!(parse_source_kind_str("chat"), Some(SourceKind::Chat));
|
||||||
|
assert_eq!(parse_source_kind_str("email"), Some(SourceKind::Email));
|
||||||
|
assert_eq!(
|
||||||
|
parse_source_kind_str("document"),
|
||||||
|
Some(SourceKind::Document)
|
||||||
|
);
|
||||||
|
assert_eq!(parse_source_kind_str("unknown"), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn get_llm_rpc_includes_current_backend_in_value_and_log() {
|
||||||
|
let (_tmp, mut cfg) = test_config();
|
||||||
|
cfg.memory_tree.llm_backend = crate::openhuman::config::LlmBackend::Local;
|
||||||
|
let outcome = get_llm_rpc(&cfg).await.unwrap();
|
||||||
|
assert_eq!(outcome.value.current, "local");
|
||||||
|
assert_eq!(
|
||||||
|
outcome.logs,
|
||||||
|
vec!["memory_tree::read: get_llm current=local".to_string()]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn clear_composio_sync_state_removes_only_target_namespace() {
|
||||||
|
let tmp = TempDir::new().unwrap();
|
||||||
|
let _memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap();
|
||||||
|
let db_path = tmp.path().join("memory").join("memory.db");
|
||||||
|
let conn = rusqlite::Connection::open(&db_path).unwrap();
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO kv_namespace (namespace, key, value_json, updated_at)
|
||||||
|
VALUES (?1, 'cursor', '{}', 1.0)",
|
||||||
|
params![KV_NAMESPACE],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO kv_namespace (namespace, key, value_json, updated_at)
|
||||||
|
VALUES ('other-namespace', 'cursor', '{}', 2.0)",
|
||||||
|
[],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
drop(conn);
|
||||||
|
|
||||||
|
let removed = clear_composio_sync_state(&db_path).unwrap();
|
||||||
|
assert_eq!(removed, 1);
|
||||||
|
|
||||||
|
let conn = rusqlite::Connection::open(&db_path).unwrap();
|
||||||
|
let composio_count: i64 = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT COUNT(*) FROM kv_namespace WHERE namespace = ?1",
|
||||||
|
params![KV_NAMESPACE],
|
||||||
|
|row| row.get(0),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let other_count: i64 = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT COUNT(*) FROM kv_namespace WHERE namespace = 'other-namespace'",
|
||||||
|
[],
|
||||||
|
|row| row.get(0),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(composio_count, 0);
|
||||||
|
assert_eq!(other_count, 1);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
+5
-5
@@ -21,13 +21,13 @@ use chrono::{TimeZone, Utc};
|
|||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
|
|
||||||
use crate::openhuman::config::Config;
|
use crate::openhuman::config::Config;
|
||||||
use crate::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage};
|
use crate::openhuman::memory::ingest_pipeline::ingest_chat;
|
||||||
use crate::openhuman::memory_tree::ingest::ingest_chat;
|
use crate::openhuman::memory::jobs::testing::drain_until_idle;
|
||||||
use crate::openhuman::memory_tree::jobs::testing::drain_until_idle;
|
use crate::openhuman::memory::retrieval::{
|
||||||
use crate::openhuman::memory_tree::retrieval::{
|
|
||||||
fetch_leaves, query_source, query_topic, search_entities,
|
fetch_leaves, query_source, query_topic, search_entities,
|
||||||
};
|
};
|
||||||
use crate::openhuman::memory_tree::types::SourceKind;
|
use crate::openhuman::memory_store::chunks::types::SourceKind;
|
||||||
|
use crate::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage};
|
||||||
|
|
||||||
/// Shared test config — disables embedding for deterministic inert behaviour.
|
/// Shared test config — disables embedding for deterministic inert behaviour.
|
||||||
fn bench_config() -> (TempDir, Config) {
|
fn bench_config() -> (TempDir, Config) {
|
||||||
+42
-44
@@ -23,13 +23,11 @@ use std::collections::VecDeque;
|
|||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
|
|
||||||
use crate::openhuman::config::Config;
|
use crate::openhuman::config::Config;
|
||||||
use crate::openhuman::memory_tree::content_store::read as content_read;
|
use crate::openhuman::memory::retrieval::types::{hit_from_chunk, hit_from_summary, RetrievalHit};
|
||||||
use crate::openhuman::memory_tree::retrieval::types::{
|
use crate::openhuman::memory::score::embed::{build_embedder_from_config, cosine_similarity};
|
||||||
hit_from_chunk, hit_from_summary, RetrievalHit,
|
use crate::openhuman::memory_store::chunks::store::{get_chunk, get_chunk_embedding};
|
||||||
};
|
use crate::openhuman::memory_store::content::read as content_read;
|
||||||
use crate::openhuman::memory_tree::score::embed::{build_embedder_from_config, cosine_similarity};
|
use crate::openhuman::memory_tree::tree::store;
|
||||||
use crate::openhuman::memory_tree::store::{get_chunk, get_chunk_embedding};
|
|
||||||
use crate::openhuman::memory_tree::tree_source::store;
|
|
||||||
|
|
||||||
/// Walk the summary hierarchy down one step (or more if `max_depth > 1`)
|
/// Walk the summary hierarchy down one step (or more if `max_depth > 1`)
|
||||||
/// and return the hydrated child hits. Children at level 1 are raw chunks;
|
/// and return the hydrated child hits. Children at level 1 are raw chunks;
|
||||||
@@ -257,16 +255,17 @@ fn walk_with_embeddings(
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::openhuman::memory_tree::content_store;
|
use crate::openhuman::memory::chat::{test_override, ChatProvider, StaticChatProvider};
|
||||||
use crate::openhuman::memory_tree::store::upsert_chunks;
|
use crate::openhuman::memory_store::chunks::store::upsert_chunks;
|
||||||
use crate::openhuman::memory_tree::tree_source::bucket_seal::{
|
use crate::openhuman::memory_store::chunks::types::{
|
||||||
append_leaf, LabelStrategy, LeafRef,
|
chunk_id, Chunk, Metadata, SourceKind, SourceRef,
|
||||||
};
|
};
|
||||||
use crate::openhuman::memory_tree::tree_source::registry::get_or_create_source_tree;
|
use crate::openhuman::memory_store::content as content_store;
|
||||||
use crate::openhuman::memory_tree::tree_source::summariser::inert::InertSummariser;
|
use crate::openhuman::memory_store::trees::types::TreeKind;
|
||||||
use crate::openhuman::memory_tree::tree_source::types::TreeKind;
|
use crate::openhuman::memory_tree::sources::registry::get_or_create_source_tree;
|
||||||
use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef};
|
use crate::openhuman::memory_tree::tree::bucket_seal::{append_leaf, LabelStrategy, LeafRef};
|
||||||
use chrono::Utc;
|
use chrono::Utc;
|
||||||
|
use std::sync::Arc;
|
||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
|
|
||||||
fn test_config() -> (TempDir, Config) {
|
fn test_config() -> (TempDir, Config) {
|
||||||
@@ -284,7 +283,8 @@ mod tests {
|
|||||||
// Seed two 6k-token leaves so the L0 buffer seals into an L1 node.
|
// Seed two 6k-token leaves so the L0 buffer seals into an L1 node.
|
||||||
let ts = Utc::now();
|
let ts = Utc::now();
|
||||||
let tree = get_or_create_source_tree(cfg, "slack:#eng").unwrap();
|
let tree = get_or_create_source_tree(cfg, "slack:#eng").unwrap();
|
||||||
let summariser = InertSummariser::new();
|
let provider: Arc<dyn ChatProvider> =
|
||||||
|
Arc::new(StaticChatProvider::new("test summary content"));
|
||||||
let content_root = cfg.memory_tree_content_root();
|
let content_root = cfg.memory_tree_content_root();
|
||||||
std::fs::create_dir_all(&content_root).unwrap();
|
std::fs::create_dir_all(&content_root).unwrap();
|
||||||
let mut leaf_ids: Vec<String> = Vec::new();
|
let mut leaf_ids: Vec<String> = Vec::new();
|
||||||
@@ -301,8 +301,7 @@ mod tests {
|
|||||||
tags: vec![],
|
tags: vec![],
|
||||||
source_ref: Some(SourceRef::new("slack://x")),
|
source_ref: Some(SourceRef::new("slack://x")),
|
||||||
},
|
},
|
||||||
token_count: crate::openhuman::memory_tree::tree_source::types::INPUT_TOKEN_BUDGET
|
token_count: crate::openhuman::memory_store::trees::types::INPUT_TOKEN_BUDGET * 6
|
||||||
* 6
|
|
||||||
/ 10,
|
/ 10,
|
||||||
seq_in_source: seq,
|
seq_in_source: seq,
|
||||||
created_at: ts,
|
created_at: ts,
|
||||||
@@ -312,33 +311,32 @@ mod tests {
|
|||||||
// Stage to disk so `hydrate_leaf_inputs` can read the full body
|
// Stage to disk so `hydrate_leaf_inputs` can read the full body
|
||||||
// via `read_chunk_body` during the seal triggered by `append_leaf`.
|
// via `read_chunk_body` during the seal triggered by `append_leaf`.
|
||||||
let staged = content_store::stage_chunks(&content_root, &[c.clone()]).unwrap();
|
let staged = content_store::stage_chunks(&content_root, &[c.clone()]).unwrap();
|
||||||
crate::openhuman::memory_tree::store::with_connection(cfg, |conn| {
|
crate::openhuman::memory_store::chunks::store::with_connection(cfg, |conn| {
|
||||||
let tx = conn.unchecked_transaction()?;
|
let tx = conn.unchecked_transaction()?;
|
||||||
crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?;
|
crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(
|
||||||
|
&tx, &staged,
|
||||||
|
)?;
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
.unwrap();
|
.unwrap();
|
||||||
leaf_ids.push(c.id.clone());
|
leaf_ids.push(c.id.clone());
|
||||||
append_leaf(
|
let leaf = LeafRef {
|
||||||
cfg,
|
chunk_id: c.id.clone(),
|
||||||
&tree,
|
token_count: crate::openhuman::memory_store::trees::types::INPUT_TOKEN_BUDGET * 6
|
||||||
&LeafRef {
|
/ 10,
|
||||||
chunk_id: c.id.clone(),
|
timestamp: ts,
|
||||||
token_count:
|
content: c.content.clone(),
|
||||||
crate::openhuman::memory_tree::tree_source::types::INPUT_TOKEN_BUDGET * 6
|
entities: vec![],
|
||||||
/ 10,
|
topics: vec![],
|
||||||
timestamp: ts,
|
score: 0.5,
|
||||||
content: c.content.clone(),
|
};
|
||||||
entities: vec![],
|
test_override::with_provider(Arc::clone(&provider), async {
|
||||||
topics: vec![],
|
append_leaf(cfg, &tree, &leaf, &LabelStrategy::Empty)
|
||||||
score: 0.5,
|
.await
|
||||||
},
|
.unwrap()
|
||||||
&summariser,
|
})
|
||||||
&LabelStrategy::Empty,
|
.await;
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
}
|
}
|
||||||
// Fetch the sealed L1 summary id from the tree row.
|
// Fetch the sealed L1 summary id from the tree row.
|
||||||
let refreshed = store::get_tree(cfg, &tree.id).unwrap().unwrap();
|
let refreshed = store::get_tree(cfg, &tree.id).unwrap().unwrap();
|
||||||
@@ -430,9 +428,9 @@ mod tests {
|
|||||||
// (or similar — the key invariant is that BFS returns all siblings at
|
// (or similar — the key invariant is that BFS returns all siblings at
|
||||||
// one depth before any descendant at a deeper depth).
|
// one depth before any descendant at a deeper depth).
|
||||||
|
|
||||||
use crate::openhuman::memory_tree::store::with_connection;
|
use crate::openhuman::memory_store::chunks::store::with_connection;
|
||||||
use crate::openhuman::memory_tree::tree_source::store as tree_store;
|
use crate::openhuman::memory_store::trees::types::{SummaryNode, Tree, TreeStatus};
|
||||||
use crate::openhuman::memory_tree::tree_source::types::{SummaryNode, Tree, TreeStatus};
|
use crate::openhuman::memory_tree::tree::store as tree_store;
|
||||||
|
|
||||||
/// Build a tiny 2-level tree directly via store inserts so we can
|
/// Build a tiny 2-level tree directly via store inserts so we can
|
||||||
/// assert BFS ordering without needing ~100 leaves to cascade L1→L2
|
/// assert BFS ordering without needing ~100 leaves to cascade L1→L2
|
||||||
@@ -499,9 +497,9 @@ mod tests {
|
|||||||
let content_root = cfg.memory_tree_content_root();
|
let content_root = cfg.memory_tree_content_root();
|
||||||
std::fs::create_dir_all(&content_root).unwrap();
|
std::fs::create_dir_all(&content_root).unwrap();
|
||||||
let staged = content_store::stage_chunks(&content_root, &all_leaves).unwrap();
|
let staged = content_store::stage_chunks(&content_root, &all_leaves).unwrap();
|
||||||
crate::openhuman::memory_tree::store::with_connection(cfg, |conn| {
|
crate::openhuman::memory_store::chunks::store::with_connection(cfg, |conn| {
|
||||||
let tx = conn.unchecked_transaction()?;
|
let tx = conn.unchecked_transaction()?;
|
||||||
crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?;
|
crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(&tx, &staged)?;
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
+11
-9
@@ -13,10 +13,10 @@
|
|||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
|
|
||||||
use crate::openhuman::config::Config;
|
use crate::openhuman::config::Config;
|
||||||
use crate::openhuman::memory_tree::content_store::read as content_read;
|
use crate::openhuman::memory::retrieval::types::{hit_from_chunk, RetrievalHit};
|
||||||
use crate::openhuman::memory_tree::retrieval::types::{hit_from_chunk, RetrievalHit};
|
use crate::openhuman::memory::score::store::get_score;
|
||||||
use crate::openhuman::memory_tree::score::store::get_score;
|
use crate::openhuman::memory_store::chunks::store::get_chunk;
|
||||||
use crate::openhuman::memory_tree::store::get_chunk;
|
use crate::openhuman::memory_store::content::read as content_read;
|
||||||
|
|
||||||
/// Max batch size. Callers that pass more than this get truncated with a
|
/// Max batch size. Callers that pass more than this get truncated with a
|
||||||
/// warn log — no error surface so the LLM sees a partial result.
|
/// warn log — no error surface so the LLM sees a partial result.
|
||||||
@@ -96,9 +96,11 @@ pub async fn fetch_leaves(config: &Config, chunk_ids: &[String]) -> Result<Vec<R
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::openhuman::memory_tree::content_store;
|
use crate::openhuman::memory_store::chunks::store::upsert_chunks;
|
||||||
use crate::openhuman::memory_tree::store::upsert_chunks;
|
use crate::openhuman::memory_store::chunks::types::{
|
||||||
use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef};
|
chunk_id, Chunk, Metadata, SourceKind, SourceRef,
|
||||||
|
};
|
||||||
|
use crate::openhuman::memory_store::content as content_store;
|
||||||
use chrono::{TimeZone, Utc};
|
use chrono::{TimeZone, Utc};
|
||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
|
|
||||||
@@ -107,9 +109,9 @@ mod tests {
|
|||||||
std::fs::create_dir_all(&content_root).expect("create content_root for test");
|
std::fs::create_dir_all(&content_root).expect("create content_root for test");
|
||||||
let staged = content_store::stage_chunks(&content_root, chunks)
|
let staged = content_store::stage_chunks(&content_root, chunks)
|
||||||
.expect("stage_chunks for test chunks");
|
.expect("stage_chunks for test chunks");
|
||||||
crate::openhuman::memory_tree::store::with_connection(cfg, |conn| {
|
crate::openhuman::memory_store::chunks::store::with_connection(cfg, |conn| {
|
||||||
let tx = conn.unchecked_transaction()?;
|
let tx = conn.unchecked_transaction()?;
|
||||||
crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?;
|
crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(&tx, &staged)?;
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
+44
-36
@@ -13,10 +13,10 @@ use anyhow::Result;
|
|||||||
use chrono::Duration;
|
use chrono::Duration;
|
||||||
|
|
||||||
use crate::openhuman::config::Config;
|
use crate::openhuman::config::Config;
|
||||||
use crate::openhuman::memory_tree::retrieval::types::{NodeKind, QueryResponse, RetrievalHit};
|
use crate::openhuman::memory::retrieval::types::{NodeKind, QueryResponse, RetrievalHit};
|
||||||
use crate::openhuman::memory_tree::tree_global::recap::{recap, RecapOutput};
|
use crate::openhuman::memory_store::trees::registry::get_or_create_global_tree;
|
||||||
use crate::openhuman::memory_tree::tree_global::registry::get_or_create_global_tree;
|
use crate::openhuman::memory_store::trees::types::TreeKind;
|
||||||
use crate::openhuman::memory_tree::tree_source::types::TreeKind;
|
use crate::openhuman::memory_tree::global::recap::{recap, RecapOutput};
|
||||||
|
|
||||||
/// Return the global digest for the given window in days. Always returns a
|
/// Return the global digest for the given window in days. Always returns a
|
||||||
/// [`QueryResponse`]; the response is empty if the global tree has no
|
/// [`QueryResponse`]; the response is empty if the global tree has no
|
||||||
@@ -87,16 +87,17 @@ fn recap_to_hits(recap: RecapOutput, tree_id: &str, tree_scope: &str) -> Vec<Ret
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::openhuman::memory_tree::content_store;
|
use crate::openhuman::memory::chat::{test_override, ChatProvider, StaticChatProvider};
|
||||||
use crate::openhuman::memory_tree::store::upsert_chunks;
|
use crate::openhuman::memory_store::chunks::store::upsert_chunks;
|
||||||
use crate::openhuman::memory_tree::tree_global::digest::{end_of_day_digest, DigestOutcome};
|
use crate::openhuman::memory_store::chunks::types::{
|
||||||
use crate::openhuman::memory_tree::tree_source::bucket_seal::{
|
chunk_id, Chunk, Metadata, SourceKind, SourceRef,
|
||||||
append_leaf, LabelStrategy, LeafRef,
|
|
||||||
};
|
};
|
||||||
use crate::openhuman::memory_tree::tree_source::registry::get_or_create_source_tree;
|
use crate::openhuman::memory_store::content as content_store;
|
||||||
use crate::openhuman::memory_tree::tree_source::summariser::inert::InertSummariser;
|
use crate::openhuman::memory_tree::global::digest::{end_of_day_digest, DigestOutcome};
|
||||||
use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef};
|
use crate::openhuman::memory_tree::sources::registry::get_or_create_source_tree;
|
||||||
|
use crate::openhuman::memory_tree::tree::bucket_seal::{append_leaf, LabelStrategy, LeafRef};
|
||||||
use chrono::{DateTime, Utc};
|
use chrono::{DateTime, Utc};
|
||||||
|
use std::sync::Arc;
|
||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
|
|
||||||
fn stage_test_chunks(cfg: &Config, chunks: &[Chunk]) {
|
fn stage_test_chunks(cfg: &Config, chunks: &[Chunk]) {
|
||||||
@@ -104,9 +105,9 @@ mod tests {
|
|||||||
std::fs::create_dir_all(&content_root).expect("create content_root for test");
|
std::fs::create_dir_all(&content_root).expect("create content_root for test");
|
||||||
let staged = content_store::stage_chunks(&content_root, chunks)
|
let staged = content_store::stage_chunks(&content_root, chunks)
|
||||||
.expect("stage_chunks for test chunks");
|
.expect("stage_chunks for test chunks");
|
||||||
crate::openhuman::memory_tree::store::with_connection(cfg, |conn| {
|
crate::openhuman::memory_store::chunks::store::with_connection(cfg, |conn| {
|
||||||
let tx = conn.unchecked_transaction()?;
|
let tx = conn.unchecked_transaction()?;
|
||||||
crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?;
|
crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(&tx, &staged)?;
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
@@ -125,16 +126,21 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn seed_daily_digest(cfg: &Config) {
|
async fn seed_daily_digest(cfg: &Config) {
|
||||||
let summariser = InertSummariser::new();
|
let provider: Arc<dyn ChatProvider> =
|
||||||
|
Arc::new(StaticChatProvider::new("test summary content"));
|
||||||
let day = Utc::now().date_naive();
|
let day = Utc::now().date_naive();
|
||||||
let ts = day.and_hms_opt(12, 0, 0).unwrap().and_utc();
|
let ts = day.and_hms_opt(12, 0, 0).unwrap().and_utc();
|
||||||
seed_source_for_day(cfg, "slack:#eng", ts).await;
|
seed_source_for_day(cfg, "slack:#eng", ts).await;
|
||||||
end_of_day_digest(cfg, day, &summariser).await.unwrap();
|
test_override::with_provider(provider, async {
|
||||||
|
end_of_day_digest(cfg, day).await.unwrap()
|
||||||
|
})
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn seed_source_for_day(cfg: &Config, scope: &str, ts: DateTime<Utc>) {
|
async fn seed_source_for_day(cfg: &Config, scope: &str, ts: DateTime<Utc>) {
|
||||||
let tree = get_or_create_source_tree(cfg, scope).unwrap();
|
let tree = get_or_create_source_tree(cfg, scope).unwrap();
|
||||||
let summariser = InertSummariser::new();
|
let provider: Arc<dyn ChatProvider> =
|
||||||
|
Arc::new(StaticChatProvider::new("test summary content"));
|
||||||
for seq in 0..2u32 {
|
for seq in 0..2u32 {
|
||||||
let c = Chunk {
|
let c = Chunk {
|
||||||
id: chunk_id(SourceKind::Chat, scope, seq, "test-content"),
|
id: chunk_id(SourceKind::Chat, scope, seq, "test-content"),
|
||||||
@@ -155,23 +161,21 @@ mod tests {
|
|||||||
};
|
};
|
||||||
upsert_chunks(cfg, &[c.clone()]).unwrap();
|
upsert_chunks(cfg, &[c.clone()]).unwrap();
|
||||||
stage_test_chunks(cfg, &[c.clone()]);
|
stage_test_chunks(cfg, &[c.clone()]);
|
||||||
append_leaf(
|
let leaf = LeafRef {
|
||||||
cfg,
|
chunk_id: c.id.clone(),
|
||||||
&tree,
|
token_count: 30_000,
|
||||||
&LeafRef {
|
timestamp: ts,
|
||||||
chunk_id: c.id.clone(),
|
content: c.content.clone(),
|
||||||
token_count: 30_000,
|
entities: vec![],
|
||||||
timestamp: ts,
|
topics: vec![],
|
||||||
content: c.content.clone(),
|
score: 0.5,
|
||||||
entities: vec![],
|
};
|
||||||
topics: vec![],
|
test_override::with_provider(Arc::clone(&provider), async {
|
||||||
score: 0.5,
|
append_leaf(cfg, &tree, &leaf, &LabelStrategy::Empty)
|
||||||
},
|
.await
|
||||||
&summariser,
|
.unwrap()
|
||||||
&LabelStrategy::Empty,
|
})
|
||||||
)
|
.await;
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -205,11 +209,15 @@ mod tests {
|
|||||||
// if this ever returned Skipped the rest of the suite would trivially
|
// if this ever returned Skipped the rest of the suite would trivially
|
||||||
// pass which would be misleading.
|
// pass which would be misleading.
|
||||||
let (_tmp, cfg) = test_config();
|
let (_tmp, cfg) = test_config();
|
||||||
let summariser = InertSummariser::new();
|
let provider: Arc<dyn ChatProvider> =
|
||||||
|
Arc::new(StaticChatProvider::new("test summary content"));
|
||||||
let day = Utc::now().date_naive();
|
let day = Utc::now().date_naive();
|
||||||
let ts = day.and_hms_opt(12, 0, 0).unwrap().and_utc();
|
let ts = day.and_hms_opt(12, 0, 0).unwrap().and_utc();
|
||||||
seed_source_for_day(&cfg, "slack:#eng", ts).await;
|
seed_source_for_day(&cfg, "slack:#eng", ts).await;
|
||||||
let outcome = end_of_day_digest(&cfg, day, &summariser).await.unwrap();
|
let outcome = test_override::with_provider(provider, async {
|
||||||
|
end_of_day_digest(&cfg, day).await.unwrap()
|
||||||
|
})
|
||||||
|
.await;
|
||||||
assert!(matches!(outcome, DigestOutcome::Emitted { .. }));
|
assert!(matches!(outcome, DigestOutcome::Emitted { .. }));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+33
-38
@@ -14,12 +14,12 @@ use chrono::{TimeZone, Utc};
|
|||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
|
|
||||||
use crate::openhuman::config::Config;
|
use crate::openhuman::config::Config;
|
||||||
use crate::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage};
|
use crate::openhuman::memory::ingest_pipeline::ingest_chat;
|
||||||
use crate::openhuman::memory_tree::ingest::ingest_chat;
|
use crate::openhuman::memory::retrieval::{
|
||||||
use crate::openhuman::memory_tree::retrieval::{
|
|
||||||
drill_down, fetch_leaves, query_global, query_source, query_topic, search_entities,
|
drill_down, fetch_leaves, query_global, query_source, query_topic, search_entities,
|
||||||
};
|
};
|
||||||
use crate::openhuman::memory_tree::types::SourceKind;
|
use crate::openhuman::memory_store::chunks::types::SourceKind;
|
||||||
|
use crate::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage};
|
||||||
|
|
||||||
fn test_config() -> (TempDir, Config) {
|
fn test_config() -> (TempDir, Config) {
|
||||||
let tmp = TempDir::new().unwrap();
|
let tmp = TempDir::new().unwrap();
|
||||||
@@ -124,7 +124,7 @@ async fn end_to_end_three_chat_batches() {
|
|||||||
|
|
||||||
// ── fetch_leaves: find a guaranteed leaf hit from alice's topic results
|
// ── fetch_leaves: find a guaranteed leaf hit from alice's topic results
|
||||||
// and assert that fetch_leaves hydrates it correctly.
|
// and assert that fetch_leaves hydrates it correctly.
|
||||||
use crate::openhuman::memory_tree::retrieval::types::NodeKind;
|
use crate::openhuman::memory::retrieval::types::NodeKind;
|
||||||
let leaf_hit = by_email
|
let leaf_hit = by_email
|
||||||
.hits
|
.hits
|
||||||
.iter()
|
.iter()
|
||||||
@@ -164,9 +164,9 @@ async fn topic_entity_surfaces_after_ingest() {
|
|||||||
/// handler, so the test drains the queue before inspecting.
|
/// handler, so the test drains the queue before inspecting.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn ingest_populates_chunk_embeddings() {
|
async fn ingest_populates_chunk_embeddings() {
|
||||||
use crate::openhuman::memory_tree::jobs::drain_until_idle;
|
use crate::openhuman::memory::jobs::drain_until_idle;
|
||||||
use crate::openhuman::memory_tree::score::embed::EMBEDDING_DIM;
|
use crate::openhuman::memory::score::embed::EMBEDDING_DIM;
|
||||||
use crate::openhuman::memory_tree::store::get_chunk_embedding;
|
use crate::openhuman::memory_store::chunks::store::get_chunk_embedding;
|
||||||
|
|
||||||
let (_tmp, cfg) = test_config();
|
let (_tmp, cfg) = test_config();
|
||||||
let out = ingest_chat(&cfg, "slack:#eng", "alice", vec![], chat_about_phoenix(0))
|
let out = ingest_chat(&cfg, "slack:#eng", "alice", vec![], chat_about_phoenix(0))
|
||||||
@@ -192,20 +192,21 @@ async fn ingest_populates_chunk_embeddings() {
|
|||||||
/// the seal from firing on short batches.
|
/// the seal from firing on short batches.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn seal_populates_summary_embedding() {
|
async fn seal_populates_summary_embedding() {
|
||||||
use crate::openhuman::memory_tree::content_store;
|
use crate::openhuman::memory::chat::{test_override, ChatProvider, StaticChatProvider};
|
||||||
use crate::openhuman::memory_tree::score::embed::EMBEDDING_DIM;
|
use crate::openhuman::memory::score::embed::EMBEDDING_DIM;
|
||||||
use crate::openhuman::memory_tree::store::upsert_chunks;
|
use crate::openhuman::memory_store::chunks::store::upsert_chunks;
|
||||||
use crate::openhuman::memory_tree::tree_source::bucket_seal::{
|
use crate::openhuman::memory_store::chunks::types::{
|
||||||
append_leaf, LabelStrategy, LeafRef,
|
chunk_id, Chunk, Metadata, SourceKind, SourceRef,
|
||||||
};
|
};
|
||||||
use crate::openhuman::memory_tree::tree_source::registry::get_or_create_source_tree;
|
use crate::openhuman::memory_store::content as content_store;
|
||||||
use crate::openhuman::memory_tree::tree_source::store as src_store;
|
use crate::openhuman::memory_tree::sources::registry::get_or_create_source_tree;
|
||||||
use crate::openhuman::memory_tree::tree_source::summariser::inert::InertSummariser;
|
use crate::openhuman::memory_tree::tree::bucket_seal::{append_leaf, LabelStrategy, LeafRef};
|
||||||
use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef};
|
use crate::openhuman::memory_tree::tree::store as src_store;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
let (_tmp, cfg) = test_config();
|
let (_tmp, cfg) = test_config();
|
||||||
let tree = get_or_create_source_tree(&cfg, "slack:#seal-test").unwrap();
|
let tree = get_or_create_source_tree(&cfg, "slack:#seal-test").unwrap();
|
||||||
let summariser = InertSummariser::new();
|
let provider: Arc<dyn ChatProvider> = Arc::new(StaticChatProvider::new("test summary content"));
|
||||||
let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap();
|
let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap();
|
||||||
|
|
||||||
let mk_chunk = |seq: u32, tokens: u32| Chunk {
|
let mk_chunk = |seq: u32, tokens: u32| Chunk {
|
||||||
@@ -233,9 +234,9 @@ async fn seal_populates_summary_embedding() {
|
|||||||
std::fs::create_dir_all(&content_root).expect("create content_root for test");
|
std::fs::create_dir_all(&content_root).expect("create content_root for test");
|
||||||
let staged = content_store::stage_chunks(&content_root, &[c1.clone(), c2.clone()])
|
let staged = content_store::stage_chunks(&content_root, &[c1.clone(), c2.clone()])
|
||||||
.expect("stage_chunks for test chunks");
|
.expect("stage_chunks for test chunks");
|
||||||
crate::openhuman::memory_tree::store::with_connection(&cfg, |conn| {
|
crate::openhuman::memory_store::chunks::store::with_connection(&cfg, |conn| {
|
||||||
let tx = conn.unchecked_transaction()?;
|
let tx = conn.unchecked_transaction()?;
|
||||||
crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?;
|
crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(&tx, &staged)?;
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
@@ -251,24 +252,18 @@ async fn seal_populates_summary_embedding() {
|
|||||||
topics: vec![],
|
topics: vec![],
|
||||||
score: 0.5,
|
score: 0.5,
|
||||||
};
|
};
|
||||||
append_leaf(
|
test_override::with_provider(Arc::clone(&provider), async {
|
||||||
&cfg,
|
append_leaf(&cfg, &tree, &leaf_of(&c1), &LabelStrategy::Empty)
|
||||||
&tree,
|
.await
|
||||||
&leaf_of(&c1),
|
.unwrap()
|
||||||
&summariser,
|
})
|
||||||
&LabelStrategy::Empty,
|
.await;
|
||||||
)
|
let sealed = test_override::with_provider(Arc::clone(&provider), async {
|
||||||
.await
|
append_leaf(&cfg, &tree, &leaf_of(&c2), &LabelStrategy::Empty)
|
||||||
.unwrap();
|
.await
|
||||||
let sealed = append_leaf(
|
.unwrap()
|
||||||
&cfg,
|
})
|
||||||
&tree,
|
.await;
|
||||||
&leaf_of(&c2),
|
|
||||||
&summariser,
|
|
||||||
&LabelStrategy::Empty,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(sealed.len(), 1, "expected one seal at the budget crossing");
|
assert_eq!(sealed.len(), 1, "expected one seal at the budget crossing");
|
||||||
|
|
||||||
// #1574 cutover: the seal path no longer writes the legacy
|
// #1574 cutover: the seal path no longer writes the legacy
|
||||||
@@ -8,7 +8,7 @@
|
|||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use crate::openhuman::config::Config;
|
use crate::openhuman::config::Config;
|
||||||
use crate::openhuman::memory_tree::retrieval::{
|
use crate::openhuman::memory::retrieval::{
|
||||||
drill_down::drill_down,
|
drill_down::drill_down,
|
||||||
fetch::fetch_leaves,
|
fetch::fetch_leaves,
|
||||||
global::query_global,
|
global::query_global,
|
||||||
@@ -17,8 +17,8 @@ use crate::openhuman::memory_tree::retrieval::{
|
|||||||
topic::query_topic,
|
topic::query_topic,
|
||||||
types::{EntityMatch, QueryResponse, RetrievalHit},
|
types::{EntityMatch, QueryResponse, RetrievalHit},
|
||||||
};
|
};
|
||||||
use crate::openhuman::memory_tree::score::extract::EntityKind;
|
use crate::openhuman::memory::score::extract::EntityKind;
|
||||||
use crate::openhuman::memory_tree::types::SourceKind;
|
use crate::openhuman::memory_store::chunks::types::SourceKind;
|
||||||
use crate::rpc::RpcOutcome;
|
use crate::rpc::RpcOutcome;
|
||||||
|
|
||||||
// ── query_source ──────────────────────────────────────────────────────
|
// ── query_source ──────────────────────────────────────────────────────
|
||||||
@@ -307,9 +307,9 @@ mod tests {
|
|||||||
//! initialises the schema idempotently on first access, so read-only
|
//! initialises the schema idempotently on first access, so read-only
|
||||||
//! calls return empty responses rather than erroring.
|
//! calls return empty responses rather than erroring.
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::openhuman::memory_tree::content_store;
|
use crate::openhuman::memory_store::chunks::store::upsert_chunks;
|
||||||
use crate::openhuman::memory_tree::store::upsert_chunks;
|
use crate::openhuman::memory_store::chunks::types::{chunk_id, Chunk, Metadata, SourceRef};
|
||||||
use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceRef};
|
use crate::openhuman::memory_store::content as content_store;
|
||||||
use chrono::{TimeZone, Utc};
|
use chrono::{TimeZone, Utc};
|
||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
|
|
||||||
@@ -318,9 +318,9 @@ mod tests {
|
|||||||
std::fs::create_dir_all(&content_root).expect("create content_root for test");
|
std::fs::create_dir_all(&content_root).expect("create content_root for test");
|
||||||
let staged = content_store::stage_chunks(&content_root, chunks)
|
let staged = content_store::stage_chunks(&content_root, chunks)
|
||||||
.expect("stage_chunks for test chunks");
|
.expect("stage_chunks for test chunks");
|
||||||
crate::openhuman::memory_tree::store::with_connection(cfg, |conn| {
|
crate::openhuman::memory_store::chunks::store::with_connection(cfg, |conn| {
|
||||||
let tx = conn.unchecked_transaction()?;
|
let tx = conn.unchecked_transaction()?;
|
||||||
crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?;
|
crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(&tx, &staged)?;
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
+47
-1
@@ -18,7 +18,7 @@ use serde_json::{Map, Value};
|
|||||||
use crate::core::all::{ControllerFuture, RegisteredController};
|
use crate::core::all::{ControllerFuture, RegisteredController};
|
||||||
use crate::core::{ControllerSchema, FieldSchema, TypeSchema};
|
use crate::core::{ControllerSchema, FieldSchema, TypeSchema};
|
||||||
use crate::openhuman::config::rpc as config_rpc;
|
use crate::openhuman::config::rpc as config_rpc;
|
||||||
use crate::openhuman::memory_tree::retrieval::rpc as retrieval_rpc;
|
use crate::openhuman::memory::retrieval::rpc as retrieval_rpc;
|
||||||
use crate::rpc::RpcOutcome;
|
use crate::rpc::RpcOutcome;
|
||||||
|
|
||||||
const NAMESPACE: &str = "memory_tree";
|
const NAMESPACE: &str = "memory_tree";
|
||||||
@@ -385,3 +385,49 @@ fn parse_value<T: DeserializeOwned>(v: Value) -> Result<T, String> {
|
|||||||
fn to_json<T: serde::Serialize>(outcome: RpcOutcome<T>) -> Result<Value, String> {
|
fn to_json<T: serde::Serialize>(outcome: RpcOutcome<T>) -> Result<Value, String> {
|
||||||
outcome.into_cli_compatible_json()
|
outcome.into_cli_compatible_json()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn all_controller_schemas_cover_every_registered_retrieval_function() {
|
||||||
|
let schemas = all_controller_schemas();
|
||||||
|
let functions: Vec<&str> = schemas.iter().map(|s| s.function).collect();
|
||||||
|
assert_eq!(
|
||||||
|
functions,
|
||||||
|
vec![
|
||||||
|
"query_source",
|
||||||
|
"query_global",
|
||||||
|
"query_topic",
|
||||||
|
"search_entities",
|
||||||
|
"drill_down",
|
||||||
|
"fetch_leaves",
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn registered_controllers_use_memory_tree_namespace() {
|
||||||
|
let controllers = all_registered_controllers();
|
||||||
|
assert_eq!(controllers.len(), 6);
|
||||||
|
assert!(controllers.iter().all(|c| c.schema.namespace == NAMESPACE));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_schema_returns_error_output() {
|
||||||
|
let schema = schemas("not_a_real_function");
|
||||||
|
assert_eq!(schema.namespace, NAMESPACE);
|
||||||
|
assert_eq!(schema.function, "unknown");
|
||||||
|
assert_eq!(schema.outputs.len(), 1);
|
||||||
|
assert_eq!(schema.outputs[0].name, "error");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn query_global_schema_requires_time_window_days() {
|
||||||
|
let schema = schemas("query_global");
|
||||||
|
assert_eq!(schema.inputs.len(), 1);
|
||||||
|
assert_eq!(schema.inputs[0].name, "time_window_days");
|
||||||
|
assert!(schema.inputs[0].required);
|
||||||
|
}
|
||||||
|
}
|
||||||
+5
-5
@@ -19,9 +19,9 @@ use anyhow::{Context, Result};
|
|||||||
use rusqlite::params_from_iter;
|
use rusqlite::params_from_iter;
|
||||||
|
|
||||||
use crate::openhuman::config::Config;
|
use crate::openhuman::config::Config;
|
||||||
use crate::openhuman::memory_tree::retrieval::types::EntityMatch;
|
use crate::openhuman::memory::retrieval::types::EntityMatch;
|
||||||
use crate::openhuman::memory_tree::score::extract::EntityKind;
|
use crate::openhuman::memory::score::extract::EntityKind;
|
||||||
use crate::openhuman::memory_tree::store::with_connection;
|
use crate::openhuman::memory_store::chunks::store::with_connection;
|
||||||
|
|
||||||
const DEFAULT_LIMIT: usize = 5;
|
const DEFAULT_LIMIT: usize = 5;
|
||||||
const MAX_LIMIT: usize = 100;
|
const MAX_LIMIT: usize = 100;
|
||||||
@@ -156,8 +156,8 @@ fn row_to_match(row: &rusqlite::Row<'_>) -> rusqlite::Result<EntityMatch> {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage};
|
use crate::openhuman::memory::ingest_pipeline::ingest_chat;
|
||||||
use crate::openhuman::memory_tree::ingest::ingest_chat;
|
use crate::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage};
|
||||||
use chrono::{TimeZone, Utc};
|
use chrono::{TimeZone, Utc};
|
||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
|
|
||||||
+47
-49
@@ -21,14 +21,12 @@ use anyhow::Result;
|
|||||||
use chrono::{Duration, Utc};
|
use chrono::{Duration, Utc};
|
||||||
|
|
||||||
use crate::openhuman::config::Config;
|
use crate::openhuman::config::Config;
|
||||||
use crate::openhuman::memory_tree::content_store::read as content_read;
|
use crate::openhuman::memory::retrieval::types::{hit_from_summary, QueryResponse, RetrievalHit};
|
||||||
use crate::openhuman::memory_tree::retrieval::types::{
|
use crate::openhuman::memory::score::embed::{build_embedder_from_config, cosine_similarity};
|
||||||
hit_from_summary, QueryResponse, RetrievalHit,
|
use crate::openhuman::memory_store::chunks::types::SourceKind;
|
||||||
};
|
use crate::openhuman::memory_store::content::read as content_read;
|
||||||
use crate::openhuman::memory_tree::score::embed::{build_embedder_from_config, cosine_similarity};
|
use crate::openhuman::memory_store::trees::types::{SummaryNode, Tree, TreeKind};
|
||||||
use crate::openhuman::memory_tree::tree_source::store;
|
use crate::openhuman::memory_tree::tree::store;
|
||||||
use crate::openhuman::memory_tree::tree_source::types::{SummaryNode, Tree, TreeKind};
|
|
||||||
use crate::openhuman::memory_tree::types::SourceKind;
|
|
||||||
|
|
||||||
const DEFAULT_LIMIT: usize = 10;
|
const DEFAULT_LIMIT: usize = 10;
|
||||||
|
|
||||||
@@ -306,15 +304,16 @@ fn filter_by_window(hits: Vec<RetrievalHit>, window_days: u32) -> Vec<RetrievalH
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::openhuman::memory_tree::content_store;
|
use crate::openhuman::memory::chat::{test_override, ChatProvider, StaticChatProvider};
|
||||||
use crate::openhuman::memory_tree::store::upsert_chunks;
|
use crate::openhuman::memory_store::chunks::store::upsert_chunks;
|
||||||
use crate::openhuman::memory_tree::tree_source::bucket_seal::{
|
use crate::openhuman::memory_store::chunks::types::{
|
||||||
append_leaf, LabelStrategy, LeafRef,
|
chunk_id, Chunk, Metadata, SourceKind, SourceRef,
|
||||||
};
|
};
|
||||||
use crate::openhuman::memory_tree::tree_source::registry::get_or_create_source_tree;
|
use crate::openhuman::memory_store::content as content_store;
|
||||||
use crate::openhuman::memory_tree::tree_source::summariser::inert::InertSummariser;
|
use crate::openhuman::memory_tree::sources::registry::get_or_create_source_tree;
|
||||||
use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef};
|
use crate::openhuman::memory_tree::tree::bucket_seal::{append_leaf, LabelStrategy, LeafRef};
|
||||||
use chrono::{DateTime, TimeZone};
|
use chrono::{DateTime, TimeZone};
|
||||||
|
use std::sync::Arc;
|
||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
|
|
||||||
fn test_config() -> (TempDir, Config) {
|
fn test_config() -> (TempDir, Config) {
|
||||||
@@ -330,7 +329,8 @@ mod tests {
|
|||||||
|
|
||||||
async fn seed_source(cfg: &Config, scope: &str, ts: DateTime<Utc>) {
|
async fn seed_source(cfg: &Config, scope: &str, ts: DateTime<Utc>) {
|
||||||
let tree = get_or_create_source_tree(cfg, scope).unwrap();
|
let tree = get_or_create_source_tree(cfg, scope).unwrap();
|
||||||
let summariser = InertSummariser::new();
|
let provider: Arc<dyn ChatProvider> =
|
||||||
|
Arc::new(StaticChatProvider::new("test summary content"));
|
||||||
let content_root = cfg.memory_tree_content_root();
|
let content_root = cfg.memory_tree_content_root();
|
||||||
std::fs::create_dir_all(&content_root).unwrap();
|
std::fs::create_dir_all(&content_root).unwrap();
|
||||||
for seq in 0..2u32 {
|
for seq in 0..2u32 {
|
||||||
@@ -346,8 +346,7 @@ mod tests {
|
|||||||
tags: vec!["eng".into()],
|
tags: vec!["eng".into()],
|
||||||
source_ref: Some(SourceRef::new(format!("slack://{scope}/{seq}"))),
|
source_ref: Some(SourceRef::new(format!("slack://{scope}/{seq}"))),
|
||||||
},
|
},
|
||||||
token_count: crate::openhuman::memory_tree::tree_source::types::INPUT_TOKEN_BUDGET
|
token_count: crate::openhuman::memory_store::trees::types::INPUT_TOKEN_BUDGET * 6
|
||||||
* 6
|
|
||||||
/ 10,
|
/ 10,
|
||||||
seq_in_source: seq,
|
seq_in_source: seq,
|
||||||
created_at: ts,
|
created_at: ts,
|
||||||
@@ -358,32 +357,31 @@ mod tests {
|
|||||||
// via `read_chunk_body` during the seal triggered by `append_leaf`,
|
// via `read_chunk_body` during the seal triggered by `append_leaf`,
|
||||||
// and `collect_hits_and_nodes` can read summary bodies for the API.
|
// and `collect_hits_and_nodes` can read summary bodies for the API.
|
||||||
let staged = content_store::stage_chunks(&content_root, &[c.clone()]).unwrap();
|
let staged = content_store::stage_chunks(&content_root, &[c.clone()]).unwrap();
|
||||||
crate::openhuman::memory_tree::store::with_connection(cfg, |conn| {
|
crate::openhuman::memory_store::chunks::store::with_connection(cfg, |conn| {
|
||||||
let tx = conn.unchecked_transaction()?;
|
let tx = conn.unchecked_transaction()?;
|
||||||
crate::openhuman::memory_tree::store::upsert_staged_chunks_tx(&tx, &staged)?;
|
crate::openhuman::memory_store::chunks::store::upsert_staged_chunks_tx(
|
||||||
|
&tx, &staged,
|
||||||
|
)?;
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
.unwrap();
|
.unwrap();
|
||||||
append_leaf(
|
let leaf = LeafRef {
|
||||||
cfg,
|
chunk_id: c.id.clone(),
|
||||||
&tree,
|
token_count: crate::openhuman::memory_store::trees::types::INPUT_TOKEN_BUDGET * 6
|
||||||
&LeafRef {
|
/ 10,
|
||||||
chunk_id: c.id.clone(),
|
timestamp: ts,
|
||||||
token_count:
|
content: c.content.clone(),
|
||||||
crate::openhuman::memory_tree::tree_source::types::INPUT_TOKEN_BUDGET * 6
|
entities: vec![],
|
||||||
/ 10,
|
topics: vec![],
|
||||||
timestamp: ts,
|
score: 0.5,
|
||||||
content: c.content.clone(),
|
};
|
||||||
entities: vec![],
|
test_override::with_provider(Arc::clone(&provider), async {
|
||||||
topics: vec![],
|
append_leaf(cfg, &tree, &leaf, &LabelStrategy::Empty)
|
||||||
score: 0.5,
|
.await
|
||||||
},
|
.unwrap()
|
||||||
&summariser,
|
})
|
||||||
&LabelStrategy::Empty,
|
.await;
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -527,8 +525,8 @@ mod tests {
|
|||||||
/// the ingest path writes by default.
|
/// the ingest path writes by default.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn query_reranks_by_cosine_similarity() {
|
async fn query_reranks_by_cosine_similarity() {
|
||||||
use crate::openhuman::memory_tree::score::embed::{pack_embedding, EMBEDDING_DIM};
|
use crate::openhuman::memory::score::embed::{pack_embedding, EMBEDDING_DIM};
|
||||||
use crate::openhuman::memory_tree::tree_source::store as src_store;
|
use crate::openhuman::memory_tree::tree::store as src_store;
|
||||||
|
|
||||||
let (_tmp, cfg) = test_config();
|
let (_tmp, cfg) = test_config();
|
||||||
let ts = Utc::now();
|
let ts = Utc::now();
|
||||||
@@ -548,17 +546,17 @@ mod tests {
|
|||||||
|
|
||||||
// Write directly via raw UPDATE so we replace whatever the
|
// Write directly via raw UPDATE so we replace whatever the
|
||||||
// seal-time inert embedder wrote.
|
// seal-time inert embedder wrote.
|
||||||
use crate::openhuman::memory_tree::store::with_connection;
|
use crate::openhuman::memory_store::chunks::store::with_connection;
|
||||||
let phoenix_tree = src_store::get_tree_by_scope(
|
let phoenix_tree = src_store::get_tree_by_scope(
|
||||||
&cfg,
|
&cfg,
|
||||||
crate::openhuman::memory_tree::tree_source::types::TreeKind::Source,
|
crate::openhuman::memory_store::trees::types::TreeKind::Source,
|
||||||
"slack:#phoenix",
|
"slack:#phoenix",
|
||||||
)
|
)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let unrelated_tree = src_store::get_tree_by_scope(
|
let unrelated_tree = src_store::get_tree_by_scope(
|
||||||
&cfg,
|
&cfg,
|
||||||
crate::openhuman::memory_tree::tree_source::types::TreeKind::Source,
|
crate::openhuman::memory_store::trees::types::TreeKind::Source,
|
||||||
"slack:#unrelated",
|
"slack:#unrelated",
|
||||||
)
|
)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
@@ -597,7 +595,7 @@ mod tests {
|
|||||||
// The practical test here: construct a hypothetical query
|
// The practical test here: construct a hypothetical query
|
||||||
// vector equal to phoenix_vec, then verify that running the
|
// vector equal to phoenix_vec, then verify that running the
|
||||||
// rerank helper with that vector places phoenix first.
|
// rerank helper with that vector places phoenix first.
|
||||||
use crate::openhuman::memory_tree::score::embed::cosine_similarity;
|
use crate::openhuman::memory::score::embed::cosine_similarity;
|
||||||
let query_vec = phoenix_vec.clone();
|
let query_vec = phoenix_vec.clone();
|
||||||
let phoenix_sim = cosine_similarity(&query_vec, &phoenix_vec);
|
let phoenix_sim = cosine_similarity(&query_vec, &phoenix_vec);
|
||||||
let unrelated_sim = cosine_similarity(&query_vec, &unrelated_vec);
|
let unrelated_sim = cosine_similarity(&query_vec, &unrelated_vec);
|
||||||
@@ -629,9 +627,9 @@ mod tests {
|
|||||||
/// summaries that do have embeddings when a `query` is supplied.
|
/// summaries that do have embeddings when a `query` is supplied.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn legacy_null_embedding_rows_sort_last() {
|
async fn legacy_null_embedding_rows_sort_last() {
|
||||||
use crate::openhuman::memory_tree::score::embed::{pack_embedding, EMBEDDING_DIM};
|
use crate::openhuman::memory::score::embed::{pack_embedding, EMBEDDING_DIM};
|
||||||
use crate::openhuman::memory_tree::tree_source::store as src_store;
|
use crate::openhuman::memory_store::trees::types::TreeKind;
|
||||||
use crate::openhuman::memory_tree::tree_source::types::TreeKind;
|
use crate::openhuman::memory_tree::tree::store as src_store;
|
||||||
|
|
||||||
let (_tmp, cfg) = test_config();
|
let (_tmp, cfg) = test_config();
|
||||||
let ts = Utc::now();
|
let ts = Utc::now();
|
||||||
@@ -655,7 +653,7 @@ mod tests {
|
|||||||
v[0] = 1.0;
|
v[0] = 1.0;
|
||||||
let blob = pack_embedding(&v);
|
let blob = pack_embedding(&v);
|
||||||
|
|
||||||
use crate::openhuman::memory_tree::store::with_connection;
|
use crate::openhuman::memory_store::chunks::store::with_connection;
|
||||||
with_connection(&cfg, |conn| {
|
with_connection(&cfg, |conn| {
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE mem_tree_summaries SET embedding = ?1 WHERE id = ?2",
|
"UPDATE mem_tree_summaries SET embedding = ?1 WHERE id = ?2",
|
||||||
+229
-21
@@ -17,14 +17,12 @@ use anyhow::Result;
|
|||||||
use chrono::{Duration, TimeZone, Utc};
|
use chrono::{Duration, TimeZone, Utc};
|
||||||
|
|
||||||
use crate::openhuman::config::Config;
|
use crate::openhuman::config::Config;
|
||||||
use crate::openhuman::memory_tree::content_store::read as content_read;
|
use crate::openhuman::memory::retrieval::types::{hit_from_summary, QueryResponse, RetrievalHit};
|
||||||
use crate::openhuman::memory_tree::retrieval::types::{
|
use crate::openhuman::memory::score::embed::{build_embedder_from_config, cosine_similarity};
|
||||||
hit_from_summary, QueryResponse, RetrievalHit,
|
use crate::openhuman::memory::score::store::{lookup_entity, EntityHit};
|
||||||
};
|
use crate::openhuman::memory_store::content::read as content_read;
|
||||||
use crate::openhuman::memory_tree::score::embed::{build_embedder_from_config, cosine_similarity};
|
use crate::openhuman::memory_store::trees::types::{Tree, TreeKind};
|
||||||
use crate::openhuman::memory_tree::score::store::{lookup_entity, EntityHit};
|
use crate::openhuman::memory_tree::tree::store;
|
||||||
use crate::openhuman::memory_tree::tree_source::store;
|
|
||||||
use crate::openhuman::memory_tree::tree_source::types::{Tree, TreeKind};
|
|
||||||
|
|
||||||
const DEFAULT_LIMIT: usize = 10;
|
const DEFAULT_LIMIT: usize = 10;
|
||||||
/// How many rows we pull from the entity index before filtering. We give
|
/// How many rows we pull from the entity index before filtering. We give
|
||||||
@@ -175,9 +173,9 @@ async fn rerank_by_semantic_similarity(
|
|||||||
query: &str,
|
query: &str,
|
||||||
hits: Vec<RetrievalHit>,
|
hits: Vec<RetrievalHit>,
|
||||||
) -> Result<Vec<RetrievalHit>> {
|
) -> Result<Vec<RetrievalHit>> {
|
||||||
use crate::openhuman::memory_tree::retrieval::types::NodeKind;
|
use crate::openhuman::memory::retrieval::types::NodeKind;
|
||||||
use crate::openhuman::memory_tree::store::get_chunk_embedding;
|
use crate::openhuman::memory_store::chunks::store::get_chunk_embedding;
|
||||||
use crate::openhuman::memory_tree::tree_source::store as src_store;
|
use crate::openhuman::memory_tree::tree::store as src_store;
|
||||||
|
|
||||||
let embedder = build_embedder_from_config(config)?;
|
let embedder = build_embedder_from_config(config)?;
|
||||||
let query_vec = embedder.embed(query).await?;
|
let query_vec = embedder.embed(query).await?;
|
||||||
@@ -318,8 +316,8 @@ async fn entity_hit_to_retrieval_hit(
|
|||||||
return Ok(Some(h));
|
return Ok(Some(h));
|
||||||
}
|
}
|
||||||
// Leaf: fetch chunk and hydrate.
|
// Leaf: fetch chunk and hydrate.
|
||||||
use crate::openhuman::memory_tree::retrieval::types::hit_from_chunk;
|
use crate::openhuman::memory::retrieval::types::hit_from_chunk;
|
||||||
use crate::openhuman::memory_tree::store::get_chunk;
|
use crate::openhuman::memory_store::chunks::store::get_chunk;
|
||||||
let mut chunk = match get_chunk(&config_owned, &node_id)? {
|
let mut chunk = match get_chunk(&config_owned, &node_id)? {
|
||||||
Some(c) => c,
|
Some(c) => c,
|
||||||
None => {
|
None => {
|
||||||
@@ -368,8 +366,14 @@ fn filter_by_window(hits: Vec<RetrievalHit>, window_days: u32) -> Vec<RetrievalH
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::openhuman::memory_tree::canonicalize::chat::{ChatBatch, ChatMessage};
|
use crate::openhuman::memory::ingest_pipeline::ingest_chat;
|
||||||
use crate::openhuman::memory_tree::ingest::ingest_chat;
|
use crate::openhuman::memory::score::extract::EntityKind;
|
||||||
|
use crate::openhuman::memory::score::store::EntityHit;
|
||||||
|
use crate::openhuman::memory_store::chunks::store::upsert_chunks;
|
||||||
|
use crate::openhuman::memory_store::chunks::types::{
|
||||||
|
chunk_id, Chunk, Metadata, SourceKind, SourceRef,
|
||||||
|
};
|
||||||
|
use crate::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage};
|
||||||
use chrono::TimeZone;
|
use chrono::TimeZone;
|
||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
|
|
||||||
@@ -522,6 +526,71 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn zero_limit_uses_default_limit() {
|
||||||
|
let (_tmp, cfg) = test_config();
|
||||||
|
for i in 0..12 {
|
||||||
|
let source = format!("slack:#c{i}");
|
||||||
|
let batch = ChatBatch {
|
||||||
|
platform: "slack".into(),
|
||||||
|
channel_label: format!("#c{i}"),
|
||||||
|
messages: vec![ChatMessage {
|
||||||
|
author: "alice".into(),
|
||||||
|
timestamp: Utc::now(),
|
||||||
|
text: format!(
|
||||||
|
"Meeting {i} about Phoenix migration. alice@example.com owns it. \
|
||||||
|
Launch status looks good."
|
||||||
|
),
|
||||||
|
source_ref: None,
|
||||||
|
}],
|
||||||
|
};
|
||||||
|
ingest_chat(&cfg, &source, "alice", vec![], batch)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
let resp = query_topic(&cfg, "email:alice@example.com", None, None, 0)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(resp.hits.len() <= DEFAULT_LIMIT);
|
||||||
|
if resp.total > DEFAULT_LIMIT {
|
||||||
|
assert!(resp.truncated);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn topic_tree_root_missing_summary_row_is_ignored() {
|
||||||
|
use crate::openhuman::memory_store::chunks::store::with_connection;
|
||||||
|
use crate::openhuman::memory_store::trees::types::{Tree, TreeKind, TreeStatus};
|
||||||
|
use crate::openhuman::memory_tree::tree::store as tree_store;
|
||||||
|
|
||||||
|
let (_tmp, cfg) = test_config();
|
||||||
|
let ts = Utc::now();
|
||||||
|
let entity_id = "topic:phoenix";
|
||||||
|
let tree = Tree {
|
||||||
|
id: "test:phoenix-missing-root".into(),
|
||||||
|
kind: TreeKind::Topic,
|
||||||
|
scope: entity_id.into(),
|
||||||
|
root_id: Some("summary:missing".into()),
|
||||||
|
max_level: 1,
|
||||||
|
status: TreeStatus::Active,
|
||||||
|
created_at: ts,
|
||||||
|
last_sealed_at: Some(ts),
|
||||||
|
};
|
||||||
|
|
||||||
|
with_connection(&cfg, |conn| {
|
||||||
|
let tx = conn.unchecked_transaction()?;
|
||||||
|
tree_store::insert_tree_conn(&tx, &tree)?;
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let resp = query_topic(&cfg, entity_id, None, None, 10).await.unwrap();
|
||||||
|
assert!(resp.hits.is_empty());
|
||||||
|
assert_eq!(resp.total, 0);
|
||||||
|
}
|
||||||
|
|
||||||
// Regression: the same node_id must only appear once in `hits`, even
|
// Regression: the same node_id must only appear once in `hits`, even
|
||||||
// when the topic-tree root overlaps with its own entity-index row.
|
// when the topic-tree root overlaps with its own entity-index row.
|
||||||
// Flagged on PR #831 CodeRabbit review — see the HashMap-based merge
|
// Flagged on PR #831 CodeRabbit review — see the HashMap-based merge
|
||||||
@@ -529,14 +598,13 @@ mod tests {
|
|||||||
// caller would see two rows for the same summary.
|
// caller would see two rows for the same summary.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn duplicate_node_is_deduplicated_across_index_and_topic_tree_root() {
|
async fn duplicate_node_is_deduplicated_across_index_and_topic_tree_root() {
|
||||||
use crate::openhuman::memory_tree::score::extract::EntityKind;
|
use crate::openhuman::memory::score::resolver::CanonicalEntity;
|
||||||
use crate::openhuman::memory_tree::score::resolver::CanonicalEntity;
|
use crate::openhuman::memory::score::store as score_store;
|
||||||
use crate::openhuman::memory_tree::score::store as score_store;
|
use crate::openhuman::memory_store::chunks::store::with_connection;
|
||||||
use crate::openhuman::memory_tree::store::with_connection;
|
use crate::openhuman::memory_store::trees::types::{
|
||||||
use crate::openhuman::memory_tree::tree_source::store as tree_store;
|
|
||||||
use crate::openhuman::memory_tree::tree_source::types::{
|
|
||||||
SummaryNode, Tree, TreeKind, TreeStatus,
|
SummaryNode, Tree, TreeKind, TreeStatus,
|
||||||
};
|
};
|
||||||
|
use crate::openhuman::memory_tree::tree::store as tree_store;
|
||||||
|
|
||||||
let (_tmp, cfg) = test_config();
|
let (_tmp, cfg) = test_config();
|
||||||
let ts = Utc::now();
|
let ts = Utc::now();
|
||||||
@@ -625,4 +693,144 @@ mod tests {
|
|||||||
"total should count distinct nodes, not raw row occurrences"
|
"total should count distinct nodes, not raw row occurrences"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn stale_leaf_entity_index_row_is_skipped() {
|
||||||
|
let (_tmp, cfg) = test_config();
|
||||||
|
let hit = EntityHit {
|
||||||
|
entity_id: "email:alice@example.com".into(),
|
||||||
|
node_id: "chunk:missing".into(),
|
||||||
|
node_kind: "leaf".into(),
|
||||||
|
entity_kind: EntityKind::Email,
|
||||||
|
surface: "alice@example.com".into(),
|
||||||
|
score: 0.7,
|
||||||
|
timestamp_ms: Utc::now().timestamp_millis(),
|
||||||
|
tree_id: Some("tree:missing".into()),
|
||||||
|
is_user: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
let converted = entity_hit_to_retrieval_hit(&cfg, &hit).await.unwrap();
|
||||||
|
assert!(converted.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn leaf_hit_falls_back_to_source_scope_when_tree_lookup_misses() {
|
||||||
|
let (_tmp, cfg) = test_config();
|
||||||
|
let ts = Utc.timestamp_millis_opt(1_700_123_456_789).unwrap();
|
||||||
|
let chunk = Chunk {
|
||||||
|
id: chunk_id(
|
||||||
|
SourceKind::Chat,
|
||||||
|
"slack:#eng",
|
||||||
|
0,
|
||||||
|
"Phoenix owner is alice@example.com",
|
||||||
|
),
|
||||||
|
content: "Phoenix owner is alice@example.com".into(),
|
||||||
|
metadata: Metadata {
|
||||||
|
source_kind: SourceKind::Chat,
|
||||||
|
source_id: "slack:#eng".into(),
|
||||||
|
owner: "alice".into(),
|
||||||
|
timestamp: ts,
|
||||||
|
time_range: (ts, ts),
|
||||||
|
tags: vec!["phoenix".into()],
|
||||||
|
source_ref: Some(SourceRef::new("slack://eng/1")),
|
||||||
|
},
|
||||||
|
token_count: 8,
|
||||||
|
seq_in_source: 0,
|
||||||
|
created_at: ts,
|
||||||
|
partial_message: false,
|
||||||
|
};
|
||||||
|
upsert_chunks(&cfg, &[chunk.clone()]).unwrap();
|
||||||
|
|
||||||
|
let hit = EntityHit {
|
||||||
|
entity_id: "email:alice@example.com".into(),
|
||||||
|
node_id: chunk.id.clone(),
|
||||||
|
node_kind: "leaf".into(),
|
||||||
|
entity_kind: EntityKind::Email,
|
||||||
|
surface: "alice@example.com".into(),
|
||||||
|
score: 0.91,
|
||||||
|
timestamp_ms: ts.timestamp_millis(),
|
||||||
|
tree_id: Some("tree:missing".into()),
|
||||||
|
is_user: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
let converted = entity_hit_to_retrieval_hit(&cfg, &hit)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(converted.tree_scope, "slack:#eng");
|
||||||
|
assert_eq!(converted.tree_id, "tree:missing");
|
||||||
|
assert_eq!(converted.score, 0.91);
|
||||||
|
assert_eq!(converted.source_ref.as_deref(), Some("slack://eng/1"));
|
||||||
|
assert_eq!(converted.topics, vec!["phoenix"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn summary_hit_without_tree_id_uses_empty_scope_and_index_score() {
|
||||||
|
use crate::openhuman::memory_store::chunks::store::with_connection;
|
||||||
|
use crate::openhuman::memory_store::trees::types::{
|
||||||
|
SummaryNode, Tree, TreeKind, TreeStatus,
|
||||||
|
};
|
||||||
|
use crate::openhuman::memory_tree::tree::store as tree_store;
|
||||||
|
|
||||||
|
let (_tmp, cfg) = test_config();
|
||||||
|
let ts = Utc::now();
|
||||||
|
let tree = Tree {
|
||||||
|
id: "tree:topic:phoenix".into(),
|
||||||
|
kind: TreeKind::Topic,
|
||||||
|
scope: "topic:phoenix".into(),
|
||||||
|
root_id: Some("summary:l1:phoenix".into()),
|
||||||
|
max_level: 1,
|
||||||
|
status: TreeStatus::Active,
|
||||||
|
created_at: ts,
|
||||||
|
last_sealed_at: Some(ts),
|
||||||
|
};
|
||||||
|
let summary = SummaryNode {
|
||||||
|
id: "summary:l1:phoenix".into(),
|
||||||
|
tree_id: tree.id.clone(),
|
||||||
|
tree_kind: TreeKind::Topic,
|
||||||
|
level: 1,
|
||||||
|
parent_id: None,
|
||||||
|
child_ids: vec!["chunk:a".into()],
|
||||||
|
content: "Phoenix recap preview".into(),
|
||||||
|
token_count: 16,
|
||||||
|
entities: vec!["topic:phoenix".into()],
|
||||||
|
topics: vec!["phoenix".into()],
|
||||||
|
time_range_start: ts,
|
||||||
|
time_range_end: ts,
|
||||||
|
score: 0.12,
|
||||||
|
sealed_at: ts,
|
||||||
|
deleted: false,
|
||||||
|
embedding: None,
|
||||||
|
};
|
||||||
|
|
||||||
|
with_connection(&cfg, |conn| {
|
||||||
|
let tx = conn.unchecked_transaction()?;
|
||||||
|
tree_store::insert_tree_conn(&tx, &tree)?;
|
||||||
|
tree_store::insert_summary_tx(&tx, &summary, None, "test")?;
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let hit = EntityHit {
|
||||||
|
entity_id: "topic:phoenix".into(),
|
||||||
|
node_id: summary.id.clone(),
|
||||||
|
node_kind: "summary".into(),
|
||||||
|
entity_kind: EntityKind::Topic,
|
||||||
|
surface: "phoenix".into(),
|
||||||
|
score: 0.88,
|
||||||
|
timestamp_ms: ts.timestamp_millis(),
|
||||||
|
tree_id: None,
|
||||||
|
is_user: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
let converted = entity_hit_to_retrieval_hit(&cfg, &hit)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(converted.tree_scope, "");
|
||||||
|
assert_eq!(converted.tree_id, tree.id);
|
||||||
|
assert_eq!(converted.score, 0.88);
|
||||||
|
assert_eq!(converted.content, "Phoenix recap preview");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
+3
-3
@@ -18,9 +18,9 @@
|
|||||||
use chrono::{DateTime, Utc};
|
use chrono::{DateTime, Utc};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use crate::openhuman::memory_tree::score::extract::EntityKind;
|
use crate::openhuman::memory::score::extract::EntityKind;
|
||||||
use crate::openhuman::memory_tree::tree_source::types::{SummaryNode, Tree, TreeKind};
|
use crate::openhuman::memory_store::chunks::types::{Chunk, SourceKind};
|
||||||
use crate::openhuman::memory_tree::types::{Chunk, SourceKind};
|
use crate::openhuman::memory_store::trees::types::{SummaryNode, Tree, TreeKind};
|
||||||
|
|
||||||
/// Whether a hit represents a leaf (raw chunk) or a summary node.
|
/// Whether a hit represents a leaf (raw chunk) or a summary node.
|
||||||
#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)]
|
#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)]
|
||||||
@@ -16,8 +16,8 @@ use serde_json::{Map, Value};
|
|||||||
use crate::core::all::{ControllerFuture, RegisteredController};
|
use crate::core::all::{ControllerFuture, RegisteredController};
|
||||||
use crate::core::{ControllerSchema, FieldSchema, TypeSchema};
|
use crate::core::{ControllerSchema, FieldSchema, TypeSchema};
|
||||||
use crate::openhuman::config::rpc as config_rpc;
|
use crate::openhuman::config::rpc as config_rpc;
|
||||||
use crate::openhuman::memory_tree::read_rpc;
|
use crate::openhuman::memory::read_rpc;
|
||||||
use crate::openhuman::memory_tree::rpc as tree_rpc;
|
use crate::openhuman::memory::tree_rpc;
|
||||||
use crate::rpc::RpcOutcome;
|
use crate::rpc::RpcOutcome;
|
||||||
|
|
||||||
const NAMESPACE: &str = "memory_tree";
|
const NAMESPACE: &str = "memory_tree";
|
||||||
@@ -207,26 +207,31 @@ pub fn schemas(function: &str) -> ControllerSchema {
|
|||||||
"list_chunks" => ControllerSchema {
|
"list_chunks" => ControllerSchema {
|
||||||
namespace: NAMESPACE,
|
namespace: NAMESPACE,
|
||||||
function: "list_chunks",
|
function: "list_chunks",
|
||||||
description:
|
description: "Paginated list of chunks with optional filters by source kind / source id / \
|
||||||
"Paginated list of chunks with optional filters by source kind / source id / \
|
|
||||||
entity ids / time window / keyword. Returns chunks plus total match count for \
|
entity ids / time window / keyword. Returns chunks plus total match count for \
|
||||||
pagination.",
|
pagination.",
|
||||||
inputs: vec![
|
inputs: vec![
|
||||||
FieldSchema {
|
FieldSchema {
|
||||||
name: "source_kinds",
|
name: "source_kinds",
|
||||||
ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new(TypeSchema::String)))),
|
ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new(
|
||||||
|
TypeSchema::String,
|
||||||
|
)))),
|
||||||
comment: "Restrict to one or more source kinds (chat / email / document).",
|
comment: "Restrict to one or more source kinds (chat / email / document).",
|
||||||
required: false,
|
required: false,
|
||||||
},
|
},
|
||||||
FieldSchema {
|
FieldSchema {
|
||||||
name: "source_ids",
|
name: "source_ids",
|
||||||
ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new(TypeSchema::String)))),
|
ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new(
|
||||||
|
TypeSchema::String,
|
||||||
|
)))),
|
||||||
comment: "Restrict to one or more logical source ids.",
|
comment: "Restrict to one or more logical source ids.",
|
||||||
required: false,
|
required: false,
|
||||||
},
|
},
|
||||||
FieldSchema {
|
FieldSchema {
|
||||||
name: "entity_ids",
|
name: "entity_ids",
|
||||||
ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new(TypeSchema::String)))),
|
ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new(
|
||||||
|
TypeSchema::String,
|
||||||
|
)))),
|
||||||
comment: "Restrict to chunks indexed against any of these canonical entity ids.",
|
comment: "Restrict to chunks indexed against any of these canonical entity ids.",
|
||||||
required: false,
|
required: false,
|
||||||
},
|
},
|
||||||
@@ -296,8 +301,7 @@ pub fn schemas(function: &str) -> ControllerSchema {
|
|||||||
"list_sources" => ControllerSchema {
|
"list_sources" => ControllerSchema {
|
||||||
namespace: NAMESPACE,
|
namespace: NAMESPACE,
|
||||||
function: "list_sources",
|
function: "list_sources",
|
||||||
description:
|
description: "Distinct (source_kind, source_id) pairs with chunk counts and most-recent timestamps. \
|
||||||
"Distinct (source_kind, source_id) pairs with chunk counts and most-recent timestamps. \
|
|
||||||
`display_name` is computed from the source_id (un-slug + strip user email when known).",
|
`display_name` is computed from the source_id (un-slug + strip user email when known).",
|
||||||
inputs: vec![FieldSchema {
|
inputs: vec![FieldSchema {
|
||||||
name: "user_email_hint",
|
name: "user_email_hint",
|
||||||
@@ -316,8 +320,7 @@ pub fn schemas(function: &str) -> ControllerSchema {
|
|||||||
"search" => ControllerSchema {
|
"search" => ControllerSchema {
|
||||||
namespace: NAMESPACE,
|
namespace: NAMESPACE,
|
||||||
function: "search",
|
function: "search",
|
||||||
description:
|
description: "Keyword LIKE-search over chunk bodies. Cheap, deterministic; useful as a \
|
||||||
"Keyword LIKE-search over chunk bodies. Cheap, deterministic; useful as a \
|
|
||||||
fallback when semantic recall is unavailable.",
|
fallback when semantic recall is unavailable.",
|
||||||
inputs: vec![
|
inputs: vec![
|
||||||
FieldSchema {
|
FieldSchema {
|
||||||
@@ -343,8 +346,7 @@ pub fn schemas(function: &str) -> ControllerSchema {
|
|||||||
"recall" => ControllerSchema {
|
"recall" => ControllerSchema {
|
||||||
namespace: NAMESPACE,
|
namespace: NAMESPACE,
|
||||||
function: "recall",
|
function: "recall",
|
||||||
description:
|
description: "Semantic recall — runs the Phase 4 cosine rerank against the query embedding \
|
||||||
"Semantic recall — runs the Phase 4 cosine rerank against the query embedding \
|
|
||||||
and returns leaf chunks (not summaries) for UI display.",
|
and returns leaf chunks (not summaries) for UI display.",
|
||||||
inputs: vec![
|
inputs: vec![
|
||||||
FieldSchema {
|
FieldSchema {
|
||||||
@@ -395,14 +397,12 @@ pub fn schemas(function: &str) -> ControllerSchema {
|
|||||||
"chunks_for_entity" => ControllerSchema {
|
"chunks_for_entity" => ControllerSchema {
|
||||||
namespace: NAMESPACE,
|
namespace: NAMESPACE,
|
||||||
function: "chunks_for_entity",
|
function: "chunks_for_entity",
|
||||||
description:
|
description: "Return chunk IDs that reference an entity_id (inverse of entity_index_for). \
|
||||||
"Return chunk IDs that reference an entity_id (inverse of entity_index_for). \
|
|
||||||
Used by the Memory tab's People/Topics lenses to filter the chunk list.",
|
Used by the Memory tab's People/Topics lenses to filter the chunk list.",
|
||||||
inputs: vec![FieldSchema {
|
inputs: vec![FieldSchema {
|
||||||
name: "entity_id",
|
name: "entity_id",
|
||||||
ty: TypeSchema::String,
|
ty: TypeSchema::String,
|
||||||
comment:
|
comment: "Canonical entity id (e.g. `person:Steven Enamakel`, \
|
||||||
"Canonical entity id (e.g. `person:Steven Enamakel`, \
|
|
||||||
`email:alice@example.com`).",
|
`email:alice@example.com`).",
|
||||||
required: true,
|
required: true,
|
||||||
}],
|
}],
|
||||||
@@ -416,8 +416,7 @@ pub fn schemas(function: &str) -> ControllerSchema {
|
|||||||
"top_entities" => ControllerSchema {
|
"top_entities" => ControllerSchema {
|
||||||
namespace: NAMESPACE,
|
namespace: NAMESPACE,
|
||||||
function: "top_entities",
|
function: "top_entities",
|
||||||
description:
|
description: "Most-frequent canonical entities across the workspace, optionally narrowed by kind.",
|
||||||
"Most-frequent canonical entities across the workspace, optionally narrowed by kind.",
|
|
||||||
inputs: vec![
|
inputs: vec![
|
||||||
FieldSchema {
|
FieldSchema {
|
||||||
name: "kind",
|
name: "kind",
|
||||||
@@ -442,8 +441,7 @@ pub fn schemas(function: &str) -> ControllerSchema {
|
|||||||
"chunk_score" => ControllerSchema {
|
"chunk_score" => ControllerSchema {
|
||||||
namespace: NAMESPACE,
|
namespace: NAMESPACE,
|
||||||
function: "chunk_score",
|
function: "chunk_score",
|
||||||
description:
|
description: "Score breakdown stored in `mem_tree_score` for one chunk — used by the Memory \
|
||||||
"Score breakdown stored in `mem_tree_score` for one chunk — used by the Memory \
|
|
||||||
tab's 'why was this kept / dropped' panel.",
|
tab's 'why was this kept / dropped' panel.",
|
||||||
inputs: vec![FieldSchema {
|
inputs: vec![FieldSchema {
|
||||||
name: "chunk_id",
|
name: "chunk_id",
|
||||||
@@ -461,8 +459,7 @@ pub fn schemas(function: &str) -> ControllerSchema {
|
|||||||
"delete_chunk" => ControllerSchema {
|
"delete_chunk" => ControllerSchema {
|
||||||
namespace: NAMESPACE,
|
namespace: NAMESPACE,
|
||||||
function: "delete_chunk",
|
function: "delete_chunk",
|
||||||
description:
|
description: "Purge one chunk plus its score row, entity-index rows, and on-disk .md file. \
|
||||||
"Purge one chunk plus its score row, entity-index rows, and on-disk .md file. \
|
|
||||||
Idempotent — missing chunk returns deleted=false. Does NOT cascade through \
|
Idempotent — missing chunk returns deleted=false. Does NOT cascade through \
|
||||||
sealed summaries; UIs warn the user.",
|
sealed summaries; UIs warn the user.",
|
||||||
inputs: vec![FieldSchema {
|
inputs: vec![FieldSchema {
|
||||||
@@ -509,8 +506,7 @@ pub fn schemas(function: &str) -> ControllerSchema {
|
|||||||
"set_llm" => ControllerSchema {
|
"set_llm" => ControllerSchema {
|
||||||
namespace: NAMESPACE,
|
namespace: NAMESPACE,
|
||||||
function: "set_llm",
|
function: "set_llm",
|
||||||
description:
|
description: "Update the LLM backend selector and (optionally) per-role model choices \
|
||||||
"Update the LLM backend selector and (optionally) per-role model choices \
|
|
||||||
(`cloud_model`, `extract_model`, `summariser_model`) and persist the \
|
(`cloud_model`, `extract_model`, `summariser_model`) and persist the \
|
||||||
result to config.toml in a single atomic write. Absent model fields \
|
result to config.toml in a single atomic write. Absent model fields \
|
||||||
leave the corresponding config key unchanged so a caller flipping just \
|
leave the corresponding config key unchanged so a caller flipping just \
|
||||||
@@ -961,3 +957,51 @@ fn parse_value<T: DeserializeOwned>(v: Value) -> Result<T, String> {
|
|||||||
fn to_json<T: serde::Serialize>(outcome: RpcOutcome<T>) -> Result<Value, String> {
|
fn to_json<T: serde::Serialize>(outcome: RpcOutcome<T>) -> Result<Value, String> {
|
||||||
outcome.into_cli_compatible_json()
|
outcome.into_cli_compatible_json()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn all_controller_schemas_and_registered_controllers_stay_in_sync() {
|
||||||
|
let schemas = all_controller_schemas();
|
||||||
|
let controllers = all_registered_controllers();
|
||||||
|
assert_eq!(schemas.len(), controllers.len());
|
||||||
|
assert!(schemas.iter().all(|s| s.namespace == NAMESPACE));
|
||||||
|
assert!(controllers.iter().all(|c| c.schema.namespace == NAMESPACE));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_function_schema_returns_error_output() {
|
||||||
|
let schema = schemas("not_real");
|
||||||
|
assert_eq!(schema.namespace, NAMESPACE);
|
||||||
|
assert_eq!(schema.function, "unknown");
|
||||||
|
assert_eq!(schema.outputs.len(), 1);
|
||||||
|
assert_eq!(schema.outputs[0].name, "error");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ingest_schema_requires_source_kind_source_id_and_payload() {
|
||||||
|
let schema = schemas("ingest");
|
||||||
|
assert_eq!(schema.function, "ingest");
|
||||||
|
let required: Vec<&str> = schema
|
||||||
|
.inputs
|
||||||
|
.iter()
|
||||||
|
.filter(|f| f.required)
|
||||||
|
.map(|f| f.name)
|
||||||
|
.collect();
|
||||||
|
assert!(required.contains(&"source_kind"));
|
||||||
|
assert!(required.contains(&"source_id"));
|
||||||
|
assert!(required.contains(&"payload"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn set_llm_schema_exposes_backend_update_fields() {
|
||||||
|
let schema = schemas("set_llm");
|
||||||
|
let names: Vec<&str> = schema.inputs.iter().map(|f| f.name).collect();
|
||||||
|
assert!(names.contains(&"backend"));
|
||||||
|
assert!(names.contains(&"cloud_model"));
|
||||||
|
assert!(names.contains(&"extract_model"));
|
||||||
|
assert!(names.contains(&"summariser_model"));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -535,3 +535,42 @@ fn handle_clear_namespace(params: Map<String, Value>) -> ControllerFuture {
|
|||||||
to_json(rpc::clear_namespace(payload).await?)
|
to_json(rpc::clear_namespace(payload).await?)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn documents_schema_exposes_all_functions() {
|
||||||
|
assert_eq!(controllers().len(), FUNCTIONS.len());
|
||||||
|
assert!(FUNCTIONS.contains(&"init"));
|
||||||
|
assert!(FUNCTIONS.contains(&"doc_ingest"));
|
||||||
|
assert!(FUNCTIONS.contains(&"clear_namespace"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_document_schema_returns_none() {
|
||||||
|
assert!(schema("not_real").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn query_namespace_schema_requires_namespace_and_query() {
|
||||||
|
let schema = schema("query_namespace").unwrap();
|
||||||
|
let required: Vec<&str> = schema
|
||||||
|
.inputs
|
||||||
|
.iter()
|
||||||
|
.filter(|f| f.required)
|
||||||
|
.map(|f| f.name)
|
||||||
|
.collect();
|
||||||
|
assert!(required.contains(&"namespace"));
|
||||||
|
assert!(required.contains(&"query"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn clear_namespace_schema_requires_namespace() {
|
||||||
|
let schema = schema("clear_namespace").unwrap();
|
||||||
|
assert_eq!(schema.inputs.len(), 1);
|
||||||
|
assert_eq!(schema.inputs[0].name, "namespace");
|
||||||
|
assert!(schema.inputs[0].required);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -126,3 +126,32 @@ fn handle_write_file(params: Map<String, Value>) -> ControllerFuture {
|
|||||||
to_json(rpc::ai_write_memory_file(payload).await?)
|
to_json(rpc::ai_write_memory_file(payload).await?)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn file_schema_exposes_all_functions() {
|
||||||
|
assert_eq!(FUNCTIONS, &["list_files", "read_file", "write_file"]);
|
||||||
|
assert_eq!(controllers().len(), FUNCTIONS.len());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_file_schema_returns_none() {
|
||||||
|
assert!(schema("not_real").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn write_file_schema_requires_path_and_content() {
|
||||||
|
let schema = schema("write_file").unwrap();
|
||||||
|
let required: Vec<&str> = schema
|
||||||
|
.inputs
|
||||||
|
.iter()
|
||||||
|
.filter(|f| f.required)
|
||||||
|
.map(|f| f.name)
|
||||||
|
.collect();
|
||||||
|
assert!(required.contains(&"relative_path"));
|
||||||
|
assert!(required.contains(&"content"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -267,3 +267,43 @@ fn handle_graph_query(params: Map<String, Value>) -> ControllerFuture {
|
|||||||
to_json(rpc::graph_query(payload).await?)
|
to_json(rpc::graph_query(payload).await?)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn kv_graph_schema_exposes_all_functions() {
|
||||||
|
assert_eq!(
|
||||||
|
FUNCTIONS,
|
||||||
|
&[
|
||||||
|
"kv_set",
|
||||||
|
"kv_get",
|
||||||
|
"kv_delete",
|
||||||
|
"kv_list_namespace",
|
||||||
|
"graph_upsert",
|
||||||
|
"graph_query",
|
||||||
|
]
|
||||||
|
);
|
||||||
|
assert_eq!(controllers().len(), FUNCTIONS.len());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_kv_graph_schema_returns_none() {
|
||||||
|
assert!(schema("not_real").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn graph_upsert_schema_requires_subject_predicate_and_object() {
|
||||||
|
let schema = schema("graph_upsert").unwrap();
|
||||||
|
let required: Vec<&str> = schema
|
||||||
|
.inputs
|
||||||
|
.iter()
|
||||||
|
.filter(|f| f.required)
|
||||||
|
.map(|f| f.name)
|
||||||
|
.collect();
|
||||||
|
assert!(required.contains(&"subject"));
|
||||||
|
assert!(required.contains(&"predicate"));
|
||||||
|
assert!(required.contains(&"object"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -54,3 +54,27 @@ fn handle_learn_all(params: Map<String, Value>) -> ControllerFuture {
|
|||||||
to_json(rpc::memory_learn_all(payload).await?)
|
to_json(rpc::memory_learn_all(payload).await?)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn learn_schema_only_exposes_learn_all() {
|
||||||
|
assert_eq!(FUNCTIONS, &["learn_all"]);
|
||||||
|
assert_eq!(controllers().len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_learn_schema_returns_none() {
|
||||||
|
assert!(schema("not_real").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn learn_all_schema_has_optional_namespaces_input() {
|
||||||
|
let schema = schema("learn_all").unwrap();
|
||||||
|
assert_eq!(schema.inputs.len(), 1);
|
||||||
|
assert_eq!(schema.inputs[0].name, "namespaces");
|
||||||
|
assert!(!schema.inputs[0].required);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -85,3 +85,27 @@ fn handle_sync_all(_params: Map<String, Value>) -> ControllerFuture {
|
|||||||
fn handle_ingestion_status(_params: Map<String, Value>) -> ControllerFuture {
|
fn handle_ingestion_status(_params: Map<String, Value>) -> ControllerFuture {
|
||||||
Box::pin(async move { to_json(rpc::memory_ingestion_status().await?) })
|
Box::pin(async move { to_json(rpc::memory_ingestion_status().await?) })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sync_schema_exposes_all_functions() {
|
||||||
|
assert_eq!(FUNCTIONS, &["sync_channel", "sync_all", "ingestion_status"]);
|
||||||
|
assert_eq!(controllers().len(), FUNCTIONS.len());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_sync_schema_returns_none() {
|
||||||
|
assert!(schema("not_real").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sync_channel_schema_requires_channel_id() {
|
||||||
|
let schema = schema("sync_channel").unwrap();
|
||||||
|
assert_eq!(schema.inputs.len(), 1);
|
||||||
|
assert_eq!(schema.inputs[0].name, "channel_id");
|
||||||
|
assert!(schema.inputs[0].required);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -226,6 +226,52 @@ pub(super) fn schema(function: &str) -> Option<ControllerSchema> {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn schema_exposes_all_tool_memory_functions() {
|
||||||
|
let functions: Vec<&str> = FUNCTIONS.to_vec();
|
||||||
|
assert_eq!(
|
||||||
|
functions,
|
||||||
|
vec![
|
||||||
|
"tool_rule_put",
|
||||||
|
"tool_rule_get",
|
||||||
|
"tool_rule_list",
|
||||||
|
"tool_rule_delete",
|
||||||
|
"tool_rules_for_prompt",
|
||||||
|
"tool_rules_json",
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn controllers_match_function_count() {
|
||||||
|
let registered = controllers();
|
||||||
|
assert_eq!(registered.len(), FUNCTIONS.len());
|
||||||
|
assert!(registered.iter().all(|c| c.schema.namespace == "memory"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn schema_returns_none_for_unknown_function() {
|
||||||
|
assert!(schema("not_real").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn tool_rule_put_schema_requires_tool_name_and_rule() {
|
||||||
|
let schema = schema("tool_rule_put").unwrap();
|
||||||
|
let input_names: Vec<&str> = schema.inputs.iter().map(|f| f.name).collect();
|
||||||
|
assert!(input_names.contains(&"tool_name"));
|
||||||
|
assert!(input_names.contains(&"rule"));
|
||||||
|
assert!(schema
|
||||||
|
.inputs
|
||||||
|
.iter()
|
||||||
|
.any(|f| f.name == "tool_name" && f.required));
|
||||||
|
assert!(schema.inputs.iter().any(|f| f.name == "rule" && f.required));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn handle_tool_rule_put(params: Map<String, Value>) -> ControllerFuture {
|
fn handle_tool_rule_put(params: Map<String, Value>) -> ControllerFuture {
|
||||||
Box::pin(async move {
|
Box::pin(async move {
|
||||||
let payload = parse_params::<ToolRulePutParams>(params)?;
|
let payload = parse_params::<ToolRulePutParams>(params)?;
|
||||||
|
|||||||
+1
-1
@@ -77,7 +77,7 @@ impl EntityExtractor for CompositeExtractor {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::openhuman::memory_tree::score::extract::EntityKind;
|
use crate::openhuman::memory::score::extract::EntityKind;
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn regex_only_extractor_works() {
|
async fn regex_only_extractor_works() {
|
||||||
+2
-2
@@ -35,7 +35,7 @@ use serde::Deserialize;
|
|||||||
|
|
||||||
use super::types::{EntityKind, ExtractedEntities, ExtractedEntity, ExtractedTopic};
|
use super::types::{EntityKind, ExtractedEntities, ExtractedEntity, ExtractedTopic};
|
||||||
use super::EntityExtractor;
|
use super::EntityExtractor;
|
||||||
use crate::openhuman::memory_tree::chat::{ChatPrompt, ChatProvider};
|
use crate::openhuman::memory::chat::{ChatPrompt, ChatProvider};
|
||||||
|
|
||||||
// ── Configuration ────────────────────────────────────────────────────────
|
// ── Configuration ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -101,7 +101,7 @@ impl Default for LlmExtractorConfig {
|
|||||||
/// Holds an `Arc<dyn ChatProvider>` rather than a per-instance HTTP
|
/// Holds an `Arc<dyn ChatProvider>` rather than a per-instance HTTP
|
||||||
/// client. The provider abstraction lets a single workspace choose
|
/// client. The provider abstraction lets a single workspace choose
|
||||||
/// cloud vs local at runtime (see
|
/// cloud vs local at runtime (see
|
||||||
/// [`crate::openhuman::memory_tree::chat::build_chat_provider`]). Tests
|
/// [`crate::openhuman::memory::chat::build_chat_provider`]). Tests
|
||||||
/// can mock the provider to assert the prompt / parse behaviour without
|
/// can mock the provider to assert the prompt / parse behaviour without
|
||||||
/// a real Ollama or backend.
|
/// a real Ollama or backend.
|
||||||
pub struct LlmEntityExtractor {
|
pub struct LlmEntityExtractor {
|
||||||
+4
-4
@@ -193,7 +193,7 @@ async fn extract_soft_fallback_on_provider_failure() {
|
|||||||
// Provider always errors. extract() must NOT return Err — it must
|
// Provider always errors. extract() must NOT return Err — it must
|
||||||
// return an empty ExtractedEntities with a warn log after retry
|
// return an empty ExtractedEntities with a warn log after retry
|
||||||
// exhaustion.
|
// exhaustion.
|
||||||
use crate::openhuman::memory_tree::chat::{ChatPrompt, ChatProvider};
|
use crate::openhuman::memory::chat::{ChatPrompt, ChatProvider};
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
@@ -220,7 +220,7 @@ async fn extract_routes_through_chat_provider_and_parses_response() {
|
|||||||
// Mock provider returns canned NER+importance JSON. Verify the
|
// Mock provider returns canned NER+importance JSON. Verify the
|
||||||
// extractor parses it, recovers spans by string search, and emits the
|
// extractor parses it, recovers spans by string search, and emits the
|
||||||
// expected entities + importance signal.
|
// expected entities + importance signal.
|
||||||
use crate::openhuman::memory_tree::chat::{ChatPrompt, ChatProvider};
|
use crate::openhuman::memory::chat::{ChatPrompt, ChatProvider};
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
@@ -266,7 +266,7 @@ async fn extract_returns_empty_on_malformed_provider_response() {
|
|||||||
// Provider returns garbage. Caller must NOT see an Err — the parse
|
// Provider returns garbage. Caller must NOT see an Err — the parse
|
||||||
// failure path returns empty entities (retrying the same input would
|
// failure path returns empty entities (retrying the same input would
|
||||||
// yield the same garbage, so we don't burn retries).
|
// yield the same garbage, so we don't burn retries).
|
||||||
use crate::openhuman::memory_tree::chat::{ChatPrompt, ChatProvider};
|
use crate::openhuman::memory::chat::{ChatPrompt, ChatProvider};
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
@@ -386,7 +386,7 @@ fn into_extracted_entities_disallowed_known_kind_falls_back_to_misc() {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn build_prompt_carries_user_text_and_kind_tag() {
|
fn build_prompt_carries_user_text_and_kind_tag() {
|
||||||
use crate::openhuman::memory_tree::chat::{ChatPrompt, ChatProvider};
|
use crate::openhuman::memory::chat::{ChatPrompt, ChatProvider};
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
//! Entity extraction (Phase 2 / #708).
|
||||||
|
//!
|
||||||
|
//! Exposes [`EntityExtractor`] as a pluggable interface and a default
|
||||||
|
//! [`CompositeExtractor`] that runs a chain of extractors and merges their
|
||||||
|
//! output. Phase 2 ships with the mechanical regex extractor only; semantic
|
||||||
|
//! NER (GLiNER / LLM) plugs in later without changing any call sites.
|
||||||
|
|
||||||
|
mod extractor;
|
||||||
|
pub mod llm;
|
||||||
|
pub mod regex;
|
||||||
|
pub mod types;
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use crate::openhuman::config::Config;
|
||||||
|
use crate::openhuman::memory::chat::build_chat_runtime;
|
||||||
|
|
||||||
|
pub use extractor::{CompositeExtractor, EntityExtractor, RegexEntityExtractor};
|
||||||
|
pub use llm::{LlmEntityExtractor, LlmExtractorConfig};
|
||||||
|
pub use types::{EntityKind, ExtractedEntities, ExtractedEntity, ExtractedTopic};
|
||||||
|
|
||||||
|
/// Build the extractor used by seal handlers to label new summary nodes.
|
||||||
|
///
|
||||||
|
/// Composition:
|
||||||
|
/// - regex extractor — always on, mechanical, near-zero cost
|
||||||
|
/// - LLM extractor with `emit_topics: true` — added when the unified
|
||||||
|
/// summarization workload can be built from inference routing.
|
||||||
|
///
|
||||||
|
/// Differs from [`super::ScoringConfig::from_config`] (the chunk-admission
|
||||||
|
/// builder) in two ways: returns *just* an extractor (no thresholds /
|
||||||
|
/// weights / drop logic — none of which apply at seal time), and flips
|
||||||
|
/// `emit_topics` on so summaries surface thematic labels alongside
|
||||||
|
/// entities. Leaf-side scoring is unchanged.
|
||||||
|
pub fn build_summary_extractor(config: &Config) -> Arc<dyn EntityExtractor> {
|
||||||
|
let (provider, model) = match build_chat_runtime(config) {
|
||||||
|
Ok(runtime) => runtime,
|
||||||
|
Err(err) => {
|
||||||
|
log::warn!(
|
||||||
|
"[memory_tree::extract] summary extractor: build_chat_runtime failed: \
|
||||||
|
{err:#} — falling back to regex-only"
|
||||||
|
);
|
||||||
|
return Arc::new(CompositeExtractor::regex_only());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let cfg = LlmExtractorConfig {
|
||||||
|
model: model.clone(),
|
||||||
|
emit_topics: true,
|
||||||
|
output_language: config.output_language.clone(),
|
||||||
|
..LlmExtractorConfig::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
log::debug!(
|
||||||
|
"[memory_tree::extract] summary extractor: regex + LLM provider={} model={} \
|
||||||
|
emit_topics=true",
|
||||||
|
provider.name(),
|
||||||
|
model
|
||||||
|
);
|
||||||
|
Arc::new(CompositeExtractor::new(vec![
|
||||||
|
Box::new(RegexEntityExtractor),
|
||||||
|
Box::new(LlmEntityExtractor::new(cfg, provider)),
|
||||||
|
]))
|
||||||
|
}
|
||||||
@@ -22,7 +22,7 @@ use serde::{Deserialize, Serialize};
|
|||||||
use self::extract::{EntityExtractor, ExtractedEntities};
|
use self::extract::{EntityExtractor, ExtractedEntities};
|
||||||
use self::resolver::{canonicalise, CanonicalEntity};
|
use self::resolver::{canonicalise, CanonicalEntity};
|
||||||
use self::signals::{ScoreSignals, SignalWeights};
|
use self::signals::{ScoreSignals, SignalWeights};
|
||||||
use crate::openhuman::memory_tree::types::{approx_token_count, Chunk, SourceKind};
|
use crate::openhuman::memory_store::chunks::types::{approx_token_count, Chunk, SourceKind};
|
||||||
|
|
||||||
/// Default drop threshold. Chunks with `total < DEFAULT_DROP_THRESHOLD`
|
/// Default drop threshold. Chunks with `total < DEFAULT_DROP_THRESHOLD`
|
||||||
/// are tombstoned and never reach the chunk store.
|
/// are tombstoned and never reach the chunk store.
|
||||||
@@ -105,29 +105,20 @@ impl ScoringConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build a [`ScoringConfig`] from the workspace [`Config`]. The
|
/// Build a [`ScoringConfig`] from the workspace [`Config`].
|
||||||
/// resolution rules match `build_summary_extractor`:
|
|
||||||
///
|
///
|
||||||
/// - `llm_backend = "cloud"` (default): always wires the LLM extractor
|
/// The LLM extractor follows the unified summarization workload routing.
|
||||||
/// against the cloud provider, using the configured
|
/// Construction errors fall back to regex-only with a warn log; scoring
|
||||||
/// `cloud_llm_model` (defaulting to `summarization-v1`).
|
/// never blocks on LLM availability.
|
||||||
/// - `llm_backend = "local"`: wires the LLM extractor only when both
|
|
||||||
/// `llm_extractor_endpoint` and `llm_extractor_model` are set;
|
|
||||||
/// otherwise falls back to [`Self::default_regex_only`].
|
|
||||||
///
|
|
||||||
/// Construction errors in the chat provider (rare — only client-builder
|
|
||||||
/// failures) fall back to regex-only with a warn log; scoring never
|
|
||||||
/// blocks on LLM availability.
|
|
||||||
pub fn from_config(config: &crate::openhuman::config::Config) -> Self {
|
pub fn from_config(config: &crate::openhuman::config::Config) -> Self {
|
||||||
use crate::openhuman::memory_tree::chat::{build_chat_provider, ChatConsumer};
|
use crate::openhuman::memory::chat::build_chat_runtime;
|
||||||
|
|
||||||
let model = match extract::resolve_extractor_model(config) {
|
let (provider, model) = match build_chat_runtime(config) {
|
||||||
Some(m) => m,
|
Ok(runtime) => runtime,
|
||||||
None => {
|
Err(err) => {
|
||||||
log::debug!(
|
log::warn!(
|
||||||
"[memory_tree::score] llm_extractor not resolvable for memory_provider={:?} \
|
"[memory::score] build_chat_runtime failed: {err:#} — \
|
||||||
— using regex-only",
|
falling back to regex-only"
|
||||||
config.memory_provider.as_deref().unwrap_or("cloud")
|
|
||||||
);
|
);
|
||||||
return Self::default_regex_only();
|
return Self::default_regex_only();
|
||||||
}
|
}
|
||||||
@@ -139,23 +130,12 @@ impl ScoringConfig {
|
|||||||
..extract::LlmExtractorConfig::default()
|
..extract::LlmExtractorConfig::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
match build_chat_provider(config, ChatConsumer::Extract) {
|
log::info!(
|
||||||
Ok(provider) => {
|
"[memory::score] using LlmEntityExtractor provider={} model={}",
|
||||||
log::info!(
|
provider.name(),
|
||||||
"[memory_tree::score] using LlmEntityExtractor provider={} model={}",
|
model
|
||||||
provider.name(),
|
);
|
||||||
model
|
Self::with_llm_extractor(Arc::new(extract::LlmEntityExtractor::new(cfg, provider)))
|
||||||
);
|
|
||||||
Self::with_llm_extractor(Arc::new(extract::LlmEntityExtractor::new(cfg, provider)))
|
|
||||||
}
|
|
||||||
Err(err) => {
|
|
||||||
log::warn!(
|
|
||||||
"[memory_tree::score] build_chat_provider failed: {err:#} — \
|
|
||||||
falling back to regex-only"
|
|
||||||
);
|
|
||||||
Self::default_regex_only()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -175,7 +155,7 @@ impl ScoringConfig {
|
|||||||
/// 4. Apply final admission gate against `drop_threshold`.
|
/// 4. Apply final admission gate against `drop_threshold`.
|
||||||
pub async fn score_chunk(chunk: &Chunk, cfg: &ScoringConfig) -> Result<ScoreResult> {
|
pub async fn score_chunk(chunk: &Chunk, cfg: &ScoringConfig) -> Result<ScoreResult> {
|
||||||
log::debug!(
|
log::debug!(
|
||||||
"[memory_tree::score] score_chunk chunk_id={} tokens={}",
|
"[memory::score] score_chunk chunk_id={} tokens={}",
|
||||||
chunk.id,
|
chunk.id,
|
||||||
chunk.token_count
|
chunk.token_count
|
||||||
);
|
);
|
||||||
@@ -201,7 +181,7 @@ pub async fn score_chunk(chunk: &Chunk, cfg: &ScoringConfig) -> Result<ScoreResu
|
|||||||
let llm_consulted = if in_band {
|
let llm_consulted = if in_band {
|
||||||
if let Some(llm) = cfg.llm_extractor.as_ref() {
|
if let Some(llm) = cfg.llm_extractor.as_ref() {
|
||||||
log::debug!(
|
log::debug!(
|
||||||
"[memory_tree::score] borderline chunk_id={} cheap_total={:.3} — consulting LLM",
|
"[memory::score] borderline chunk_id={} cheap_total={:.3} — consulting LLM",
|
||||||
chunk.id,
|
chunk.id,
|
||||||
cheap_total
|
cheap_total
|
||||||
);
|
);
|
||||||
@@ -219,7 +199,7 @@ pub async fn score_chunk(chunk: &Chunk, cfg: &ScoringConfig) -> Result<ScoreResu
|
|||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
log::warn!(
|
log::warn!(
|
||||||
"[memory_tree::score] LLM extractor `{}` failed: {e} — \
|
"[memory::score] LLM extractor `{}` failed: {e} — \
|
||||||
falling back to cheap signals only",
|
falling back to cheap signals only",
|
||||||
llm.name()
|
llm.name()
|
||||||
);
|
);
|
||||||
@@ -231,7 +211,7 @@ pub async fn score_chunk(chunk: &Chunk, cfg: &ScoringConfig) -> Result<ScoreResu
|
|||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
log::debug!(
|
log::debug!(
|
||||||
"[memory_tree::score] short-circuit chunk_id={} cheap_total={:.3} \
|
"[memory::score] short-circuit chunk_id={} cheap_total={:.3} \
|
||||||
({}, skipping LLM)",
|
({}, skipping LLM)",
|
||||||
chunk.id,
|
chunk.id,
|
||||||
cheap_total,
|
cheap_total,
|
||||||
@@ -289,7 +269,7 @@ pub async fn score_chunk(chunk: &Chunk, cfg: &ScoringConfig) -> Result<ScoreResu
|
|||||||
|
|
||||||
if !kept {
|
if !kept {
|
||||||
log::debug!(
|
log::debug!(
|
||||||
"[memory_tree::score] drop chunk_id={} total={:.3} reason={:?} llm_consulted={}",
|
"[memory::score] drop chunk_id={} total={:.3} reason={:?} llm_consulted={}",
|
||||||
chunk.id,
|
chunk.id,
|
||||||
total,
|
total,
|
||||||
drop_reason,
|
drop_reason,
|
||||||
+2
-2
@@ -1,5 +1,5 @@
|
|||||||
use super::*;
|
use super::*;
|
||||||
use crate::openhuman::memory_tree::types::{chunk_id, Chunk, Metadata, SourceKind};
|
use crate::openhuman::memory_store::chunks::types::{chunk_id, Chunk, Metadata, SourceKind};
|
||||||
use chrono::Utc;
|
use chrono::Utc;
|
||||||
|
|
||||||
fn test_chunk(content: &str) -> Chunk {
|
fn test_chunk(content: &str) -> Chunk {
|
||||||
@@ -7,7 +7,7 @@ fn test_chunk(content: &str) -> Chunk {
|
|||||||
Chunk {
|
Chunk {
|
||||||
id: chunk_id(SourceKind::Email, "t1", 0, "test-content"),
|
id: chunk_id(SourceKind::Email, "t1", 0, "test-content"),
|
||||||
content: content.to_string(),
|
content: content.to_string(),
|
||||||
token_count: crate::openhuman::memory_tree::types::approx_token_count(content),
|
token_count: crate::openhuman::memory_store::chunks::types::approx_token_count(content),
|
||||||
metadata: meta,
|
metadata: meta,
|
||||||
seq_in_source: 0,
|
seq_in_source: 0,
|
||||||
created_at: Utc::now(),
|
created_at: Utc::now(),
|
||||||
+3
-3
@@ -9,7 +9,7 @@
|
|||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use crate::openhuman::memory_tree::score::extract::{EntityKind, ExtractedEntities};
|
use crate::openhuman::memory::score::extract::{EntityKind, ExtractedEntities};
|
||||||
|
|
||||||
/// Canonicalised entity — same shape as [`ExtractedEntity`] plus a stable
|
/// Canonicalised entity — same shape as [`ExtractedEntity`] plus a stable
|
||||||
/// `canonical_id` suitable for indexing.
|
/// `canonical_id` suitable for indexing.
|
||||||
@@ -103,7 +103,7 @@ pub fn canonical_id_for(kind: EntityKind, surface: &str) -> String {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::openhuman::memory_tree::score::extract::ExtractedEntity;
|
use crate::openhuman::memory::score::extract::ExtractedEntity;
|
||||||
|
|
||||||
fn entity(kind: EntityKind, text: &str) -> ExtractedEntity {
|
fn entity(kind: EntityKind, text: &str) -> ExtractedEntity {
|
||||||
ExtractedEntity {
|
ExtractedEntity {
|
||||||
@@ -173,7 +173,7 @@ mod tests {
|
|||||||
|
|
||||||
// ── Topic canonicalisation (#709 / Phase 3c topic-tree scope) ────
|
// ── Topic canonicalisation (#709 / Phase 3c topic-tree scope) ────
|
||||||
|
|
||||||
use crate::openhuman::memory_tree::score::extract::ExtractedTopic;
|
use crate::openhuman::memory::score::extract::ExtractedTopic;
|
||||||
|
|
||||||
fn topic(label: &str, score: f32) -> ExtractedTopic {
|
fn topic(label: &str, score: f32) -> ExtractedTopic {
|
||||||
ExtractedTopic {
|
ExtractedTopic {
|
||||||
+2
-2
@@ -13,7 +13,7 @@
|
|||||||
//! Ingest adapters can attach these tags during canonicalisation when the
|
//! Ingest adapters can attach these tags during canonicalisation when the
|
||||||
//! upstream source supports the distinction. Absent tags → neutral score.
|
//! upstream source supports the distinction. Absent tags → neutral score.
|
||||||
|
|
||||||
use crate::openhuman::memory_tree::types::Metadata;
|
use crate::openhuman::memory_store::chunks::types::Metadata;
|
||||||
|
|
||||||
/// Tag set when the user replied to this message/thread.
|
/// Tag set when the user replied to this message/thread.
|
||||||
pub const TAG_REPLY: &str = "reply";
|
pub const TAG_REPLY: &str = "reply";
|
||||||
@@ -67,7 +67,7 @@ pub fn score(meta: &Metadata) -> f32 {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::openhuman::memory_tree::types::SourceKind;
|
use crate::openhuman::memory_store::chunks::types::SourceKind;
|
||||||
use chrono::Utc;
|
use chrono::Utc;
|
||||||
|
|
||||||
fn meta(tags: &[&str]) -> Metadata {
|
fn meta(tags: &[&str]) -> Metadata {
|
||||||
+1
-1
@@ -8,7 +8,7 @@
|
|||||||
//! context (e.g., channel size, thread participant count) is a future
|
//! context (e.g., channel size, thread participant count) is a future
|
||||||
//! refinement when we actually have that metadata at ingest.
|
//! refinement when we actually have that metadata at ingest.
|
||||||
|
|
||||||
use crate::openhuman::memory_tree::types::{Metadata, SourceKind};
|
use crate::openhuman::memory_store::chunks::types::{Metadata, SourceKind};
|
||||||
|
|
||||||
/// Base weight for each source kind.
|
/// Base weight for each source kind.
|
||||||
///
|
///
|
||||||
+4
-4
@@ -3,8 +3,8 @@
|
|||||||
|
|
||||||
use super::{interaction, metadata_weight, source_weight, token_count, unique_words};
|
use super::{interaction, metadata_weight, source_weight, token_count, unique_words};
|
||||||
use super::{ScoreSignals, SignalWeights};
|
use super::{ScoreSignals, SignalWeights};
|
||||||
use crate::openhuman::memory_tree::score::extract::ExtractedEntities;
|
use crate::openhuman::memory::score::extract::ExtractedEntities;
|
||||||
use crate::openhuman::memory_tree::types::Metadata;
|
use crate::openhuman::memory_store::chunks::types::Metadata;
|
||||||
|
|
||||||
/// Compute all signals for a chunk.
|
/// Compute all signals for a chunk.
|
||||||
///
|
///
|
||||||
@@ -95,10 +95,10 @@ pub fn combine_cheap_only(signals: &ScoreSignals, w: &SignalWeights) -> f32 {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::openhuman::memory_tree::score::extract::{
|
use crate::openhuman::memory::score::extract::{
|
||||||
EntityKind, ExtractedEntities, ExtractedEntity,
|
EntityKind, ExtractedEntities, ExtractedEntity,
|
||||||
};
|
};
|
||||||
use crate::openhuman::memory_tree::types::SourceKind;
|
use crate::openhuman::memory_store::chunks::types::SourceKind;
|
||||||
use chrono::Utc;
|
use chrono::Utc;
|
||||||
|
|
||||||
fn meta(tags: &[&str], kind: SourceKind) -> Metadata {
|
fn meta(tags: &[&str], kind: SourceKind) -> Metadata {
|
||||||
+1
-1
@@ -10,7 +10,7 @@
|
|||||||
//! Finer distinction (DM vs channel on Slack specifically) requires richer
|
//! Finer distinction (DM vs channel on Slack specifically) requires richer
|
||||||
//! ingest-time metadata and is deferred.
|
//! ingest-time metadata and is deferred.
|
||||||
|
|
||||||
use crate::openhuman::memory_tree::types::{DataSource, Metadata, SourceKind};
|
use crate::openhuman::memory_store::chunks::types::{DataSource, Metadata, SourceKind};
|
||||||
|
|
||||||
const PROVIDER_PREFIX: &str = "provider:";
|
const PROVIDER_PREFIX: &str = "provider:";
|
||||||
|
|
||||||
+42
@@ -63,3 +63,45 @@ impl SignalWeights {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn score_signals_default_to_zero() {
|
||||||
|
let signals = ScoreSignals::default();
|
||||||
|
assert_eq!(signals.token_count, 0.0);
|
||||||
|
assert_eq!(signals.unique_words, 0.0);
|
||||||
|
assert_eq!(signals.metadata_weight, 0.0);
|
||||||
|
assert_eq!(signals.source_weight, 0.0);
|
||||||
|
assert_eq!(signals.interaction, 0.0);
|
||||||
|
assert_eq!(signals.entity_density, 0.0);
|
||||||
|
assert_eq!(signals.llm_importance, 0.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn signal_weights_default_match_expected_priorities() {
|
||||||
|
let weights = SignalWeights::default();
|
||||||
|
assert_eq!(weights.token_count, 1.0);
|
||||||
|
assert_eq!(weights.unique_words, 1.0);
|
||||||
|
assert_eq!(weights.metadata_weight, 1.5);
|
||||||
|
assert_eq!(weights.source_weight, 1.5);
|
||||||
|
assert_eq!(weights.interaction, 3.0);
|
||||||
|
assert_eq!(weights.entity_density, 1.0);
|
||||||
|
assert_eq!(weights.llm_importance, 0.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn with_llm_enabled_only_changes_llm_weight() {
|
||||||
|
let default = SignalWeights::default();
|
||||||
|
let enabled = SignalWeights::with_llm_enabled();
|
||||||
|
assert_eq!(enabled.token_count, default.token_count);
|
||||||
|
assert_eq!(enabled.unique_words, default.unique_words);
|
||||||
|
assert_eq!(enabled.metadata_weight, default.metadata_weight);
|
||||||
|
assert_eq!(enabled.source_weight, default.source_weight);
|
||||||
|
assert_eq!(enabled.interaction, default.interaction);
|
||||||
|
assert_eq!(enabled.entity_density, default.entity_density);
|
||||||
|
assert_eq!(enabled.llm_importance, 2.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -14,10 +14,10 @@ use serde::{Deserialize, Serialize};
|
|||||||
|
|
||||||
use crate::openhuman::composio::providers::profile::{is_self_identity_any_toolkit, IdentityKind};
|
use crate::openhuman::composio::providers::profile::{is_self_identity_any_toolkit, IdentityKind};
|
||||||
use crate::openhuman::config::Config;
|
use crate::openhuman::config::Config;
|
||||||
use crate::openhuman::memory_tree::score::extract::EntityKind;
|
use crate::openhuman::memory::score::extract::EntityKind;
|
||||||
use crate::openhuman::memory_tree::score::resolver::CanonicalEntity;
|
use crate::openhuman::memory::score::resolver::CanonicalEntity;
|
||||||
use crate::openhuman::memory_tree::score::signals::ScoreSignals;
|
use crate::openhuman::memory::score::signals::ScoreSignals;
|
||||||
use crate::openhuman::memory_tree::store::with_connection;
|
use crate::openhuman::memory_store::chunks::store::with_connection;
|
||||||
|
|
||||||
/// Map a memory-tree `EntityKind` to the Composio identity-registry
|
/// Map a memory-tree `EntityKind` to the Composio identity-registry
|
||||||
/// [`IdentityKind`] used for self-matching, or `None` for kinds that
|
/// [`IdentityKind`] used for self-matching, or `None` for kinds that
|
||||||
@@ -304,7 +304,7 @@ pub(crate) fn index_summary_entity_ids_tx(
|
|||||||
Some((kind, _)) => kind,
|
Some((kind, _)) => kind,
|
||||||
None => {
|
None => {
|
||||||
log::warn!(
|
log::warn!(
|
||||||
"[memory_tree::score::store] summary entity id missing ':' — \
|
"[memory::score::store] summary entity id missing ':' — \
|
||||||
storing as-is: {canonical_id}"
|
storing as-is: {canonical_id}"
|
||||||
);
|
);
|
||||||
canonical_id.as_str()
|
canonical_id.as_str()
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user