From 2484ea40f23feb8e85f3a0109e97e9a7e9885d92 Mon Sep 17 00:00:00 2001 From: Steven Enamakel <31011319+senamakel@users.noreply.github.com> Date: Sun, 3 May 2026 15:22:22 -0700 Subject: [PATCH] feat(memory): ingestion pipeline + tree-architecture docs + ops/schemas split (#1142) --- src/openhuman/channels/tests/memory.rs | 3 +- src/openhuman/config/schema/storage_memory.rs | 5 +- src/openhuman/memory/README.md | 44 + src/openhuman/memory/conversations/README.md | 34 + src/openhuman/memory/conversations/bus.rs | 4 + src/openhuman/memory/conversations/store.rs | 29 + .../memory/conversations/store_tests.rs | 3 + src/openhuman/memory/conversations/types.rs | 7 + src/openhuman/memory/embeddings.rs | 7 - src/openhuman/memory/global.rs | 63 +- src/openhuman/memory/ingestion/README.md | 18 + .../memory/{ingestion.rs => ingestion/mod.rs} | 27 +- .../parse.rs} | 4 +- .../queue.rs} | 4 +- .../regex.rs} | 0 .../rules.rs} | 4 +- .../state.rs} | 0 .../tests.rs} | 7 +- .../types.rs} | 0 src/openhuman/memory/mod.rs | 10 +- src/openhuman/memory/ops.rs | 1414 ----------------- src/openhuman/memory/ops/documents.rs | 491 ++++++ src/openhuman/memory/ops/envelope.rs | 82 + src/openhuman/memory/ops/files.rs | 104 ++ src/openhuman/memory/ops/helpers.rs | 473 ++++++ src/openhuman/memory/ops/kv_graph.rs | 136 ++ src/openhuman/memory/ops/learn.rs | 152 ++ src/openhuman/memory/ops/mod.rs | 69 + src/openhuman/memory/ops/sync.rs | 112 ++ src/openhuman/memory/ops_tests.rs | 8 +- src/openhuman/memory/response_cache.rs | 236 --- src/openhuman/memory/rpc_models.rs | 6 +- src/openhuman/memory/rpc_models_tests.rs | 6 +- src/openhuman/memory/schemas.rs | 1292 --------------- src/openhuman/memory/schemas/documents.rs | 537 +++++++ src/openhuman/memory/schemas/files.rs | 128 ++ src/openhuman/memory/schemas/kv_graph.rs | 269 ++++ src/openhuman/memory/schemas/learn.rs | 56 + src/openhuman/memory/schemas/mod.rs | 109 ++ src/openhuman/memory/schemas/sync.rs | 87 + src/openhuman/memory/schemas_tests.rs | 3 + .../memory/slack_ingestion/README.md | 37 + .../memory/slack_ingestion/bucketer.rs | 2 +- src/openhuman/memory/slack_ingestion/rpc.rs | 6 + .../memory/slack_ingestion/schemas.rs | 3 + src/openhuman/memory/store/README.md | 17 + src/openhuman/memory/store/client.rs | 8 +- src/openhuman/memory/store/client_tests.rs | 3 + src/openhuman/memory/store/factories.rs | 2 +- src/openhuman/memory/store/memory_trait.rs | 2 +- src/openhuman/memory/store/types.rs | 20 + src/openhuman/memory/store/unified/README.md | 22 + .../memory/store/unified/documents.rs | 26 +- .../memory/store/unified/documents_tests.rs | 5 +- src/openhuman/memory/store/unified/events.rs | 3 + .../memory/store/unified/events_tests.rs | 2 + src/openhuman/memory/store/unified/graph.rs | 12 + src/openhuman/memory/store/unified/helpers.rs | 10 +- src/openhuman/memory/store/unified/init.rs | 18 +- src/openhuman/memory/store/unified/kv.rs | 12 + src/openhuman/memory/store/unified/mod.rs | 7 +- src/openhuman/memory/store/unified/profile.rs | 3 + .../memory/store/unified/profile_tests.rs | 2 + src/openhuman/memory/store/unified/query.rs | 19 + .../memory/store/unified/query_tests.rs | 5 +- .../memory/store/unified/segments.rs | 4 + .../memory/store/unified/segments_tests.rs | 2 + src/openhuman/memory/tree/README.md | 57 + .../memory/tree/canonicalize/README.md | 17 + .../memory/tree/canonicalize/document.rs | 3 + .../memory/tree/canonicalize/email.rs | 3 + src/openhuman/memory/tree/canonicalize/mod.rs | 2 + src/openhuman/memory/tree/chunker.rs | 2 +- .../memory/tree/content_store/README.md | 18 + .../memory/tree/content_store/atomic.rs | 1 + .../memory/tree/content_store/compose.rs | 9 + src/openhuman/memory/tree/ingest.rs | 6 + src/openhuman/memory/tree/jobs/README.md | 36 + .../memory/tree/jobs/handlers/README.md | 20 + .../memory/tree/jobs/handlers/mod.rs | 52 +- src/openhuman/memory/tree/jobs/mod.rs | 2 +- src/openhuman/memory/tree/jobs/scheduler.rs | 5 + src/openhuman/memory/tree/jobs/testing.rs | 2 + src/openhuman/memory/tree/jobs/types.rs | 27 +- src/openhuman/memory/tree/jobs/worker.rs | 9 + src/openhuman/memory/tree/mod.rs | 6 +- src/openhuman/memory/tree/retrieval/README.md | 30 + .../memory/tree/retrieval/drill_down.rs | 20 +- src/openhuman/memory/tree/retrieval/global.rs | 18 +- .../memory/tree/retrieval/integration_test.rs | 10 +- src/openhuman/memory/tree/retrieval/rpc.rs | 19 + .../memory/tree/retrieval/schemas.rs | 8 +- src/openhuman/memory/tree/retrieval/source.rs | 26 +- src/openhuman/memory/tree/retrieval/topic.rs | 12 +- src/openhuman/memory/tree/retrieval/types.rs | 4 +- src/openhuman/memory/tree/rpc.rs | 10 + src/openhuman/memory/tree/schemas.rs | 5 + src/openhuman/memory/tree/score/README.md | 23 + .../memory/tree/score/embed/README.md | 18 + .../memory/tree/score/embed/inert.rs | 1 + .../memory/tree/score/embed/ollama.rs | 3 +- .../memory/tree/score/extract/README.md | 20 + .../memory/tree/score/extract/extractor.rs | 5 + .../memory/tree/score/extract/llm.rs | 2 + .../memory/tree/score/extract/types.rs | 3 + .../memory/tree/score/signals/README.md | 14 + .../memory/tree/score/signals/interaction.rs | 4 + .../memory/tree/score/signals/ops.rs | 3 + .../memory/tree/score/signals/token_count.rs | 13 +- .../memory/tree/score/signals/types.rs | 4 + .../memory/tree/score/signals/unique_words.rs | 2 + src/openhuman/memory/tree/score/store.rs | 3 + src/openhuman/memory/tree/store.rs | 8 + src/openhuman/memory/tree/store_tests.rs | 3 + .../memory/tree/tree_global/README.md | 20 + .../{global_tree => tree_global}/digest.rs | 38 +- .../digest_tests.rs | 14 +- .../tree/{global_tree => tree_global}/mod.rs | 0 .../{global_tree => tree_global}/recap.rs | 22 +- .../{global_tree => tree_global}/registry.rs | 18 +- .../tree/{global_tree => tree_global}/seal.rs | 34 +- .../memory/tree/tree_source/README.md | 25 + .../bucket_seal.rs | 36 +- .../bucket_seal_tests.rs | 14 +- .../{source_tree => tree_source}/flush.rs | 18 +- .../tree/{source_tree => tree_source}/mod.rs | 0 .../{source_tree => tree_source}/registry.rs | 10 +- .../{source_tree => tree_source}/store.rs | 6 +- .../store_tests.rs | 3 + .../tree/tree_source/summariser/README.md | 16 + .../summariser/inert.rs | 11 +- .../summariser/llm.rs | 20 +- .../summariser/mod.rs | 8 +- .../{source_tree => tree_source}/types.rs | 6 + .../memory/tree/tree_topic/README.md | 24 + .../{topic_tree => tree_topic}/backfill.rs | 32 +- .../{topic_tree => tree_topic}/curator.rs | 32 +- .../{topic_tree => tree_topic}/hotness.rs | 6 +- .../tree/{topic_tree => tree_topic}/mod.rs | 2 +- .../{topic_tree => tree_topic}/registry.rs | 39 +- .../{topic_tree => tree_topic}/routing.rs | 46 +- .../tree/{topic_tree => tree_topic}/store.rs | 4 +- .../tree/{topic_tree => tree_topic}/types.rs | 0 src/openhuman/memory/tree/types.rs | 1 + src/openhuman/memory/tree/util/README.md | 12 + src/openhuman/screen_intelligence/tests.rs | 2 +- src/openhuman/tools/impl/memory/forget.rs | 3 +- src/openhuman/tools/impl/memory/recall.rs | 3 +- src/openhuman/tools/impl/memory/store.rs | 3 +- tests/json_rpc_e2e.rs | 4 + tests/memory_graph_sync_e2e.rs | 5 +- tests/screen_intelligence_vision_e2e.rs | 4 +- tests/subconscious_e2e.rs | 2 +- 153 files changed, 4116 insertions(+), 3290 deletions(-) create mode 100644 src/openhuman/memory/conversations/README.md delete mode 100644 src/openhuman/memory/embeddings.rs create mode 100644 src/openhuman/memory/ingestion/README.md rename src/openhuman/memory/{ingestion.rs => ingestion/mod.rs} (91%) rename src/openhuman/memory/{ingestion_parse.rs => ingestion/parse.rs} (99%) rename src/openhuman/memory/{ingestion_queue.rs => ingestion/queue.rs} (98%) rename src/openhuman/memory/{ingestion_regex.rs => ingestion/regex.rs} (100%) rename src/openhuman/memory/{ingestion_rules.rs => ingestion/rules.rs} (98%) rename src/openhuman/memory/{ingestion_state.rs => ingestion/state.rs} (100%) rename src/openhuman/memory/{ingestion_tests.rs => ingestion/tests.rs} (96%) rename src/openhuman/memory/{ingestion_types.rs => ingestion/types.rs} (100%) delete mode 100644 src/openhuman/memory/ops.rs create mode 100644 src/openhuman/memory/ops/documents.rs create mode 100644 src/openhuman/memory/ops/envelope.rs create mode 100644 src/openhuman/memory/ops/files.rs create mode 100644 src/openhuman/memory/ops/helpers.rs create mode 100644 src/openhuman/memory/ops/kv_graph.rs create mode 100644 src/openhuman/memory/ops/learn.rs create mode 100644 src/openhuman/memory/ops/mod.rs create mode 100644 src/openhuman/memory/ops/sync.rs delete mode 100644 src/openhuman/memory/response_cache.rs delete mode 100644 src/openhuman/memory/schemas.rs create mode 100644 src/openhuman/memory/schemas/documents.rs create mode 100644 src/openhuman/memory/schemas/files.rs create mode 100644 src/openhuman/memory/schemas/kv_graph.rs create mode 100644 src/openhuman/memory/schemas/learn.rs create mode 100644 src/openhuman/memory/schemas/mod.rs create mode 100644 src/openhuman/memory/schemas/sync.rs create mode 100644 src/openhuman/memory/slack_ingestion/README.md create mode 100644 src/openhuman/memory/store/README.md create mode 100644 src/openhuman/memory/store/unified/README.md create mode 100644 src/openhuman/memory/tree/README.md create mode 100644 src/openhuman/memory/tree/canonicalize/README.md create mode 100644 src/openhuman/memory/tree/content_store/README.md create mode 100644 src/openhuman/memory/tree/jobs/README.md create mode 100644 src/openhuman/memory/tree/jobs/handlers/README.md create mode 100644 src/openhuman/memory/tree/retrieval/README.md create mode 100644 src/openhuman/memory/tree/score/README.md create mode 100644 src/openhuman/memory/tree/score/embed/README.md create mode 100644 src/openhuman/memory/tree/score/extract/README.md create mode 100644 src/openhuman/memory/tree/score/signals/README.md create mode 100644 src/openhuman/memory/tree/tree_global/README.md rename src/openhuman/memory/tree/{global_tree => tree_global}/digest.rs (92%) rename src/openhuman/memory/tree/{global_tree => tree_global}/digest_tests.rs (96%) rename src/openhuman/memory/tree/{global_tree => tree_global}/mod.rs (100%) rename src/openhuman/memory/tree/{global_tree => tree_global}/recap.rs (95%) rename src/openhuman/memory/tree/{global_tree => tree_global}/registry.rs (88%) rename src/openhuman/memory/tree/{global_tree => tree_global}/seal.rs (95%) create mode 100644 src/openhuman/memory/tree/tree_source/README.md rename src/openhuman/memory/tree/{source_tree => tree_source}/bucket_seal.rs (96%) rename src/openhuman/memory/tree/{source_tree => tree_source}/bucket_seal_tests.rs (97%) rename src/openhuman/memory/tree/{source_tree => tree_source}/flush.rs (93%) rename src/openhuman/memory/tree/{source_tree => tree_source}/mod.rs (100%) rename src/openhuman/memory/tree/{source_tree => tree_source}/registry.rs (95%) rename src/openhuman/memory/tree/{source_tree => tree_source}/store.rs (99%) rename src/openhuman/memory/tree/{source_tree => tree_source}/store_tests.rs (97%) create mode 100644 src/openhuman/memory/tree/tree_source/summariser/README.md rename src/openhuman/memory/tree/{source_tree => tree_source}/summariser/inert.rs (92%) rename src/openhuman/memory/tree/{source_tree => tree_source}/summariser/llm.rs (96%) rename src/openhuman/memory/tree/{source_tree => tree_source}/summariser/mod.rs (94%) rename src/openhuman/memory/tree/{source_tree => tree_source}/types.rs (95%) create mode 100644 src/openhuman/memory/tree/tree_topic/README.md rename src/openhuman/memory/tree/{topic_tree => tree_topic}/backfill.rs (94%) rename src/openhuman/memory/tree/{topic_tree => tree_topic}/curator.rs (92%) rename src/openhuman/memory/tree/{topic_tree => tree_topic}/hotness.rs (97%) rename src/openhuman/memory/tree/{topic_tree => tree_topic}/mod.rs (96%) rename src/openhuman/memory/tree/{topic_tree => tree_topic}/registry.rs (88%) rename src/openhuman/memory/tree/{topic_tree => tree_topic}/routing.rs (90%) rename src/openhuman/memory/tree/{topic_tree => tree_topic}/store.rs (98%) rename src/openhuman/memory/tree/{topic_tree => tree_topic}/types.rs (100%) create mode 100644 src/openhuman/memory/tree/util/README.md diff --git a/src/openhuman/channels/tests/memory.rs b/src/openhuman/channels/tests/memory.rs index ec17c57bb..e83b89075 100644 --- a/src/openhuman/channels/tests/memory.rs +++ b/src/openhuman/channels/tests/memory.rs @@ -5,7 +5,8 @@ use super::super::context::{ use super::super::runtime::process_channel_message; use super::super::{traits, Channel}; use super::common::{HistoryCaptureProvider, NoopMemory, RecordingChannel}; -use crate::openhuman::memory::{embeddings::NoopEmbedding, Memory, MemoryCategory, UnifiedMemory}; +use crate::openhuman::embeddings::NoopEmbedding; +use crate::openhuman::memory::{Memory, MemoryCategory, UnifiedMemory}; use crate::openhuman::providers; use std::collections::HashMap; use std::sync::{Arc, Mutex}; diff --git a/src/openhuman/config/schema/storage_memory.rs b/src/openhuman/config/schema/storage_memory.rs index 574806bd0..ff8ef9b87 100644 --- a/src/openhuman/config/schema/storage_memory.rs +++ b/src/openhuman/config/schema/storage_memory.rs @@ -44,8 +44,6 @@ pub struct MemoryConfig { #[serde(default = "default_min_relevance_score")] pub min_relevance_score: f64, #[serde(default)] - pub response_cache_enabled: bool, - #[serde(default)] pub sqlite_open_timeout_secs: Option, } @@ -71,7 +69,6 @@ impl Default for MemoryConfig { embedding_model: default_embedding_model(), embedding_dimensions: default_embedding_dims(), min_relevance_score: default_min_relevance_score(), - response_cache_enabled: false, sqlite_open_timeout_secs: None, } } @@ -138,7 +135,7 @@ pub struct MemoryTreeConfig { pub llm_extractor_timeout_ms: Option, /// Ollama endpoint for the summariser - /// (`memory::tree::source_tree::summariser::llm::LlmSummariser`). + /// (`memory::tree::tree_source::summariser::llm::LlmSummariser`). /// When unset, bucket-seal cascades use `InertSummariser` — sealed /// nodes contain concatenated+truncated child text instead of a /// real LLM summary. Soft failures fall back to inert per seal. diff --git a/src/openhuman/memory/README.md b/src/openhuman/memory/README.md index 48d6993d1..d48fc7cf1 100644 --- a/src/openhuman/memory/README.md +++ b/src/openhuman/memory/README.md @@ -2,6 +2,50 @@ Persistent knowledge layer. Owns the unified store (SQLite + FTS5 + vector embeddings + graph relations), document ingestion pipelines, namespace + KV operations, conversation history, and retrieval scoring. Does NOT own raw provider embedding APIs (`local_ai/`), agent prompt assembly (`agent/memory_loader.rs`), or per-channel ingestion adapters beyond the bundled Slack importer. +## Architecture + +The module is organised in concentric layers — the contract on the +inside, the persistent backend around it, the ingestion + retrieval +pipelines on top, and the per-domain glue at the edge: + +```text + ┌──────────────────────────────────────┐ + │ conversations/ slack_ingestion/ │ per-domain plumbing + ├──────────────────────────────────────┤ + │ tree/ (bucket-seal LLD pipeline) │ new retrieval architecture + ├──────────────────────────────────────┤ + │ ingestion/ (extract chunks) │ document ingestion + ├──────────────────────────────────────┤ + │ store/ (UnifiedMemory backend) │ SQLite + FTS5 + vectors + ├──────────────────────────────────────┤ + │ traits.rs (Memory trait) │ contract + └──────────────────────────────────────┘ +``` + +- **`traits.rs`** — `Memory`, `MemoryEntry`, `MemoryCategory`, + `RecallOpts`. The backend-agnostic contract every store implements. +- **`store/`** — `UnifiedMemory` is the production backend (SQLite + with FTS5 for keyword search, vector tables for embeddings, and + graph tables for entity/relation triples) plus the `MemoryClient` + handle used by the rest of the process. +- **`ingestion/`** — chunking + extraction pipeline (entities, + relations, embeddings) and the background `IngestionQueue` worker. +- **`tree/`** — the new bucket-seal retrieval architecture from + `docs/MEMORY_ARCHITECTURE_LLD.md`: `canonicalize` (normalise + inputs), `chunker` and `content_store` (durable chunks), + `score`/`retrieval` (ranking surface), + `tree_source`/`tree_topic`/`tree_global` (the three concentric + trees the LLD calls for), and `jobs` (background seals/summaries). +- **`conversations/`** — workspace-backed JSONL chat thread/message + history. See `conversations/README.md`. +- **`slack_ingestion/`** — Slack provider plumbing (bucketer + + ingest wrapper + RPC). See `slack_ingestion/README.md`. + +The legacy memory store (`store/` + `ingestion/`) and the new +`tree/` pipeline coexist for now — `tree/` is replacing the older +retrieval surface incrementally and both must remain wired into RPC +until the migration completes. + ## Public surface - `pub trait Memory` / `pub struct MemoryEntry` / `pub enum MemoryCategory` / `pub struct RecallOpts` — `traits.rs:11-100` — backend contract for any memory store. diff --git a/src/openhuman/memory/conversations/README.md b/src/openhuman/memory/conversations/README.md new file mode 100644 index 000000000..d98ab0306 --- /dev/null +++ b/src/openhuman/memory/conversations/README.md @@ -0,0 +1,34 @@ +# conversations + +Workspace-backed conversation thread/message storage. Lives at +`/memory/conversations/` as plain JSONL — easy to inspect, +recover, and back up. Used by the desktop UI for chat threads and by +non-web channel adapters (Slack, Telegram, …) so all surfaces share one +persistence path. + +## Files + +- **`mod.rs`** — re-exports the public surface + (`ConversationStore`, `ConversationThread`, `ConversationMessage`, + `CreateConversationThread`, `ConversationMessagePatch`, + `ConversationPurgeStats`, free-function shims, and + `register_conversation_persistence_subscriber`). +- **`types.rs`** — wire/storage structs: thread metadata, message + records, create requests, partial-update patches. +- **`store.rs`** — `ConversationStore` plus free-function shims. + Thread metadata is appended to `threads.jsonl` (upsert/delete log); + messages live in `threads/.jsonl`. A process-wide mutex + serialises every on-disk mutation. +- **`bus.rs`** — `EventHandler` that mirrors inbound `DomainEvent` + channel messages into the store, so non-web providers persist + alongside UI-driven threads. +- **`store_tests.rs`** — unit tests covering upsert, append, label/ + title updates, deletion, and purge. + +## Where it fits + +Sits next to the unified memory store but is intentionally separate: +the conversation log is append-only chat history with no embeddings or +graph relations. Ingestion into the searchable memory tree happens via +`tree/` and the per-provider ingestion modules (e.g. `slack_ingestion/`) +— this folder only owns durable transcript storage. diff --git a/src/openhuman/memory/conversations/bus.rs b/src/openhuman/memory/conversations/bus.rs index 4681ba9cb..95b6237ca 100644 --- a/src/openhuman/memory/conversations/bus.rs +++ b/src/openhuman/memory/conversations/bus.rs @@ -1,3 +1,7 @@ +//! Event-bus subscriber that mirrors inbound channel messages into the +//! workspace-backed conversation store, so non-web channels (Slack, Telegram, +//! etc.) persist alongside UI-driven threads. + use std::path::{Path, PathBuf}; use std::sync::{Arc, OnceLock}; diff --git a/src/openhuman/memory/conversations/store.rs b/src/openhuman/memory/conversations/store.rs index 88599084f..18428595f 100644 --- a/src/openhuman/memory/conversations/store.rs +++ b/src/openhuman/memory/conversations/store.rs @@ -1,3 +1,10 @@ +//! JSONL-backed thread and message store. Thread metadata lives in +//! `threads.jsonl` (append-only upsert/delete log); each thread's messages +//! are appended to a per-thread JSONL file under `threads/.jsonl`. +//! +//! All on-disk mutations serialise through a single process-wide mutex so +//! concurrent RPC handlers don't interleave writes. + use std::collections::BTreeMap; use std::fs::{self, File, OpenOptions}; use std::hash::{Hash, Hasher}; @@ -28,12 +35,14 @@ fn redact_title_for_log(title: &str) -> String { ) } +/// Counts returned by [`purge_threads`] — how much was deleted. #[derive(Debug, Clone, Copy, Default)] pub struct ConversationPurgeStats { pub thread_count: usize, pub message_count: usize, } +/// Workspace-rooted handle that reads and writes the JSONL conversation log. #[derive(Debug, Clone)] pub struct ConversationStore { workspace_dir: PathBuf, @@ -57,10 +66,12 @@ enum ThreadLogEntry { } impl ConversationStore { + /// Construct a store rooted at the given workspace directory. pub fn new(workspace_dir: PathBuf) -> Self { Self { workspace_dir } } + /// Create or update a thread, appending an `Upsert` entry to `threads.jsonl`. pub fn ensure_thread( &self, request: CreateConversationThread, @@ -88,11 +99,13 @@ impl ConversationStore { .ok_or_else(|| format!("thread {} missing after ensure", request.id)) } + /// List all live threads (folding the upsert/delete log). pub fn list_threads(&self) -> Result, String> { let _guard = CONVERSATION_STORE_LOCK.lock(); self.list_threads_unlocked() } + /// Read every persisted message for a thread in append order. pub fn get_messages(&self, thread_id: &str) -> Result, String> { let _guard = CONVERSATION_STORE_LOCK.lock(); if !self.thread_exists_unlocked(thread_id)? { @@ -101,6 +114,7 @@ impl ConversationStore { read_jsonl::(&self.thread_messages_path(thread_id)) } + /// Append a message to the thread's JSONL file. Errors if the thread is missing. pub fn append_message( &self, thread_id: &str, @@ -125,6 +139,7 @@ impl ConversationStore { Ok(message) } + /// Rewrite the thread title via a new `Upsert` log entry, preserving labels. pub fn update_thread_title( &self, thread_id: &str, @@ -157,6 +172,7 @@ impl ConversationStore { .ok_or_else(|| format!("thread {} missing after title update", thread_id)) } + /// Replace the label set on a thread via a new `Upsert` log entry. pub fn update_thread_labels( &self, thread_id: &str, @@ -188,6 +204,7 @@ impl ConversationStore { .ok_or_else(|| format!("thread {} missing after labels update", thread_id)) } + /// Apply a patch to one message and rewrite the thread's JSONL file in place. pub fn update_message( &self, thread_id: &str, @@ -219,6 +236,8 @@ impl ConversationStore { Ok(updated) } + /// Append a `Delete` entry and remove the thread's messages file. Returns + /// `false` if the thread did not exist. pub fn delete_thread(&self, thread_id: &str, deleted_at: &str) -> Result { let _guard = CONVERSATION_STORE_LOCK.lock(); if !self.thread_exists_unlocked(thread_id)? { @@ -252,6 +271,7 @@ impl ConversationStore { Ok(true) } + /// Wipe the entire conversation directory and re-create an empty layout. pub fn purge_threads(&self) -> Result { let _guard = CONVERSATION_STORE_LOCK.lock(); let stats = self.purge_stats_unlocked()?; @@ -486,6 +506,7 @@ where Ok(()) } +/// Free-function shim around [`ConversationStore::ensure_thread`]. pub fn ensure_thread( workspace_dir: PathBuf, request: CreateConversationThread, @@ -493,10 +514,12 @@ pub fn ensure_thread( ConversationStore::new(workspace_dir).ensure_thread(request) } +/// Free-function shim around [`ConversationStore::list_threads`]. pub fn list_threads(workspace_dir: PathBuf) -> Result, String> { ConversationStore::new(workspace_dir).list_threads() } +/// Free-function shim around [`ConversationStore::get_messages`]. pub fn get_messages( workspace_dir: PathBuf, thread_id: &str, @@ -504,6 +527,7 @@ pub fn get_messages( ConversationStore::new(workspace_dir).get_messages(thread_id) } +/// Free-function shim around [`ConversationStore::append_message`]. pub fn append_message( workspace_dir: PathBuf, thread_id: &str, @@ -512,6 +536,7 @@ pub fn append_message( ConversationStore::new(workspace_dir).append_message(thread_id, message) } +/// Free-function shim around [`ConversationStore::update_thread_title`]. pub fn update_thread_title( workspace_dir: PathBuf, thread_id: &str, @@ -521,6 +546,7 @@ pub fn update_thread_title( ConversationStore::new(workspace_dir).update_thread_title(thread_id, title, updated_at) } +/// Free-function shim around [`ConversationStore::update_thread_labels`]. pub fn update_thread_labels( workspace_dir: PathBuf, thread_id: &str, @@ -530,6 +556,7 @@ pub fn update_thread_labels( ConversationStore::new(workspace_dir).update_thread_labels(thread_id, labels, updated_at) } +/// Free-function shim around [`ConversationStore::update_message`]. pub fn update_message( workspace_dir: PathBuf, thread_id: &str, @@ -539,10 +566,12 @@ pub fn update_message( ConversationStore::new(workspace_dir).update_message(thread_id, message_id, patch) } +/// Free-function shim around [`ConversationStore::purge_threads`]. pub fn purge_threads(workspace_dir: PathBuf) -> Result { ConversationStore::new(workspace_dir).purge_threads() } +/// Free-function shim around [`ConversationStore::delete_thread`]. pub fn delete_thread( workspace_dir: PathBuf, thread_id: &str, diff --git a/src/openhuman/memory/conversations/store_tests.rs b/src/openhuman/memory/conversations/store_tests.rs index 234fc9d6c..626eb70b8 100644 --- a/src/openhuman/memory/conversations/store_tests.rs +++ b/src/openhuman/memory/conversations/store_tests.rs @@ -1,3 +1,6 @@ +//! Unit tests for the JSONL-backed [`ConversationStore`], exercising thread +//! upsert, message append, label/title updates, deletion and purge semantics. + use tempfile::TempDir; use super::*; diff --git a/src/openhuman/memory/conversations/types.rs b/src/openhuman/memory/conversations/types.rs index d4148d650..0beed6b5a 100644 --- a/src/openhuman/memory/conversations/types.rs +++ b/src/openhuman/memory/conversations/types.rs @@ -1,6 +1,10 @@ +//! Wire/storage types for the workspace-backed conversation store: threads, +//! messages, create requests, and partial-update patches. + use serde::{Deserialize, Serialize}; use serde_json::Value; +/// A persisted conversation thread, mirroring one entry in `threads.jsonl`. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct ConversationThread { @@ -16,6 +20,7 @@ pub struct ConversationThread { pub labels: Vec, } +/// A single message appended to a thread's JSONL log. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct ConversationMessage { @@ -29,6 +34,7 @@ pub struct ConversationMessage { pub created_at: String, } +/// Input payload to create-or-update a thread via [`super::ensure_thread`]. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct CreateConversationThread { @@ -39,6 +45,7 @@ pub struct CreateConversationThread { pub labels: Option>, } +/// Partial update to apply to a stored message (e.g. rewriting `extraMetadata`). #[derive(Debug, Clone, Serialize, Deserialize, Default)] #[serde(rename_all = "camelCase")] pub struct ConversationMessagePatch { diff --git a/src/openhuman/memory/embeddings.rs b/src/openhuman/memory/embeddings.rs deleted file mode 100644 index b096591b5..000000000 --- a/src/openhuman/memory/embeddings.rs +++ /dev/null @@ -1,7 +0,0 @@ -//! Re-exports from the top-level `openhuman::embeddings` module. -//! -//! The canonical embedding logic now lives in `src/openhuman/embeddings/`. -//! This file keeps the old `memory::embeddings::*` import paths working so -//! that existing call sites do not need to change immediately. - -pub use crate::openhuman::embeddings::*; diff --git a/src/openhuman/memory/global.rs b/src/openhuman/memory/global.rs index f3ddd064d..f42c94596 100644 --- a/src/openhuman/memory/global.rs +++ b/src/openhuman/memory/global.rs @@ -47,7 +47,14 @@ pub fn init(workspace_dir: PathBuf) -> Result { Ok(GLOBAL_CLIENT.get().cloned().unwrap_or(client)) } -/// Initialise using the default `.openhuman/workspace` directory. +/// Initialise using the default `~/.openhuman/workspace` directory. +/// +/// **TEST-ONLY.** Production code must call [`init`] with the real workspace +/// directory at startup wiring. If this function ran first in production it +/// would pin the singleton to `~/.openhuman/workspace`, causing every +/// subsequent `init(custom_workspace)` to silently no-op and return the wrong +/// handle (`OnceLock::set` is one-shot). +#[cfg(test)] pub fn init_default() -> Result { let workspace_dir = crate::openhuman::config::default_root_openhuman_dir() .map_err(|e| e.to_string())? @@ -55,16 +62,27 @@ pub fn init_default() -> Result { init(workspace_dir) } -/// Returns the global memory client, lazily initialising with default paths -/// if not yet set up. +/// Returns the global memory client. /// -/// Prefer calling `init()` explicitly at startup so errors surface early. +/// Returns `Err` if [`init`] has not yet been called. There is **no** lazy +/// fallback: a fallback would pin the global to `~/.openhuman/workspace` on +/// the first stray call (test, early RPC, etc.), and `OnceLock::set` is +/// one-shot, so the real `init(custom_workspace)` would silently no-op +/// afterwards and every caller would get the wrong workspace. +/// +/// Callers that can tolerate "not yet ready" should use +/// [`client_if_ready`] instead. pub fn client() -> Result { - if let Some(c) = GLOBAL_CLIENT.get() { - return Ok(Arc::clone(c)); - } - // Lazy fallback — initialise with defaults. - init_default() + client_from(&GLOBAL_CLIENT) +} + +/// Implementation backing [`client`] — extracted so unit tests can pass a +/// freshly-constructed local `OnceLock` and assert the uninitialised-error +/// contract without racing the process-global singleton. +fn client_from(slot: &OnceLock) -> Result { + slot.get().cloned().ok_or_else(|| { + "memory global accessed before init — call init(workspace) at startup".to_string() + }) } /// Returns the global client if already initialised, without lazy init. @@ -107,13 +125,30 @@ mod tests { } #[tokio::test] - async fn client_returns_a_handle_either_via_lazy_init_or_existing() { - // Bind TempDir at test scope so its directory outlives any lazy - // init — the global client holds the path and can be used later in - // this test (and potentially by other tests in the same binary). + async fn client_returns_a_handle_after_explicit_init() { + // Bind TempDir at test scope so its directory outlives the global + // client — the singleton holds the path and may be used later in + // this test binary. let tmp = TempDir::new().unwrap(); + // Explicit init: client() no longer lazily initialises. let _ = client_if_ready().or_else(|| init(tmp.path().join("ws")).ok()); - let c = client().expect("global client should be available"); + let c = client().expect("global client should be available after init"); let _arc: Arc = c; } + + #[tokio::test] + async fn client_errs_clearly_when_not_initialised() { + // Use a fresh local `OnceLock` rather than the process-global one: + // other tests may have already called `init()` on the singleton, so + // an `is_none`-gated check on `GLOBAL_CLIENT` would race / silently + // skip. `client_from` lets us assert the contract deterministically. + let local: OnceLock = OnceLock::new(); + match client_from(&local) { + Ok(_) => panic!("client_from(empty) must error"), + Err(err) => assert!( + err.contains("init"), + "error should mention init contract, got: {err}" + ), + } + } } diff --git a/src/openhuman/memory/ingestion/README.md b/src/openhuman/memory/ingestion/README.md new file mode 100644 index 000000000..79249011d --- /dev/null +++ b/src/openhuman/memory/ingestion/README.md @@ -0,0 +1,18 @@ +# Memory ingestion + +Pipeline that turns raw document text into chunks plus extracted entities and relations, then upserts everything into `UnifiedMemory`. Runs synchronously when callers need the result (`MemoryClient::ingest_doc`) and as a background worker for fire-and-forget submissions (`MemoryClient::put_doc`). + +## Files + +- **`mod.rs`** — adds `ingest_document` and `extract_graph` to `UnifiedMemory`, plus the internal `upsert_graph_relations` helper. Re-exports the public types and the queue / state surface. +- **`types.rs`** — public ingestion API: `MemoryIngestionRequest` / `MemoryIngestionResult` / `MemoryIngestionConfig`, `ExtractionMode` (sentence vs chunk), `ExtractedEntity` / `ExtractedRelation`, `DEFAULT_MEMORY_EXTRACTION_MODEL`. Crate-internal intermediates (`RawEntity`, `RawRelation`, `ExtractionUnit`, `ExtractionAccumulator`, `ParsedIngestion`) live here too. +- **`parse.rs`** — `parse_document` pipeline: chunking, header / metadata enrichment, alias resolution, regex- and rule-driven extraction. Produces a `ParsedIngestion`. +- **`regex.rs`** — lazily-initialised regexes (email headers, named emails, graph facts, ownership, preferences, action items, recipients, spatial relations, dates, person names) plus `sanitize_entity_name`, `sanitize_fact_text`, `classify_entity`. +- **`rules.rs`** — semantic validation rules for graph predicates (allowed head/tail entity types) and the `ExtractionAccumulator` impl that gates `add_entity` / `add_relation` on those rules. +- **`queue.rs`** — `IngestionQueue` (cloneable submit handle) plus `IngestionJob` and the background worker started via `start_worker_with_state`. The worker shares an `IngestionState` with synchronous callers so all ingestion serialises through the same singleton lock. +- **`state.rs`** — `IngestionState` / `IngestionStatusSnapshot`: queue depth, in-flight metadata, last-completed status, and the `tokio::sync::Mutex` that enforces single-threaded extraction (the local LLM path can't be re-entered safely). +- **`tests.rs`** — pipeline coverage exercising `parse_document`, regex extraction, and `UnifiedMemory::ingest_document` end-to-end. + +## How it fits + +`MemoryClient` owns the singleton `IngestionQueue` and forwards to it from `put_doc` (background) or `ingest_doc` (synchronous, behind the same lock). Every ingestion run publishes `MemoryIngestionStarted` / `MemoryIngestionCompleted` events on the global event bus so the UI status pill and `openhuman.memory_ingestion_status` RPC stay in sync. Output rows feed `UnifiedMemory`'s `memory_docs`, `vector_chunks`, and `graph_namespace` tables. diff --git a/src/openhuman/memory/ingestion.rs b/src/openhuman/memory/ingestion/mod.rs similarity index 91% rename from src/openhuman/memory/ingestion.rs rename to src/openhuman/memory/ingestion/mod.rs index 25dc71620..bb029cb6a 100644 --- a/src/openhuman/memory/ingestion.rs +++ b/src/openhuman/memory/ingestion/mod.rs @@ -11,28 +11,32 @@ //! 5. **Persistence**: Upserting the document, text chunks, and graph relations into //! the memory store. -#[path = "ingestion_parse.rs"] -mod ingestion_parse; -#[path = "ingestion_regex.rs"] -mod ingestion_regex; -#[path = "ingestion_rules.rs"] -mod ingestion_rules; -#[path = "ingestion_types.rs"] -mod ingestion_types; +mod parse; +mod regex; +mod rules; +mod types; -pub use ingestion_types::{ +pub mod queue; +pub mod state; + +pub use queue::{IngestionJob, IngestionQueue}; +pub use state::{IngestionState, IngestionStatusSnapshot}; +pub use types::{ ExtractedEntity, ExtractedRelation, ExtractionMode, MemoryIngestionConfig, MemoryIngestionRequest, MemoryIngestionResult, DEFAULT_MEMORY_EXTRACTION_MODEL, }; -use ingestion_parse::{enrich_document_metadata, parse_document}; -use ingestion_types::ParsedIngestion; +use parse::{enrich_document_metadata, parse_document}; use serde_json::json; +use types::ParsedIngestion; use crate::openhuman::memory::store::types::NamespaceDocumentInput; use crate::openhuman::memory::UnifiedMemory; impl UnifiedMemory { + /// Run the full ingestion pipeline for a document: parse + chunk + extract + /// entities/relations, upsert the document row + vector chunks, and write + /// the extracted relations into the namespace graph. pub async fn ingest_document( &self, request: MemoryIngestionRequest, @@ -155,5 +159,4 @@ impl UnifiedMemory { } #[cfg(test)] -#[path = "ingestion_tests.rs"] mod tests; diff --git a/src/openhuman/memory/ingestion_parse.rs b/src/openhuman/memory/ingestion/parse.rs similarity index 99% rename from src/openhuman/memory/ingestion_parse.rs rename to src/openhuman/memory/ingestion/parse.rs index eb7998ec1..9623cb399 100644 --- a/src/openhuman/memory/ingestion_parse.rs +++ b/src/openhuman/memory/ingestion/parse.rs @@ -5,12 +5,12 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use serde_json::{json, Map, Value}; -use super::ingestion_regex::{ +use super::regex::{ action_item_regex, classify_entity, email_header_regex, explicit_owner_regex, explicit_preference_regex, graph_fact_regex, named_email_regex, recipient_regex, sanitize_entity_name, sanitize_fact_text, spatial_regex, will_review_regex, }; -use super::ingestion_types::{ +use super::types::{ ExtractedEntity, ExtractedRelation, ExtractionAccumulator, ExtractionMode, ExtractionUnit, MemoryIngestionConfig, ParsedIngestion, RawEntity, RawRelation, DEFAULT_CHUNK_TOKENS, }; diff --git a/src/openhuman/memory/ingestion_queue.rs b/src/openhuman/memory/ingestion/queue.rs similarity index 98% rename from src/openhuman/memory/ingestion_queue.rs rename to src/openhuman/memory/ingestion/queue.rs index 32df6ffab..dc0823d07 100644 --- a/src/openhuman/memory/ingestion_queue.rs +++ b/src/openhuman/memory/ingestion/queue.rs @@ -12,9 +12,9 @@ use std::time::Instant; use tokio::sync::mpsc; +use super::state::IngestionState; +use super::MemoryIngestionConfig; use crate::core::event_bus::{publish_global, DomainEvent}; -use crate::openhuman::memory::ingestion::MemoryIngestionConfig; -use crate::openhuman::memory::ingestion_state::IngestionState; use crate::openhuman::memory::store::{NamespaceDocumentInput, UnifiedMemory}; /// A job submitted to the ingestion worker. diff --git a/src/openhuman/memory/ingestion_regex.rs b/src/openhuman/memory/ingestion/regex.rs similarity index 100% rename from src/openhuman/memory/ingestion_regex.rs rename to src/openhuman/memory/ingestion/regex.rs diff --git a/src/openhuman/memory/ingestion_rules.rs b/src/openhuman/memory/ingestion/rules.rs similarity index 98% rename from src/openhuman/memory/ingestion_rules.rs rename to src/openhuman/memory/ingestion/rules.rs index aa16412fc..bcda067ca 100644 --- a/src/openhuman/memory/ingestion_rules.rs +++ b/src/openhuman/memory/ingestion/rules.rs @@ -4,8 +4,8 @@ use std::collections::BTreeSet; use serde_json::{Map, Value}; -use super::ingestion_regex::sanitize_entity_name; -use super::ingestion_types::{ExtractionAccumulator, RawEntity, RawRelation}; +use super::regex::sanitize_entity_name; +use super::types::{ExtractionAccumulator, RawEntity, RawRelation}; use crate::openhuman::memory::UnifiedMemory; /// A validation rule for semantic relationships. diff --git a/src/openhuman/memory/ingestion_state.rs b/src/openhuman/memory/ingestion/state.rs similarity index 100% rename from src/openhuman/memory/ingestion_state.rs rename to src/openhuman/memory/ingestion/state.rs diff --git a/src/openhuman/memory/ingestion_tests.rs b/src/openhuman/memory/ingestion/tests.rs similarity index 96% rename from src/openhuman/memory/ingestion_tests.rs rename to src/openhuman/memory/ingestion/tests.rs index b53286309..28342e000 100644 --- a/src/openhuman/memory/ingestion_tests.rs +++ b/src/openhuman/memory/ingestion/tests.rs @@ -1,11 +1,14 @@ +//! Tests for the ingestion pipeline — `parse_document`, regex extraction, +//! and `UnifiedMemory::ingest_document` end-to-end. + use std::sync::Arc; use serde_json::json; use tempfile::TempDir; +use crate::openhuman::embeddings::NoopEmbedding; use crate::openhuman::memory::{ - embeddings::NoopEmbedding, MemoryIngestionConfig, MemoryIngestionRequest, - NamespaceDocumentInput, UnifiedMemory, + MemoryIngestionConfig, MemoryIngestionRequest, NamespaceDocumentInput, UnifiedMemory, }; /// Test config for the heuristic-only ingestion pipeline. diff --git a/src/openhuman/memory/ingestion_types.rs b/src/openhuman/memory/ingestion/types.rs similarity index 100% rename from src/openhuman/memory/ingestion_types.rs rename to src/openhuman/memory/ingestion/types.rs diff --git a/src/openhuman/memory/mod.rs b/src/openhuman/memory/mod.rs index df1a2f979..872c78d4f 100644 --- a/src/openhuman/memory/mod.rs +++ b/src/openhuman/memory/mod.rs @@ -7,11 +7,8 @@ pub mod chunker; pub mod conversations; -pub mod embeddings; pub mod global; pub mod ingestion; -pub mod ingestion_queue; -pub mod ingestion_state; pub mod ops; pub mod rpc_models; pub mod schemas; @@ -21,11 +18,10 @@ pub mod traits; pub mod tree; pub use ingestion::{ - ExtractedEntity, ExtractedRelation, ExtractionMode, MemoryIngestionConfig, - MemoryIngestionRequest, MemoryIngestionResult, DEFAULT_MEMORY_EXTRACTION_MODEL, + ExtractedEntity, ExtractedRelation, ExtractionMode, IngestionJob, IngestionQueue, + IngestionState, IngestionStatusSnapshot, MemoryIngestionConfig, MemoryIngestionRequest, + MemoryIngestionResult, DEFAULT_MEMORY_EXTRACTION_MODEL, }; -pub use ingestion_queue::{IngestionJob, IngestionQueue}; -pub use ingestion_state::{IngestionState, IngestionStatusSnapshot}; pub use ops as rpc; pub use ops::*; pub use rpc_models::*; diff --git a/src/openhuman/memory/ops.rs b/src/openhuman/memory/ops.rs deleted file mode 100644 index 9a205ded3..000000000 --- a/src/openhuman/memory/ops.rs +++ /dev/null @@ -1,1414 +0,0 @@ -//! RPC operations for the memory system. -//! -//! This module implements the handlers for memory-related RPC requests, including -//! document management, semantic queries, key-value storage, and knowledge graph -//! operations. It manages the active memory client and provides utility functions -//! for formatting and filtering memory results. - -use crate::openhuman::config::Config; -use crate::openhuman::memory::store::GraphRelationRecord; -use crate::openhuman::memory::{ - ApiEnvelope, ApiError, ApiMeta, DeleteDocumentRequest, DeleteDocumentResponse, EmptyRequest, - ListDocumentsRequest, ListDocumentsResponse, ListMemoryFilesRequest, ListMemoryFilesResponse, - ListNamespacesResponse, MemoryClient, MemoryClientRef, MemoryDocumentSummary, - MemoryIngestionConfig, MemoryIngestionRequest, MemoryIngestionResult, MemoryInitRequest, - MemoryInitResponse, MemoryItemKind, MemoryRecallItem, MemoryRetrievalChunk, - MemoryRetrievalContext, MemoryRetrievalEntity, MemoryRetrievalRelation, NamespaceDocumentInput, - NamespaceMemoryHit, NamespaceRetrievalContext, PaginationMeta, QueryNamespaceRequest, - QueryNamespaceResponse, ReadMemoryFileRequest, ReadMemoryFileResponse, RecallContextRequest, - RecallContextResponse, RecallMemoriesRequest, RecallMemoriesResponse, WriteMemoryFileRequest, - WriteMemoryFileResponse, -}; -use crate::rpc::RpcOutcome; -use chrono::TimeZone; -use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; -use std::collections::{BTreeMap, BTreeSet}; -use std::path::{Component, Path, PathBuf}; - -// The global memory client singleton lives in `super::global`. -// All access goes through `active_memory_client()` below. - -/// Generates a unique request ID for memory operations. -/// -/// This ID is used for tracing and logging purposes in the API response metadata. -fn memory_request_id() -> String { - uuid::Uuid::new_v4().to_string() -} - -/// Converts an iterator of memory counts into a BTreeMap. -/// -/// This is a convenience helper for populating the `counts` field in the API metadata. -fn memory_counts( - entries: impl IntoIterator, -) -> BTreeMap { - entries - .into_iter() - .map(|(key, value)| (key.to_string(), value)) - .collect() -} - -/// Wraps data in an RPC API envelope. -/// -/// This standardises the response format for memory-related RPC methods. -fn envelope( - data: T, - counts: Option>, - pagination: Option, -) -> RpcOutcome> { - RpcOutcome::new( - ApiEnvelope { - data: Some(data), - error: None, - meta: ApiMeta { - request_id: memory_request_id(), - latency_seconds: None, - cached: None, - counts, - pagination, - }, - }, - vec![], - ) -} - -/// Wraps an error in an RPC API envelope. -/// -/// This provides a consistent error reporting format for the memory system. -fn error_envelope(code: &str, message: String) -> RpcOutcome> { - RpcOutcome::new( - ApiEnvelope { - data: None, - error: Some(ApiError { - code: code.to_string(), - message, - details: None, - }), - meta: ApiMeta { - request_id: memory_request_id(), - latency_seconds: None, - cached: None, - counts: None, - pagination: None, - }, - }, - vec![], - ) -} - -/// Formats a floating-point timestamp as an RFC3339 string. -/// -/// Returns `None` if the timestamp is invalid (NaN, infinite, or negative). -fn timestamp_to_rfc3339(timestamp: f64) -> Option { - if !timestamp.is_finite() || timestamp < 0.0 { - return None; - } - - let secs = timestamp.trunc() as i64; - let nanos = ((timestamp.fract().abs()) * 1_000_000_000.0).round() as u32; - chrono::Utc - .timestamp_opt(secs, nanos.min(999_999_999)) - .single() - .map(|value| value.to_rfc3339()) -} - -/// Maps a memory item kind to a human-readable label. -fn memory_kind_label(kind: &MemoryItemKind) -> &'static str { - match kind { - MemoryItemKind::Document => "document", - MemoryItemKind::Kv => "kv", - MemoryItemKind::Episodic => "episodic", - MemoryItemKind::Event => "event", - } -} - -/// Generates a unique string identity for a graph relation. -/// -/// The identity is composed of the namespace, subject, predicate, and object. -fn relation_identity(relation: &GraphRelationRecord) -> String { - format!( - "{}|{}|{}|{}", - relation.namespace.as_deref().unwrap_or("global"), - relation.subject.as_str(), - relation.predicate.as_str(), - relation.object.as_str() - ) -} - -/// Formats relation metadata into a JSON Value. -fn relation_metadata(relation: &GraphRelationRecord) -> Value { - json!({ - "namespace": relation.namespace.clone(), - "attrs": relation.attrs.clone(), - "order_index": relation.order_index, - "document_ids": relation.document_ids.clone(), - "chunk_ids": relation.chunk_ids.clone(), - "updated_at": timestamp_to_rfc3339(relation.updated_at), - }) -} - -/// Formats chunk metadata into a JSON Value. -fn chunk_metadata(hit: &NamespaceMemoryHit) -> Value { - json!({ - "kind": memory_kind_label(&hit.kind), - "namespace": hit.namespace.clone(), - "key": hit.key.clone(), - "title": hit.title.clone(), - "category": hit.category.clone(), - "source_type": hit.source_type.clone(), - "score_breakdown": { - "keyword_relevance": hit.score_breakdown.keyword_relevance, - "vector_similarity": hit.score_breakdown.vector_similarity, - "graph_relevance": hit.score_breakdown.graph_relevance, - "episodic_relevance": hit.score_breakdown.episodic_relevance, - "freshness": hit.score_breakdown.freshness, - "final_score": hit.score_breakdown.final_score, - } - }) -} - -/// Extracts an entity type for a specific role (subject/object) from relation attributes. -fn extract_entity_type(attrs: &Value, role: &str) -> Option { - attrs - .get("entity_types") - .and_then(|et| et.get(role)) - .and_then(|v| v.as_str()) - .filter(|s| !s.is_empty()) - .map(|s| s.to_string()) -} - -/// Transforms memory hits into a retrieval context with deduplicated entities and relations. -pub(crate) fn build_retrieval_context(hits: &[NamespaceMemoryHit]) -> MemoryRetrievalContext { - let mut entity_types: BTreeMap> = BTreeMap::new(); - let mut relations = BTreeMap::new(); - let chunks = hits - .iter() - .map(|hit| { - // Extract supporting relations from each hit to populate entities and relations - for relation in &hit.supporting_relations { - if !relation.subject.trim().is_empty() { - let entry = entity_types.entry(relation.subject.clone()).or_insert(None); - // Use the first non-empty entity type found for this subject - if entry.is_none() { - *entry = extract_entity_type(&relation.attrs, "subject"); - } - } - if !relation.object.trim().is_empty() { - let entry = entity_types.entry(relation.object.clone()).or_insert(None); - // Use the first non-empty entity type found for this object - if entry.is_none() { - *entry = extract_entity_type(&relation.attrs, "object"); - } - } - // Deduplicate relations based on their unique identity - relations - .entry(relation_identity(relation)) - .or_insert_with(|| MemoryRetrievalRelation { - subject: relation.subject.clone(), - predicate: relation.predicate.clone(), - object: relation.object.clone(), - score: None, - evidence_count: Some(relation.evidence_count), - metadata: relation_metadata(relation), - }); - } - - MemoryRetrievalChunk { - chunk_id: hit.chunk_id.clone(), - document_id: hit.document_id.clone(), - content: hit.content.clone(), - score: hit.score, - metadata: chunk_metadata(hit), - created_at: None, - updated_at: timestamp_to_rfc3339(hit.updated_at), - } - }) - .collect(); - - MemoryRetrievalContext { - entities: entity_types - .into_iter() - .map(|(name, entity_type)| MemoryRetrievalEntity { - id: None, - name, - entity_type, - score: None, - metadata: json!({}), - }) - .collect(), - relations: relations.into_values().collect(), - chunks, - } -} - -/// Formats memory hits into a natural-language context message for LLM consumption. -fn format_llm_context_message(query: Option<&str>, hits: &[NamespaceMemoryHit]) -> Option { - if hits.is_empty() { - return None; - } - - let mut parts = Vec::new(); - if let Some(query) = query { - parts.push(format!("Query: {query}")); - } - - for hit in hits { - let summary = match hit.kind { - MemoryItemKind::Document => { - let title = hit.title.clone().unwrap_or_else(|| hit.key.clone()); - format!("{title}: {}", hit.content.trim()) - } - MemoryItemKind::Kv => format!("[kv:{}] {}", hit.key, hit.content.trim()), - MemoryItemKind::Episodic => { - format!("[episodic:{}] {}", hit.key, hit.content.trim()) - } - MemoryItemKind::Event => { - format!("[event:{}] {}", hit.key, hit.content.trim()) - } - }; - parts.push(summary); - - // Include typed relations if present for better LLM reasoning - if !hit.supporting_relations.is_empty() { - let relations = hit - .supporting_relations - .iter() - .map(|relation| { - let subject_type = extract_entity_type(&relation.attrs, "subject"); - let object_type = extract_entity_type(&relation.attrs, "object"); - let subject_label = match subject_type { - Some(t) => format!("{} ({})", relation.subject, t), - None => relation.subject.clone(), - }; - let object_label = match object_type { - Some(t) => format!("{} ({})", relation.object, t), - None => relation.object.clone(), - }; - format!( - "{} -[{}]-> {}", - subject_label, relation.predicate, object_label - ) - }) - .collect::>() - .join("; "); - parts.push(format!("Relations: {relations}")); - } - } - - Some(parts.join("\n\n")) -} - -/// Filters memory hits to only include those matching specific document IDs. -fn filter_hits_by_document_ids( - hits: Vec, - document_ids: Option<&[String]>, -) -> Vec { - let Some(document_ids) = document_ids else { - return hits; - }; - let allowed = document_ids.iter().cloned().collect::>(); - hits.into_iter() - .filter(|hit| { - hit.document_id - .as_ref() - .map(|document_id| allowed.contains(document_id)) - .unwrap_or(false) - }) - .collect() -} - -/// Returns the retrieval context if `include_references` is true and context is not empty. -fn maybe_retrieval_context( - include_references: bool, - context: MemoryRetrievalContext, -) -> Option { - if !include_references { - return None; - } - if context.entities.is_empty() && context.relations.is_empty() && context.chunks.is_empty() { - return None; - } - Some(context) -} - -/// Returns the current workspace directory from configuration. -async fn current_workspace_dir() -> Result { - Config::load_or_init() - .await - .map(|config| config.workspace_dir) - .map_err(|e| format!("load config: {e}")) -} - -/// Returns the active memory client from the process-global singleton, -/// lazily initializing it if necessary. -async fn active_memory_client() -> Result { - if let Some(client) = super::global::client_if_ready() { - return Ok(client); - } - let workspace_dir = current_workspace_dir().await?; - super::global::init(workspace_dir) -} - -/// Validates that a relative path does not escape the memory directory. -fn validate_memory_relative_path(path: &str) -> Result<(), String> { - let candidate = Path::new(path); - if candidate.as_os_str().is_empty() { - return Err("memory path must not be empty".to_string()); - } - if candidate.is_absolute() { - return Err("absolute paths are not allowed".to_string()); - } - // Prevent traversal using .. components - for component in candidate.components() { - match component { - Component::ParentDir | Component::RootDir | Component::Prefix(_) => { - return Err("path traversal is not allowed".to_string()); - } - _ => {} - } - } - Ok(()) -} - -/// Resolves the canonical path to the memory directory within the workspace. -async fn resolve_memory_root() -> Result { - let workspace_dir = current_workspace_dir().await?; - let memory_root = workspace_dir.join("memory"); - std::fs::create_dir_all(&memory_root) - .map_err(|e| format!("create memory dir {}: {e}", memory_root.display()))?; - memory_root - .canonicalize() - .map_err(|e| format!("resolve memory dir {}: {e}", memory_root.display())) -} - -/// Resolves and canonicalizes an existing memory path, ensuring it stays within the workspace. -async fn resolve_existing_memory_path(relative_path: &str) -> Result { - validate_memory_relative_path(relative_path)?; - let workspace_dir = current_workspace_dir().await?; - let canonical_workspace = workspace_dir - .canonicalize() - .map_err(|e| format!("resolve workspace dir {}: {e}", workspace_dir.display()))?; - let full_path = workspace_dir.join(relative_path); - let resolved = full_path - .canonicalize() - .map_err(|e| format!("resolve memory path {}: {e}", full_path.display()))?; - if !resolved.starts_with(&canonical_workspace) { - return Err("memory path escapes the workspace directory".to_string()); - } - Ok(resolved) -} - -/// Resolves a path for writing, creating parent directories and ensuring it stays within the workspace. -async fn resolve_writable_memory_path(relative_path: &str) -> Result { - validate_memory_relative_path(relative_path)?; - let workspace_dir = current_workspace_dir().await?; - let canonical_workspace = workspace_dir - .canonicalize() - .map_err(|e| format!("resolve workspace dir {}: {e}", workspace_dir.display()))?; - let full_path = workspace_dir.join(relative_path); - let parent = full_path - .parent() - .ok_or_else(|| "memory path must include a file name".to_string())?; - std::fs::create_dir_all(parent) - .map_err(|e| format!("create memory path {}: {e}", parent.display()))?; - let resolved_parent = parent - .canonicalize() - .map_err(|e| format!("resolve memory parent {}: {e}", parent.display()))?; - if !resolved_parent.starts_with(&canonical_workspace) { - return Err("memory path escapes the workspace directory".to_string()); - } - let file_name = full_path - .file_name() - .ok_or_else(|| "memory path must include a file name".to_string())?; - let resolved = resolved_parent.join(file_name); - // Security check: refuse to write through symlinks to prevent hijacking - if let Ok(metadata) = std::fs::symlink_metadata(&resolved) { - if metadata.file_type().is_symlink() { - return Err(format!( - "refusing to write through symlink: {}", - resolved.display() - )); - } - } - Ok(resolved) -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct RawMemoryDocumentSummary { - document_id: String, - namespace: String, - key: String, - title: String, - source_type: String, - priority: String, - created_at: f64, - updated_at: f64, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct RawDeleteDocumentResult { - deleted: bool, - namespace: String, - document_id: String, -} - -fn parse_memory_document_summaries(raw: Value) -> Result, String> { - let documents = raw - .get("documents") - .and_then(Value::as_array) - .ok_or_else(|| "memory document list missing 'documents' array".to_string())?; - documents - .iter() - .cloned() - .map(|value| { - let raw: RawMemoryDocumentSummary = serde_json::from_value(value) - .map_err(|e| format!("decode memory document: {e}"))?; - Ok(MemoryDocumentSummary { - document_id: raw.document_id, - namespace: raw.namespace, - key: raw.key, - title: raw.title, - source_type: raw.source_type, - priority: raw.priority, - created_at: raw.created_at, - updated_at: raw.updated_at, - }) - }) - .collect() -} - -async fn query_limit_for_request( - client: &MemoryClient, - request: &QueryNamespaceRequest, -) -> Result { - let requested = request.resolved_limit(); - if request.document_ids.is_none() { - return Ok(requested); - } - - let raw = client.list_documents(Some(&request.namespace)).await?; - let documents = parse_memory_document_summaries(raw)?; - let total_documents = u32::try_from(documents.len()).unwrap_or(u32::MAX); - Ok(requested.max(total_documents)) -} - -/// Parameters for the `doc_put` RPC method. -#[derive(Debug, Deserialize)] -pub struct PutDocParams { - /// Namespace to store the document in. - pub namespace: String, - /// Unique key for the document within the namespace. - pub key: String, - /// Human-readable title for the document. - pub title: String, - /// The raw text content of the document. - pub content: String, - /// The source type of the document (e.g., "doc", "web"). - #[serde(default = "default_source_type")] - pub source_type: String, - /// Priority level for retrieval (e.g., "high", "medium", "low"). - #[serde(default = "default_priority")] - pub priority: String, - /// Optional tags for categorization and filtering. - #[serde(default)] - pub tags: Vec, - /// Additional unstructured metadata. - #[serde(default)] - pub metadata: serde_json::Value, - /// Core category for the document (e.g., "core", "user"). - #[serde(default = "default_category")] - pub category: String, - /// Optional session ID associated with the document. - #[serde(default)] - pub session_id: Option, - /// Optional explicit document ID. - #[serde(default)] - pub document_id: Option, -} - -/// Parameters for the `doc_ingest` RPC method. -#[derive(Debug, Deserialize)] -pub struct IngestDocParams { - /// Namespace to store the document in. - pub namespace: String, - /// Unique key for the document within the namespace. - pub key: String, - /// Human-readable title for the document. - pub title: String, - /// The raw text content of the document. - pub content: String, - /// The source type of the document. - #[serde(default = "default_source_type")] - pub source_type: String, - /// Priority level for retrieval. - #[serde(default = "default_priority")] - pub priority: String, - /// Optional tags for the document. - #[serde(default)] - pub tags: Vec, - /// Additional unstructured metadata. - #[serde(default)] - pub metadata: serde_json::Value, - /// Core category for the document. - #[serde(default = "default_category")] - pub category: String, - /// Optional session ID. - #[serde(default)] - pub session_id: Option, - /// Optional explicit document ID. - #[serde(default)] - pub document_id: Option, - /// Configuration for the ingestion process (chunking, etc.). - #[serde(default)] - pub config: Option, -} - -/// Parameters for RPC methods that only require a namespace. -#[derive(Debug, Deserialize)] -pub struct NamespaceOnlyParams { - /// The target namespace. - pub namespace: String, -} - -/// Parameters for the `clear_namespace` RPC method. -#[derive(Debug, Deserialize)] -pub struct ClearNamespaceParams { - /// The namespace to clear. - pub namespace: String, -} - -/// Result returned by the `clear_namespace` RPC method. -#[derive(Debug, Serialize)] -pub struct ClearNamespaceResult { - /// Whether the namespace was successfully cleared. - pub cleared: bool, - /// The namespace that was cleared. - pub namespace: String, -} - -/// Parameters for the `doc_delete` RPC method. -#[derive(Debug, Deserialize)] -pub struct DeleteDocParams { - /// The namespace containing the document. - pub namespace: String, - /// The unique ID of the document to delete. - pub document_id: String, -} - -/// Parameters for the `context_query` RPC method. -#[derive(Debug, Deserialize)] -pub struct QueryNamespaceParams { - /// The namespace to query. - pub namespace: String, - /// The natural language query string. - pub query: String, - /// Maximum number of results to return. - #[serde(default)] - pub limit: Option, -} - -/// Parameters for the `context_recall` RPC method. -#[derive(Debug, Deserialize)] -pub struct RecallNamespaceParams { - /// The namespace to recall from. - pub namespace: String, - /// Maximum number of results to return. - #[serde(default)] - pub limit: Option, -} - -/// Parameters for the `kv_set` RPC method. -#[derive(Debug, Deserialize)] -pub struct KvSetParams { - /// The namespace for the key-value pair. - #[serde(default)] - pub namespace: Option, - /// The unique key. - pub key: String, - /// The value to store. - pub value: serde_json::Value, -} - -/// Parameters for `kv_get` and `kv_delete` RPC methods. -#[derive(Debug, Deserialize)] -pub struct KvGetDeleteParams { - /// The namespace containing the key. - #[serde(default)] - pub namespace: Option, - /// The unique key. - pub key: String, -} - -/// Parameters for the `graph_upsert` RPC method. -#[derive(Debug, Deserialize)] -pub struct GraphUpsertParams { - /// The namespace for the relation. - #[serde(default)] - pub namespace: Option, - /// The subject of the relation triple. - pub subject: String, - /// The predicate (relationship) of the triple. - pub predicate: String, - /// The object of the triple. - pub object: String, - /// Additional attributes for the relation. - #[serde(default)] - pub attrs: serde_json::Value, -} - -/// Parameters for the `graph_query` RPC method. -#[derive(Debug, Deserialize)] -pub struct GraphQueryParams { - /// The namespace to query. - #[serde(default)] - pub namespace: Option, - /// Optional subject filter. - #[serde(default)] - pub subject: Option, - /// Optional predicate filter. - #[serde(default)] - pub predicate: Option, -} - -/// Result returned by the `doc_put` RPC method. -#[derive(Debug, Serialize)] -pub struct PutDocResult { - /// The unique ID of the upserted document. - pub document_id: String, -} - -fn default_source_type() -> String { - "doc".to_string() -} - -fn default_priority() -> String { - "medium".to_string() -} - -fn default_category() -> String { - "core".to_string() -} - -/// Lists all namespaces in the memory system. -pub async fn namespace_list() -> Result>, String> { - let client = active_memory_client().await?; - let namespaces = client.list_namespaces().await?; - Ok(RpcOutcome::single_log( - namespaces, - "memory namespaces listed", - )) -} - -/// Upserts a document into a namespace. -pub async fn doc_put(params: PutDocParams) -> Result, String> { - let client = active_memory_client().await?; - let document_id = client - .put_doc(NamespaceDocumentInput { - namespace: params.namespace, - key: params.key, - title: params.title, - content: params.content, - source_type: params.source_type, - priority: params.priority, - tags: params.tags, - metadata: params.metadata, - category: params.category, - session_id: params.session_id, - document_id: params.document_id, - }) - .await?; - Ok(RpcOutcome::single_log( - PutDocResult { document_id }, - "memory document upserted", - )) -} - -/// Ingests a document, performing chunking and embedding. -pub async fn doc_ingest( - params: IngestDocParams, -) -> Result, String> { - let client = active_memory_client().await?; - let result = client - .ingest_doc(MemoryIngestionRequest { - document: NamespaceDocumentInput { - namespace: params.namespace, - key: params.key, - title: params.title, - content: params.content, - source_type: params.source_type, - priority: params.priority, - tags: params.tags, - metadata: params.metadata, - category: params.category, - session_id: params.session_id, - document_id: params.document_id, - }, - config: params.config.unwrap_or_default(), - }) - .await?; - let msg = format!( - "ingested document {} — {} entities, {} relations, {} chunks", - result.document_id, result.entity_count, result.relation_count, result.chunk_count, - ); - Ok(RpcOutcome::single_log(result, &msg)) -} - -/// Lists documents, optionally filtered by namespace. -pub async fn doc_list( - params: Option, -) -> Result, String> { - let client = active_memory_client().await?; - let docs = client - .list_documents(params.as_ref().map(|v| v.namespace.as_str())) - .await?; - Ok(RpcOutcome::single_log(docs, "memory documents listed")) -} - -/// Deletes a document from a namespace. -pub async fn doc_delete(params: DeleteDocParams) -> Result, String> { - let client = active_memory_client().await?; - let result = client - .delete_document(¶ms.namespace, ¶ms.document_id) - .await?; - Ok(RpcOutcome::single_log(result, "memory document deleted")) -} - -/// Clears all data within a namespace. -pub async fn clear_namespace( - params: ClearNamespaceParams, -) -> Result, String> { - let client = active_memory_client().await?; - log::debug!( - "[memory] clear_namespace RPC: namespace={}", - params.namespace - ); - client.clear_namespace(¶ms.namespace).await?; - let msg = format!("memory namespace '{}' cleared", params.namespace); - Ok(RpcOutcome::single_log( - ClearNamespaceResult { - cleared: true, - namespace: params.namespace, - }, - &msg, - )) -} - -/// Queries a namespace for contextual information based on a natural language string. -pub async fn context_query(params: QueryNamespaceParams) -> Result, String> { - let client = active_memory_client().await?; - let result = client - .query_namespace(¶ms.namespace, ¶ms.query, params.limit.unwrap_or(10)) - .await?; - Ok(RpcOutcome::single_log(result, "memory context queried")) -} - -/// Recalls contextual information from a namespace without a specific query. -pub async fn context_recall( - params: RecallNamespaceParams, -) -> Result>, String> { - let client = active_memory_client().await?; - let result = client - .recall_namespace(¶ms.namespace, params.limit.unwrap_or(10)) - .await?; - Ok(RpcOutcome::single_log(result, "memory context recalled")) -} - -/// Sets a key-value pair in the memory store. -pub async fn kv_set(params: KvSetParams) -> Result, String> { - let client = active_memory_client().await?; - client - .kv_set(params.namespace.as_deref(), ¶ms.key, ¶ms.value) - .await?; - Ok(RpcOutcome::single_log(true, "memory kv set")) -} - -/// Retrieves a value by key from the memory store. -pub async fn kv_get( - params: KvGetDeleteParams, -) -> Result>, String> { - let client = active_memory_client().await?; - let value = client - .kv_get(params.namespace.as_deref(), ¶ms.key) - .await?; - Ok(RpcOutcome::single_log(value, "memory kv get")) -} - -/// Deletes a key-value pair from the memory store. -pub async fn kv_delete(params: KvGetDeleteParams) -> Result, String> { - let client = active_memory_client().await?; - let deleted = client - .kv_delete(params.namespace.as_deref(), ¶ms.key) - .await?; - Ok(RpcOutcome::single_log(deleted, "memory kv delete")) -} - -/// Lists all key-value entries in a namespace. -pub async fn kv_list_namespace( - params: NamespaceOnlyParams, -) -> Result>, String> { - let client = active_memory_client().await?; - let rows = client.kv_list_namespace(¶ms.namespace).await?; - Ok(RpcOutcome::single_log(rows, "memory namespace kv listed")) -} - -/// Upserts a relation triple in the knowledge graph. -pub async fn graph_upsert(params: GraphUpsertParams) -> Result, String> { - let client = active_memory_client().await?; - client - .graph_upsert( - params.namespace.as_deref(), - ¶ms.subject, - ¶ms.predicate, - ¶ms.object, - ¶ms.attrs, - ) - .await?; - Ok(RpcOutcome::single_log(true, "memory graph upserted")) -} - -/// Queries relations from the knowledge graph. -pub async fn graph_query( - params: GraphQueryParams, -) -> Result>, String> { - let client = active_memory_client().await?; - let rows = client - .graph_query( - params.namespace.as_deref(), - params.subject.as_deref(), - params.predicate.as_deref(), - ) - .await?; - Ok(RpcOutcome::single_log(rows, "memory graph queried")) -} - -/// Parameters for `memory_sync_channel`. -#[derive(Debug, serde::Deserialize)] -pub struct SyncChannelParams { - pub channel_id: String, -} - -/// Result returned by `memory_sync_channel`. -#[derive(Debug, serde::Serialize)] -pub struct SyncChannelResult { - pub requested: bool, - pub channel_id: String, -} - -/// Result returned by `memory_sync_all`. -#[derive(Debug, serde::Serialize)] -pub struct SyncAllResult { - pub requested: bool, -} - -/// Result returned by `memory_ingestion_status`. Mirrors -/// [`crate::openhuman::memory::IngestionStatusSnapshot`] but is the public RPC -/// shape — the indirection keeps internal renames from breaking the wire -/// contract. -#[derive(Debug, Clone, Default, serde::Serialize)] -pub struct IngestionStatusResult { - pub running: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub current_document_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub current_title: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub current_namespace: Option, - pub queue_depth: usize, - #[serde(skip_serializing_if = "Option::is_none")] - pub last_completed_at: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub last_document_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub last_success: Option, -} - -/// Per-namespace outcome for `memory_learn_all`. -#[derive(Debug, serde::Serialize)] -pub struct NamespaceLearnResult { - pub namespace: String, - pub status: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, -} - -/// Result returned by `memory_learn_all`. -#[derive(Debug, serde::Serialize)] -pub struct LearnAllResult { - pub namespaces_processed: usize, - pub results: Vec, -} - -/// Parameters for `memory_learn_all`. -#[derive(Debug, serde::Deserialize)] -pub struct LearnAllParams { - /// Optional list of namespaces to constrain. Defaults to all namespaces. - #[serde(default)] - pub namespaces: Option>, -} - -/// Request a memory sync for a specific channel. -/// -/// Ingestion in OpenHuman is listener/webhook-driven — there is no per-provider -/// pull mechanism yet. This RPC publishes `DomainEvent::MemorySyncRequested` so -/// that future ingestion subscribers can react to an explicit pull request. -/// The event is fire-and-forget; the caller receives confirmation that the -/// request was published, not that ingestion ran. -pub async fn memory_sync_channel( - params: SyncChannelParams, -) -> Result, String> { - tracing::info!( - "[memory.sync] memory_sync_channel: entry channel_id={}", - params.channel_id - ); - crate::core::event_bus::publish_global( - crate::core::event_bus::DomainEvent::MemorySyncRequested { - channel_id: Some(params.channel_id.clone()), - }, - ); - tracing::debug!( - "[memory.sync] memory_sync_channel: MemorySyncRequested published channel_id={}", - params.channel_id - ); - Ok(RpcOutcome::new( - SyncChannelResult { - requested: true, - channel_id: params.channel_id, - }, - vec![], - )) -} - -/// Request a memory sync for all channels. -/// -/// Publishes `DomainEvent::MemorySyncRequested { channel_id: None }` on the -/// global event bus. No consumers exist yet — this is a hook for future -/// ingestion subscribers. -pub async fn memory_sync_all() -> Result, String> { - tracing::info!("[memory.sync] memory_sync_all: entry"); - crate::core::event_bus::publish_global( - crate::core::event_bus::DomainEvent::MemorySyncRequested { channel_id: None }, - ); - tracing::debug!("[memory.sync] memory_sync_all: MemorySyncRequested(all) published"); - Ok(RpcOutcome::new(SyncAllResult { requested: true }, vec![])) -} - -/// Returns the current memory-ingestion status: whether a job is running, the -/// in-flight document, queue depth, and the most recent completion. Read-only, -/// safe to poll. -pub async fn memory_ingestion_status() -> Result, String> { - let snapshot = match crate::openhuman::memory::global::client_if_ready() { - Some(c) => c.ingestion_state().snapshot(), - // Memory not yet initialised — report idle, no in-flight job. - None => Default::default(), - }; - Ok(RpcOutcome::new( - IngestionStatusResult { - running: snapshot.running, - current_document_id: snapshot.current_document_id, - current_title: snapshot.current_title, - current_namespace: snapshot.current_namespace, - queue_depth: snapshot.queue_depth, - last_completed_at: snapshot.last_completed_at, - last_document_id: snapshot.last_document_id, - last_success: snapshot.last_success, - }, - vec![], - )) -} - -/// Run the tree summarizer over all (or a constrained set of) namespaces. -/// -/// Enumerates namespaces via `namespace_list`, then for each runs -/// `tree_summarizer_run`. Results are collected per-namespace; a failing -/// namespace does not abort the rest. Runs sequentially to avoid saturating -/// the local AI provider. -pub async fn memory_learn_all( - params: LearnAllParams, -) -> Result, String> { - tracing::info!( - "[memory.learn] memory_learn_all: entry namespaces={:?}", - params.namespaces - ); - - // Resolve the target namespace list. - let client = active_memory_client().await?; - let all_ns = client.list_namespaces().await?; - tracing::debug!("[memory.learn] available namespaces: {:?}", all_ns); - - let target_ns: Vec = match ¶ms.namespaces { - Some(requested) if !requested.is_empty() => { - let mut seen = BTreeSet::new(); - let filtered: Vec<_> = requested - .iter() - .filter(|ns| all_ns.contains(ns)) - .filter(|ns| seen.insert((*ns).clone())) - .cloned() - .collect(); - tracing::debug!("[memory.learn] constrained to namespaces: {:?}", filtered); - filtered - } - Some(requested) => { - // Explicit empty list → no-op (don't fall back to all namespaces). - let mut seen = BTreeSet::new(); - let filtered: Vec<_> = requested - .iter() - .filter(|ns| all_ns.contains(ns)) - .filter(|ns| seen.insert((*ns).clone())) - .cloned() - .collect(); - tracing::debug!( - "[memory.learn] Some([]) empty request → no-op or filtered to {:?}", - filtered - ); - filtered - } - None => { - tracing::debug!("[memory.learn] using all {} namespaces", all_ns.len()); - all_ns - } - }; - - // Short-circuit when there are no namespaces to process — avoids loading - // config (and the local_ai.enabled guard) for an empty batch. - if target_ns.is_empty() { - tracing::info!( - "[memory.learn] memory_learn_all: no namespaces to process, returning early" - ); - return Ok(RpcOutcome::new( - LearnAllResult { - namespaces_processed: 0, - results: vec![], - }, - vec![], - )); - } - - let config = crate::openhuman::config::Config::load_or_init() - .await - .map_err(|e| format!("load config: {e}"))?; - - if !config.local_ai.enabled { - return Err("memory_learn_all requires local_ai.enabled=true".to_string()); - } - - let mut results = Vec::with_capacity(target_ns.len()); - for namespace in &target_ns { - tracing::info!( - "[memory.learn] running summarization for namespace='{}'", - namespace - ); - let outcome = - crate::openhuman::tree_summarizer::ops::tree_summarizer_run(&config, namespace).await; - match outcome { - Ok(_) => { - tracing::info!("[memory.learn] namespace='{}' ok", namespace); - results.push(NamespaceLearnResult { - namespace: namespace.clone(), - status: "ok".to_string(), - error: None, - }); - } - Err(e) => { - tracing::warn!("[memory.learn] namespace='{}' error: {}", namespace, e); - results.push(NamespaceLearnResult { - namespace: namespace.clone(), - status: "error".to_string(), - error: Some(e), - }); - } - } - } - - let namespaces_processed = results.len(); - tracing::info!( - "[memory.learn] memory_learn_all: done processed={} results={:?}", - namespaces_processed, - results - .iter() - .map(|r| (&r.namespace, &r.status)) - .collect::>() - ); - Ok(RpcOutcome::new( - LearnAllResult { - namespaces_processed, - results, - }, - vec![], - )) -} - -/// Initialise the local-only (SQLite) memory subsystem for the current workspace. -/// -/// `request.jwt_token` is accepted for backward compatibility but ignored — all -/// memory operations are local. Remote/cloud sync is a future consideration. -pub async fn memory_init( - request: MemoryInitRequest, -) -> Result>, String> { - let _ = request.jwt_token; // accepted but unused — memory is local-only - let workspace_dir = current_workspace_dir().await?; - // Initialise (or return existing) global singleton. - let _ = super::global::init(workspace_dir.clone())?; - let memory_dir = workspace_dir.join("memory"); - Ok(envelope( - MemoryInitResponse { - initialized: true, - workspace_dir: workspace_dir.display().to_string(), - memory_dir: memory_dir.display().to_string(), - }, - None, - None, - )) -} - -/// Lists documents stored in memory, optionally filtered by namespace. -pub async fn memory_list_documents( - request: ListDocumentsRequest, -) -> Result>, String> { - let client = active_memory_client().await?; - let raw = client.list_documents(request.namespace.as_deref()).await?; - let documents = parse_memory_document_summaries(raw)?; - let count = documents.len(); - Ok(envelope( - ListDocumentsResponse { - namespace: request.namespace, - documents, - count, - }, - Some(memory_counts([("num_documents", count)])), - Some(PaginationMeta { - limit: count, - offset: 0, - count, - }), - )) -} - -/// Lists all namespaces that contain memory documents. -pub async fn memory_list_namespaces( - _request: EmptyRequest, -) -> Result>, String> { - let client = active_memory_client().await?; - let namespaces = client.list_namespaces().await?; - let count = namespaces.len(); - Ok(envelope( - ListNamespacesResponse { namespaces, count }, - Some(memory_counts([("num_namespaces", count)])), - None, - )) -} - -/// Deletes a specific document from a namespace. -pub async fn memory_delete_document( - request: DeleteDocumentRequest, -) -> Result>, String> { - let client = active_memory_client().await?; - let raw = client - .delete_document(&request.namespace, &request.document_id) - .await?; - let parsed: RawDeleteDocumentResult = - serde_json::from_value(raw).map_err(|e| format!("decode delete document result: {e}"))?; - Ok(envelope( - DeleteDocumentResponse { - status: if parsed.deleted { - "completed".to_string() - } else { - "not_found".to_string() - }, - namespace: parsed.namespace, - document_id: parsed.document_id, - deleted: parsed.deleted, - }, - None, - None, - )) -} - -/// Performs a semantic query against a namespace, returning a retrieval context. -pub async fn memory_query_namespace( - request: QueryNamespaceRequest, -) -> Result>, String> { - let include_references = request.include_references.unwrap_or(true); - let result = async { - let client = active_memory_client().await?; - let retrieval_limit = query_limit_for_request(client.as_ref(), &request).await?; - let mut context = client - .query_namespace_context_data(&request.namespace, &request.query, retrieval_limit) - .await?; - context.hits = filter_hits_by_document_ids(context.hits, request.document_ids.as_deref()); - Ok::(context) - } - .await; - - match result { - Ok(context) => { - let retrieval_context = build_retrieval_context(&context.hits); - let counts = memory_counts([ - ("num_entities", retrieval_context.entities.len()), - ("num_relations", retrieval_context.relations.len()), - ("num_chunks", retrieval_context.chunks.len()), - ]); - let llm_context_message = - format_llm_context_message(Some(&request.query), &context.hits); - Ok(envelope( - QueryNamespaceResponse { - context: maybe_retrieval_context(include_references, retrieval_context), - llm_context_message, - }, - Some(counts), - None, - )) - } - Err(message) => Ok(error_envelope("memory.query_namespace_failed", message)), - } -} - -/// Recalls contextual data from a namespace without a specific query. -pub async fn memory_recall_context( - request: RecallContextRequest, -) -> Result>, String> { - let include_references = request.include_references.unwrap_or(true); - let result = async { - let client = active_memory_client().await?; - client - .recall_namespace_context_data(&request.namespace, request.resolved_limit()) - .await - } - .await; - - match result { - Ok(context) => { - let retrieval_context = build_retrieval_context(&context.hits); - let counts = memory_counts([ - ("num_entities", retrieval_context.entities.len()), - ("num_relations", retrieval_context.relations.len()), - ("num_chunks", retrieval_context.chunks.len()), - ]); - let llm_context_message = format_llm_context_message(None, &context.hits); - Ok(envelope( - RecallContextResponse { - context: maybe_retrieval_context(include_references, retrieval_context), - llm_context_message, - }, - Some(counts), - None, - )) - } - Err(message) => Ok(error_envelope("memory.recall_context_failed", message)), - } -} - -/// Recalls memory items from a namespace with optional retention filtering. -pub async fn memory_recall_memories( - request: RecallMemoriesRequest, -) -> Result>, String> { - let result = async { - let client = active_memory_client().await?; - client - .recall_namespace_memories(&request.namespace, request.resolved_limit()) - .await - } - .await; - - match result { - Ok(hits) => { - let memories = hits - .into_iter() - .map(|hit| MemoryRecallItem { - kind: memory_kind_label(&hit.kind).to_string(), - id: hit.id, - content: hit.content, - score: hit.score, - retention: None, - last_accessed_at: None, - access_count: None, - stability_days: None, - }) - .collect::>(); - let count = memories.len(); - Ok(envelope( - RecallMemoriesResponse { memories }, - Some(memory_counts([("num_memories", count)])), - None, - )) - } - Err(message) => Ok(error_envelope("memory.recall_memories_failed", message)), - } -} - -/// Lists files in a memory directory. -pub async fn ai_list_memory_files( - request: ListMemoryFilesRequest, -) -> Result>, String> { - validate_memory_relative_path(&request.relative_dir)?; - let directory = resolve_existing_memory_path(&request.relative_dir).await?; - if !directory.is_dir() { - return Err(format!( - "memory directory not found: {}", - directory.display() - )); - } - let mut files = Vec::new(); - for entry in std::fs::read_dir(&directory) - .map_err(|e| format!("read memory directory {}: {e}", directory.display()))? - { - let entry = entry.map_err(|e| format!("read memory directory entry: {e}"))?; - let file_name = entry.file_name(); - let file_name = file_name.to_string_lossy(); - if !file_name.is_empty() { - files.push(file_name.to_string()); - } - } - files.sort(); - let count = files.len(); - Ok(envelope( - ListMemoryFilesResponse { - relative_dir: request.relative_dir, - files, - count, - }, - Some(memory_counts([("num_files", count)])), - None, - )) -} - -/// Reads the contents of a memory file. -pub async fn ai_read_memory_file( - request: ReadMemoryFileRequest, -) -> Result>, String> { - let path = resolve_existing_memory_path(&request.relative_path).await?; - let content = std::fs::read_to_string(&path) - .map_err(|e| format!("read memory file {}: {e}", path.display()))?; - Ok(envelope( - ReadMemoryFileResponse { - relative_path: request.relative_path, - content, - }, - None, - None, - )) -} - -/// Writes content to a memory file. -pub async fn ai_write_memory_file( - request: WriteMemoryFileRequest, -) -> Result>, String> { - let path = resolve_writable_memory_path(&request.relative_path).await?; - std::fs::write(&path, request.content.as_bytes()) - .map_err(|e| format!("write memory file {}: {e}", path.display()))?; - let bytes_written = request.content.len(); - Ok(envelope( - WriteMemoryFileResponse { - relative_path: request.relative_path, - written: true, - bytes_written, - }, - None, - None, - )) -} - -#[cfg(test)] -#[path = "ops_tests.rs"] -mod tests; diff --git a/src/openhuman/memory/ops/documents.rs b/src/openhuman/memory/ops/documents.rs new file mode 100644 index 000000000..215eec804 --- /dev/null +++ b/src/openhuman/memory/ops/documents.rs @@ -0,0 +1,491 @@ +//! Document, namespace, and recall RPC handlers — both the unified-memory +//! direct API (`doc_*`, `namespace_*`, `context_*`) and the envelope-style +//! façade (`memory_init`, `memory_list_documents`, `memory_query_namespace`, +//! `memory_recall_*`). + +use serde::{Deserialize, Serialize}; + +use crate::openhuman::memory::{ + ApiEnvelope, DeleteDocumentRequest, DeleteDocumentResponse, EmptyRequest, ListDocumentsRequest, + ListDocumentsResponse, ListNamespacesResponse, MemoryIngestionConfig, MemoryIngestionRequest, + MemoryIngestionResult, MemoryInitRequest, MemoryInitResponse, MemoryRecallItem, + NamespaceDocumentInput, NamespaceRetrievalContext, PaginationMeta, QueryNamespaceRequest, + QueryNamespaceResponse, RecallContextRequest, RecallContextResponse, RecallMemoriesRequest, + RecallMemoriesResponse, +}; +use crate::rpc::RpcOutcome; + +use super::envelope::{envelope, error_envelope, memory_counts}; +use super::helpers::{ + active_memory_client, build_retrieval_context, current_workspace_dir, + filter_hits_by_document_ids, format_llm_context_message, maybe_retrieval_context, + memory_kind_label, parse_memory_document_summaries, query_limit_for_request, + RawDeleteDocumentResult, +}; +use super::helpers::{default_category, default_priority, default_source_type}; + +/// Parameters for the `doc_put` RPC method. +#[derive(Debug, Deserialize)] +pub struct PutDocParams { + /// Namespace to store the document in. + pub namespace: String, + /// Unique key for the document within the namespace. + pub key: String, + /// Human-readable title for the document. + pub title: String, + /// The raw text content of the document. + pub content: String, + /// The source type of the document (e.g., "doc", "web"). + #[serde(default = "default_source_type")] + pub source_type: String, + /// Priority level for retrieval (e.g., "high", "medium", "low"). + #[serde(default = "default_priority")] + pub priority: String, + /// Optional tags for categorization and filtering. + #[serde(default)] + pub tags: Vec, + /// Additional unstructured metadata. + #[serde(default)] + pub metadata: serde_json::Value, + /// Core category for the document (e.g., "core", "user"). + #[serde(default = "default_category")] + pub category: String, + /// Optional session ID associated with the document. + #[serde(default)] + pub session_id: Option, + /// Optional explicit document ID. + #[serde(default)] + pub document_id: Option, +} + +/// Parameters for the `doc_ingest` RPC method. +#[derive(Debug, Deserialize)] +pub struct IngestDocParams { + /// Namespace to store the document in. + pub namespace: String, + /// Unique key for the document within the namespace. + pub key: String, + /// Human-readable title for the document. + pub title: String, + /// The raw text content of the document. + pub content: String, + /// The source type of the document. + #[serde(default = "default_source_type")] + pub source_type: String, + /// Priority level for retrieval. + #[serde(default = "default_priority")] + pub priority: String, + /// Optional tags for the document. + #[serde(default)] + pub tags: Vec, + /// Additional unstructured metadata. + #[serde(default)] + pub metadata: serde_json::Value, + /// Core category for the document. + #[serde(default = "default_category")] + pub category: String, + /// Optional session ID. + #[serde(default)] + pub session_id: Option, + /// Optional explicit document ID. + #[serde(default)] + pub document_id: Option, + /// Configuration for the ingestion process (chunking, etc.). + #[serde(default)] + pub config: Option, +} + +/// Parameters for RPC methods that only require a namespace. +#[derive(Debug, Deserialize)] +pub struct NamespaceOnlyParams { + /// The target namespace. + pub namespace: String, +} + +/// Parameters for the `clear_namespace` RPC method. +#[derive(Debug, Deserialize)] +pub struct ClearNamespaceParams { + /// The namespace to clear. + pub namespace: String, +} + +/// Result returned by the `clear_namespace` RPC method. +#[derive(Debug, Serialize)] +pub struct ClearNamespaceResult { + /// Whether the namespace was successfully cleared. + pub cleared: bool, + /// The namespace that was cleared. + pub namespace: String, +} + +/// Parameters for the `doc_delete` RPC method. +#[derive(Debug, Deserialize)] +pub struct DeleteDocParams { + /// The namespace containing the document. + pub namespace: String, + /// The unique ID of the document to delete. + pub document_id: String, +} + +/// Parameters for the `context_query` RPC method. +#[derive(Debug, Deserialize)] +pub struct QueryNamespaceParams { + /// The namespace to query. + pub namespace: String, + /// The natural language query string. + pub query: String, + /// Maximum number of results to return. + #[serde(default)] + pub limit: Option, +} + +/// Parameters for the `context_recall` RPC method. +#[derive(Debug, Deserialize)] +pub struct RecallNamespaceParams { + /// The namespace to recall from. + pub namespace: String, + /// Maximum number of results to return. + #[serde(default)] + pub limit: Option, +} + +/// Result returned by the `doc_put` RPC method. +#[derive(Debug, Serialize)] +pub struct PutDocResult { + /// The unique ID of the upserted document. + pub document_id: String, +} + +// --------------------------------------------------------------------------- +// Unified-memory direct API +// --------------------------------------------------------------------------- + +/// Lists all namespaces in the memory system. +pub async fn namespace_list() -> Result>, String> { + let client = active_memory_client().await?; + let namespaces = client.list_namespaces().await?; + Ok(RpcOutcome::single_log( + namespaces, + "memory namespaces listed", + )) +} + +/// Upserts a document into a namespace. +pub async fn doc_put(params: PutDocParams) -> Result, String> { + let client = active_memory_client().await?; + let document_id = client + .put_doc(NamespaceDocumentInput { + namespace: params.namespace, + key: params.key, + title: params.title, + content: params.content, + source_type: params.source_type, + priority: params.priority, + tags: params.tags, + metadata: params.metadata, + category: params.category, + session_id: params.session_id, + document_id: params.document_id, + }) + .await?; + Ok(RpcOutcome::single_log( + PutDocResult { document_id }, + "memory document upserted", + )) +} + +/// Ingests a document, performing chunking and embedding. +pub async fn doc_ingest( + params: IngestDocParams, +) -> Result, String> { + let client = active_memory_client().await?; + let result = client + .ingest_doc(MemoryIngestionRequest { + document: NamespaceDocumentInput { + namespace: params.namespace, + key: params.key, + title: params.title, + content: params.content, + source_type: params.source_type, + priority: params.priority, + tags: params.tags, + metadata: params.metadata, + category: params.category, + session_id: params.session_id, + document_id: params.document_id, + }, + config: params.config.unwrap_or_default(), + }) + .await?; + let msg = format!( + "ingested document — {} entities, {} relations, {} chunks", + result.entity_count, result.relation_count, result.chunk_count, + ); + Ok(RpcOutcome::single_log(result, &msg)) +} + +/// Lists documents, optionally filtered by namespace. +pub async fn doc_list( + params: Option, +) -> Result, String> { + let client = active_memory_client().await?; + let docs = client + .list_documents(params.as_ref().map(|v| v.namespace.as_str())) + .await?; + Ok(RpcOutcome::single_log(docs, "memory documents listed")) +} + +/// Deletes a document from a namespace. +pub async fn doc_delete(params: DeleteDocParams) -> Result, String> { + let client = active_memory_client().await?; + let result = client + .delete_document(¶ms.namespace, ¶ms.document_id) + .await?; + Ok(RpcOutcome::single_log(result, "memory document deleted")) +} + +/// Clears all data within a namespace. +pub async fn clear_namespace( + params: ClearNamespaceParams, +) -> Result, String> { + let client = active_memory_client().await?; + log::debug!("[memory] clear_namespace RPC invoked"); + client.clear_namespace(¶ms.namespace).await?; + let msg = "memory namespace cleared".to_string(); + Ok(RpcOutcome::single_log( + ClearNamespaceResult { + cleared: true, + namespace: params.namespace, + }, + &msg, + )) +} + +/// Queries a namespace for contextual information based on a natural language string. +pub async fn context_query(params: QueryNamespaceParams) -> Result, String> { + let client = active_memory_client().await?; + let result = client + .query_namespace(¶ms.namespace, ¶ms.query, params.limit.unwrap_or(10)) + .await?; + Ok(RpcOutcome::single_log(result, "memory context queried")) +} + +/// Recalls contextual information from a namespace without a specific query. +pub async fn context_recall( + params: RecallNamespaceParams, +) -> Result>, String> { + let client = active_memory_client().await?; + let result = client + .recall_namespace(¶ms.namespace, params.limit.unwrap_or(10)) + .await?; + Ok(RpcOutcome::single_log(result, "memory context recalled")) +} + +// --------------------------------------------------------------------------- +// Envelope-style façade (`memory_*`) +// --------------------------------------------------------------------------- + +/// Initialise the local-only (SQLite) memory subsystem for the current workspace. +/// +/// `request.jwt_token` is accepted for backward compatibility but ignored — all +/// memory operations are local. Remote/cloud sync is a future consideration. +pub async fn memory_init( + request: MemoryInitRequest, +) -> Result>, String> { + let _ = request.jwt_token; // accepted but unused — memory is local-only + let workspace_dir = current_workspace_dir().await?; + // Initialise (or return existing) global singleton. + let _ = super::super::global::init(workspace_dir.clone())?; + let memory_dir = workspace_dir.join("memory"); + Ok(envelope( + MemoryInitResponse { + initialized: true, + workspace_dir: workspace_dir.display().to_string(), + memory_dir: memory_dir.display().to_string(), + }, + None, + None, + )) +} + +/// Lists documents stored in memory, optionally filtered by namespace. +pub async fn memory_list_documents( + request: ListDocumentsRequest, +) -> Result>, String> { + let client = active_memory_client().await?; + let raw = client.list_documents(request.namespace.as_deref()).await?; + let documents = parse_memory_document_summaries(raw)?; + let count = documents.len(); + Ok(envelope( + ListDocumentsResponse { + namespace: request.namespace, + documents, + count, + }, + Some(memory_counts([("num_documents", count)])), + Some(PaginationMeta { + limit: count, + offset: 0, + count, + }), + )) +} + +/// Lists all namespaces that contain memory documents. +pub async fn memory_list_namespaces( + _request: EmptyRequest, +) -> Result>, String> { + let client = active_memory_client().await?; + let namespaces = client.list_namespaces().await?; + let count = namespaces.len(); + Ok(envelope( + ListNamespacesResponse { namespaces, count }, + Some(memory_counts([("num_namespaces", count)])), + None, + )) +} + +/// Deletes a specific document from a namespace. +pub async fn memory_delete_document( + request: DeleteDocumentRequest, +) -> Result>, String> { + let client = active_memory_client().await?; + let raw = client + .delete_document(&request.namespace, &request.document_id) + .await?; + let parsed: RawDeleteDocumentResult = + serde_json::from_value(raw).map_err(|e| format!("decode delete document result: {e}"))?; + Ok(envelope( + DeleteDocumentResponse { + status: if parsed.deleted { + "completed".to_string() + } else { + "not_found".to_string() + }, + namespace: parsed.namespace, + document_id: parsed.document_id, + deleted: parsed.deleted, + }, + None, + None, + )) +} + +/// Performs a semantic query against a namespace, returning a retrieval context. +pub async fn memory_query_namespace( + request: QueryNamespaceRequest, +) -> Result>, String> { + let include_references = request.include_references.unwrap_or(true); + let requested_limit = request.resolved_limit() as usize; + let result = async { + let client = active_memory_client().await?; + let retrieval_limit = query_limit_for_request(client.as_ref(), &request).await?; + let mut context = client + .query_namespace_context_data(&request.namespace, &request.query, retrieval_limit) + .await?; + context.hits = filter_hits_by_document_ids(context.hits, request.document_ids.as_deref()); + // `query_limit_for_request` may have over-fetched on purpose so that + // the document_id filter has enough candidates; truncate back to what + // the caller actually asked for. + if context.hits.len() > requested_limit { + context.hits.truncate(requested_limit); + } + Ok::(context) + } + .await; + + match result { + Ok(context) => { + let retrieval_context = build_retrieval_context(&context.hits); + let counts = memory_counts([ + ("num_entities", retrieval_context.entities.len()), + ("num_relations", retrieval_context.relations.len()), + ("num_chunks", retrieval_context.chunks.len()), + ]); + let llm_context_message = + format_llm_context_message(Some(&request.query), &context.hits); + Ok(envelope( + QueryNamespaceResponse { + context: maybe_retrieval_context(include_references, retrieval_context), + llm_context_message, + }, + Some(counts), + None, + )) + } + Err(message) => Ok(error_envelope("memory.query_namespace_failed", message)), + } +} + +/// Recalls contextual data from a namespace without a specific query. +pub async fn memory_recall_context( + request: RecallContextRequest, +) -> Result>, String> { + let include_references = request.include_references.unwrap_or(true); + let result = async { + let client = active_memory_client().await?; + client + .recall_namespace_context_data(&request.namespace, request.resolved_limit()) + .await + } + .await; + + match result { + Ok(context) => { + let retrieval_context = build_retrieval_context(&context.hits); + let counts = memory_counts([ + ("num_entities", retrieval_context.entities.len()), + ("num_relations", retrieval_context.relations.len()), + ("num_chunks", retrieval_context.chunks.len()), + ]); + let llm_context_message = format_llm_context_message(None, &context.hits); + Ok(envelope( + RecallContextResponse { + context: maybe_retrieval_context(include_references, retrieval_context), + llm_context_message, + }, + Some(counts), + None, + )) + } + Err(message) => Ok(error_envelope("memory.recall_context_failed", message)), + } +} + +/// Recalls memory items from a namespace with optional retention filtering. +pub async fn memory_recall_memories( + request: RecallMemoriesRequest, +) -> Result>, String> { + let result = async { + let client = active_memory_client().await?; + client + .recall_namespace_memories(&request.namespace, request.resolved_limit()) + .await + } + .await; + + match result { + Ok(hits) => { + let memories = hits + .into_iter() + .map(|hit| MemoryRecallItem { + kind: memory_kind_label(&hit.kind).to_string(), + id: hit.id, + content: hit.content, + score: hit.score, + retention: None, + last_accessed_at: None, + access_count: None, + stability_days: None, + }) + .collect::>(); + let count = memories.len(); + Ok(envelope( + RecallMemoriesResponse { memories }, + Some(memory_counts([("num_memories", count)])), + None, + )) + } + Err(message) => Ok(error_envelope("memory.recall_memories_failed", message)), + } +} diff --git a/src/openhuman/memory/ops/envelope.rs b/src/openhuman/memory/ops/envelope.rs new file mode 100644 index 000000000..265888853 --- /dev/null +++ b/src/openhuman/memory/ops/envelope.rs @@ -0,0 +1,82 @@ +//! Response envelope helpers shared across memory RPC handlers. +//! +//! These helpers standardise the `ApiEnvelope`/`ApiError` wrapping used by the +//! envelope-style memory RPC methods (init, list_documents, query_namespace, +//! recall_*, ai_*_memory_file). + +use std::collections::BTreeMap; + +use serde::Serialize; + +use crate::openhuman::memory::{ApiEnvelope, ApiError, ApiMeta, PaginationMeta}; +use crate::rpc::RpcOutcome; + +/// Generates a unique request ID for memory operations. +/// +/// This ID is used for tracing and logging purposes in the API response metadata. +pub(crate) fn memory_request_id() -> String { + uuid::Uuid::new_v4().to_string() +} + +/// Converts an iterator of memory counts into a BTreeMap. +/// +/// This is a convenience helper for populating the `counts` field in the API metadata. +pub(crate) fn memory_counts( + entries: impl IntoIterator, +) -> BTreeMap { + entries + .into_iter() + .map(|(key, value)| (key.to_string(), value)) + .collect() +} + +/// Wraps data in an RPC API envelope. +/// +/// This standardises the response format for memory-related RPC methods. +pub(crate) fn envelope( + data: T, + counts: Option>, + pagination: Option, +) -> RpcOutcome> { + RpcOutcome::new( + ApiEnvelope { + data: Some(data), + error: None, + meta: ApiMeta { + request_id: memory_request_id(), + latency_seconds: None, + cached: None, + counts, + pagination, + }, + }, + vec![], + ) +} + +/// Wraps an error in an RPC API envelope. +/// +/// This provides a consistent error reporting format for the memory system. +pub(crate) fn error_envelope( + code: &str, + message: String, +) -> RpcOutcome> { + RpcOutcome::new( + ApiEnvelope { + data: None, + error: Some(ApiError { + code: code.to_string(), + message, + details: None, + }), + meta: ApiMeta { + request_id: memory_request_id(), + latency_seconds: None, + cached: None, + counts: None, + pagination: None, + }, + }, + vec![], + ) +} diff --git a/src/openhuman/memory/ops/files.rs b/src/openhuman/memory/ops/files.rs new file mode 100644 index 000000000..91452d086 --- /dev/null +++ b/src/openhuman/memory/ops/files.rs @@ -0,0 +1,104 @@ +//! File-based memory RPC handlers (`ai_list_memory_files`, +//! `ai_read_memory_file`, `ai_write_memory_file`). +//! +//! All filesystem I/O here is performed via `tokio::fs` so the handlers stay +//! async-friendly and never block the executor. + +use crate::openhuman::memory::{ + ApiEnvelope, ListMemoryFilesRequest, ListMemoryFilesResponse, ReadMemoryFileRequest, + ReadMemoryFileResponse, WriteMemoryFileRequest, WriteMemoryFileResponse, +}; +use crate::rpc::RpcOutcome; + +use super::envelope::{envelope, memory_counts}; +use super::helpers::{ + resolve_existing_memory_path, resolve_writable_memory_path, validate_memory_relative_path, +}; + +/// Lists files in a memory directory. +pub async fn ai_list_memory_files( + request: ListMemoryFilesRequest, +) -> Result>, String> { + validate_memory_relative_path(&request.relative_dir)?; + let directory = resolve_existing_memory_path(&request.relative_dir).await?; + if !directory.is_dir() { + return Err(format!( + "memory directory not found: {}", + directory.display() + )); + } + let mut files = Vec::new(); + let mut read_dir = tokio::fs::read_dir(&directory) + .await + .map_err(|e| format!("read memory directory {}: {e}", directory.display()))?; + while let Some(entry) = read_dir + .next_entry() + .await + .map_err(|e| format!("read memory directory entry: {e}"))? + { + // Skip subdirectories and symlinks — `ai_read_memory_file` only + // consumes regular file entries, and surfacing other entry kinds + // here would just produce confusing follow-up read errors. + let file_type = entry + .file_type() + .await + .map_err(|e| format!("read memory directory entry type: {e}"))?; + if !file_type.is_file() { + continue; + } + let file_name = entry.file_name(); + let file_name = file_name.to_string_lossy(); + if !file_name.is_empty() { + files.push(file_name.to_string()); + } + } + files.sort(); + let count = files.len(); + Ok(envelope( + ListMemoryFilesResponse { + relative_dir: request.relative_dir, + files, + count, + }, + Some(memory_counts([("num_files", count)])), + None, + )) +} + +/// Reads the contents of a memory file. +pub async fn ai_read_memory_file( + request: ReadMemoryFileRequest, +) -> Result>, String> { + let path = resolve_existing_memory_path(&request.relative_path).await?; + let content = tokio::fs::read_to_string(&path) + .await + .map_err(|e| format!("read memory file {}: {e}", path.display()))?; + Ok(envelope( + ReadMemoryFileResponse { + relative_path: request.relative_path, + content, + }, + None, + None, + )) +} + +/// Writes content to a memory file. +pub async fn ai_write_memory_file( + request: WriteMemoryFileRequest, +) -> Result>, String> { + let path = resolve_writable_memory_path(&request.relative_path).await?; + tokio::fs::write(&path, request.content.as_bytes()) + .await + .map_err(|e| format!("write memory file {}: {e}", path.display()))?; + let bytes_written = request.content.len(); + Ok(envelope( + WriteMemoryFileResponse { + relative_path: request.relative_path, + written: true, + bytes_written, + }, + None, + None, + )) +} diff --git a/src/openhuman/memory/ops/helpers.rs b/src/openhuman/memory/ops/helpers.rs new file mode 100644 index 000000000..45037a00b --- /dev/null +++ b/src/openhuman/memory/ops/helpers.rs @@ -0,0 +1,473 @@ +//! Formatting helpers, default constants, path validators, and the active +//! memory-client lookup. Shared internals for the memory RPC handlers. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Component, Path, PathBuf}; + +use chrono::TimeZone; +use serde::Deserialize; +use serde_json::{json, Value}; + +use crate::openhuman::config::Config; +use crate::openhuman::memory::store::GraphRelationRecord; +use crate::openhuman::memory::{ + MemoryClient, MemoryClientRef, MemoryDocumentSummary, MemoryItemKind, MemoryRetrievalChunk, + MemoryRetrievalContext, MemoryRetrievalEntity, MemoryRetrievalRelation, NamespaceMemoryHit, + QueryNamespaceRequest, +}; + +// --------------------------------------------------------------------------- +// Formatting helpers +// --------------------------------------------------------------------------- + +/// Formats a floating-point timestamp as an RFC3339 string. +/// +/// Returns `None` if the timestamp is invalid (NaN, infinite, or negative). +pub(crate) fn timestamp_to_rfc3339(timestamp: f64) -> Option { + if !timestamp.is_finite() || timestamp < 0.0 { + return None; + } + + let secs = timestamp.trunc() as i64; + let nanos = ((timestamp.fract().abs()) * 1_000_000_000.0).round() as u32; + chrono::Utc + .timestamp_opt(secs, nanos.min(999_999_999)) + .single() + .map(|value| value.to_rfc3339()) +} + +/// Maps a memory item kind to a human-readable label. +pub(crate) fn memory_kind_label(kind: &MemoryItemKind) -> &'static str { + match kind { + MemoryItemKind::Document => "document", + MemoryItemKind::Kv => "kv", + MemoryItemKind::Episodic => "episodic", + MemoryItemKind::Event => "event", + } +} + +/// Generates a unique string identity for a graph relation. +/// +/// The identity is composed of the namespace, subject, predicate, and object. +pub(crate) fn relation_identity(relation: &GraphRelationRecord) -> String { + format!( + "{}|{}|{}|{}", + relation.namespace.as_deref().unwrap_or("global"), + relation.subject.as_str(), + relation.predicate.as_str(), + relation.object.as_str() + ) +} + +/// Formats relation metadata into a JSON Value. +pub(crate) fn relation_metadata(relation: &GraphRelationRecord) -> Value { + json!({ + "namespace": relation.namespace.clone(), + "attrs": relation.attrs.clone(), + "order_index": relation.order_index, + "document_ids": relation.document_ids.clone(), + "chunk_ids": relation.chunk_ids.clone(), + "updated_at": timestamp_to_rfc3339(relation.updated_at), + }) +} + +/// Formats chunk metadata into a JSON Value. +pub(crate) fn chunk_metadata(hit: &NamespaceMemoryHit) -> Value { + json!({ + "kind": memory_kind_label(&hit.kind), + "namespace": hit.namespace.clone(), + "key": hit.key.clone(), + "title": hit.title.clone(), + "category": hit.category.clone(), + "source_type": hit.source_type.clone(), + "score_breakdown": { + "keyword_relevance": hit.score_breakdown.keyword_relevance, + "vector_similarity": hit.score_breakdown.vector_similarity, + "graph_relevance": hit.score_breakdown.graph_relevance, + "episodic_relevance": hit.score_breakdown.episodic_relevance, + "freshness": hit.score_breakdown.freshness, + "final_score": hit.score_breakdown.final_score, + } + }) +} + +/// Extracts an entity type for a specific role (subject/object) from relation attributes. +pub(crate) fn extract_entity_type(attrs: &Value, role: &str) -> Option { + attrs + .get("entity_types") + .and_then(|et| et.get(role)) + .and_then(|v| v.as_str()) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()) +} + +/// Transforms memory hits into a retrieval context with deduplicated entities and relations. +pub(crate) fn build_retrieval_context(hits: &[NamespaceMemoryHit]) -> MemoryRetrievalContext { + let mut entity_types: BTreeMap> = BTreeMap::new(); + let mut relations = BTreeMap::new(); + let chunks = hits + .iter() + .map(|hit| { + // Extract supporting relations from each hit to populate entities and relations + for relation in &hit.supporting_relations { + if !relation.subject.trim().is_empty() { + let entry = entity_types.entry(relation.subject.clone()).or_insert(None); + // Use the first non-empty entity type found for this subject + if entry.is_none() { + *entry = extract_entity_type(&relation.attrs, "subject"); + } + } + if !relation.object.trim().is_empty() { + let entry = entity_types.entry(relation.object.clone()).or_insert(None); + // Use the first non-empty entity type found for this object + if entry.is_none() { + *entry = extract_entity_type(&relation.attrs, "object"); + } + } + // Deduplicate relations based on their unique identity + relations + .entry(relation_identity(relation)) + .or_insert_with(|| MemoryRetrievalRelation { + subject: relation.subject.clone(), + predicate: relation.predicate.clone(), + object: relation.object.clone(), + score: None, + evidence_count: Some(relation.evidence_count), + metadata: relation_metadata(relation), + }); + } + + MemoryRetrievalChunk { + chunk_id: hit.chunk_id.clone(), + document_id: hit.document_id.clone(), + content: hit.content.clone(), + score: hit.score, + metadata: chunk_metadata(hit), + created_at: None, + updated_at: timestamp_to_rfc3339(hit.updated_at), + } + }) + .collect(); + + MemoryRetrievalContext { + entities: entity_types + .into_iter() + .map(|(name, entity_type)| MemoryRetrievalEntity { + id: None, + name, + entity_type, + score: None, + metadata: json!({}), + }) + .collect(), + relations: relations.into_values().collect(), + chunks, + } +} + +/// Formats memory hits into a natural-language context message for LLM consumption. +pub(crate) fn format_llm_context_message( + query: Option<&str>, + hits: &[NamespaceMemoryHit], +) -> Option { + if hits.is_empty() { + return None; + } + + let mut parts = Vec::new(); + if let Some(query) = query { + parts.push(format!("Query: {query}")); + } + + for hit in hits { + let summary = match hit.kind { + MemoryItemKind::Document => { + let title = hit.title.clone().unwrap_or_else(|| hit.key.clone()); + format!("{title}: {}", hit.content.trim()) + } + MemoryItemKind::Kv => format!("[kv:{}] {}", hit.key, hit.content.trim()), + MemoryItemKind::Episodic => { + format!("[episodic:{}] {}", hit.key, hit.content.trim()) + } + MemoryItemKind::Event => { + format!("[event:{}] {}", hit.key, hit.content.trim()) + } + }; + parts.push(summary); + + // Include typed relations if present for better LLM reasoning + if !hit.supporting_relations.is_empty() { + let relations = hit + .supporting_relations + .iter() + .map(|relation| { + let subject_type = extract_entity_type(&relation.attrs, "subject"); + let object_type = extract_entity_type(&relation.attrs, "object"); + let subject_label = match subject_type { + Some(t) => format!("{} ({})", relation.subject, t), + None => relation.subject.clone(), + }; + let object_label = match object_type { + Some(t) => format!("{} ({})", relation.object, t), + None => relation.object.clone(), + }; + format!( + "{} -[{}]-> {}", + subject_label, relation.predicate, object_label + ) + }) + .collect::>() + .join("; "); + parts.push(format!("Relations: {relations}")); + } + } + + Some(parts.join("\n\n")) +} + +/// Filters memory hits to only include those matching specific document IDs. +pub(crate) fn filter_hits_by_document_ids( + hits: Vec, + document_ids: Option<&[String]>, +) -> Vec { + let Some(document_ids) = document_ids else { + return hits; + }; + let allowed = document_ids.iter().cloned().collect::>(); + hits.into_iter() + .filter(|hit| { + hit.document_id + .as_ref() + .map(|document_id| allowed.contains(document_id)) + .unwrap_or(false) + }) + .collect() +} + +/// Returns the retrieval context if `include_references` is true and context is not empty. +pub(crate) fn maybe_retrieval_context( + include_references: bool, + context: MemoryRetrievalContext, +) -> Option { + if !include_references { + return None; + } + if context.entities.is_empty() && context.relations.is_empty() && context.chunks.is_empty() { + return None; + } + Some(context) +} + +// --------------------------------------------------------------------------- +// Default constants +// --------------------------------------------------------------------------- + +pub(crate) fn default_source_type() -> String { + "doc".to_string() +} + +pub(crate) fn default_priority() -> String { + "medium".to_string() +} + +pub(crate) fn default_category() -> String { + "core".to_string() +} + +// --------------------------------------------------------------------------- +// Workspace + memory-client lookup +// --------------------------------------------------------------------------- + +/// Subdirectory under the workspace where the file-based memory RPCs operate. +/// `ai_*_memory_file` handlers MUST resolve all caller-supplied relative paths +/// against this directory — never the workspace root — to avoid leaking access +/// to repo files such as `Cargo.toml`, `.env`, or source files. +const MEMORY_SUBDIR: &str = "memory"; + +/// Returns the current workspace directory from configuration. +pub(crate) async fn current_workspace_dir() -> Result { + Config::load_or_init() + .await + .map(|config| config.workspace_dir) + .map_err(|e| format!("load config: {e}")) +} + +/// Returns the active memory client from the process-global singleton, +/// auto-initialising from the configured workspace if startup wiring hasn't +/// done so yet. +/// +/// The auto-init resolves the workspace via [`current_workspace_dir`], which +/// goes through `Config::load_or_init` — the same path startup wiring uses. +/// It does **not** fall back to `~/.openhuman/workspace`; that hazard is the +/// one [`crate::openhuman::memory::global::client`] guards against, and it +/// remains guarded for any caller that bypasses this helper. +pub(crate) async fn active_memory_client() -> Result { + if let Some(client) = super::super::global::client_if_ready() { + return Ok(client); + } + let workspace_dir = current_workspace_dir().await?; + super::super::global::init(workspace_dir) +} + +// --------------------------------------------------------------------------- +// Path validators (used by file-based memory handlers) +// --------------------------------------------------------------------------- + +/// Validates that a relative path does not escape the memory directory. +/// +/// An empty path is allowed and refers to the memory root itself +/// (`/memory`); read-style helpers can resolve it to that +/// directory. Write helpers reject empty paths separately because they +/// require a file name component. +pub(crate) fn validate_memory_relative_path(path: &str) -> Result<(), String> { + let candidate = Path::new(path); + if candidate.as_os_str().is_empty() { + return Ok(()); + } + if candidate.is_absolute() { + return Err("absolute paths are not allowed".to_string()); + } + // Prevent traversal using .. components + for component in candidate.components() { + match component { + Component::ParentDir | Component::RootDir | Component::Prefix(_) => { + return Err("path traversal is not allowed".to_string()); + } + _ => {} + } + } + Ok(()) +} + +/// Resolves the canonical path to the memory directory within the workspace. +pub(crate) async fn resolve_memory_root() -> Result { + let workspace_dir = current_workspace_dir().await?; + let memory_root = workspace_dir.join(MEMORY_SUBDIR); + tokio::fs::create_dir_all(&memory_root) + .await + .map_err(|e| format!("create memory dir {}: {e}", memory_root.display()))?; + memory_root + .canonicalize() + .map_err(|e| format!("resolve memory dir {}: {e}", memory_root.display())) +} + +/// Resolves and canonicalizes an existing memory path, ensuring it stays within +/// the `/memory` directory (not the workspace root). An empty +/// `relative_path` resolves to the memory root itself. +pub(crate) async fn resolve_existing_memory_path(relative_path: &str) -> Result { + validate_memory_relative_path(relative_path)?; + let memory_root = resolve_memory_root().await?; + let full_path = if relative_path.is_empty() { + memory_root.clone() + } else { + memory_root.join(relative_path) + }; + let resolved = full_path + .canonicalize() + .map_err(|e| format!("resolve memory path {}: {e}", full_path.display()))?; + if !resolved.starts_with(&memory_root) { + return Err("memory path escapes the memory directory".to_string()); + } + Ok(resolved) +} + +/// Resolves a path for writing, creating parent directories and ensuring it +/// stays within the `/memory` directory (not the workspace root). +pub(crate) async fn resolve_writable_memory_path(relative_path: &str) -> Result { + validate_memory_relative_path(relative_path)?; + let memory_root = resolve_memory_root().await?; + let full_path = memory_root.join(relative_path); + let parent = full_path + .parent() + .ok_or_else(|| "memory path must include a file name".to_string())?; + tokio::fs::create_dir_all(parent) + .await + .map_err(|e| format!("create memory path {}: {e}", parent.display()))?; + let resolved_parent = parent + .canonicalize() + .map_err(|e| format!("resolve memory parent {}: {e}", parent.display()))?; + if !resolved_parent.starts_with(&memory_root) { + return Err("memory path escapes the memory directory".to_string()); + } + let file_name = full_path + .file_name() + .ok_or_else(|| "memory path must include a file name".to_string())?; + let resolved = resolved_parent.join(file_name); + // Security check: refuse to write through symlinks to prevent hijacking + if let Ok(metadata) = tokio::fs::symlink_metadata(&resolved).await { + if metadata.file_type().is_symlink() { + return Err(format!( + "refusing to write through symlink: {}", + resolved.display() + )); + } + } + Ok(resolved) +} + +// --------------------------------------------------------------------------- +// Document summary parsing + query-limit resolution (shared by documents.rs) +// --------------------------------------------------------------------------- + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct RawMemoryDocumentSummary { + document_id: String, + namespace: String, + key: String, + title: String, + source_type: String, + priority: String, + created_at: f64, + updated_at: f64, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct RawDeleteDocumentResult { + pub deleted: bool, + pub namespace: String, + pub document_id: String, +} + +pub(crate) fn parse_memory_document_summaries( + raw: Value, +) -> Result, String> { + let documents = raw + .get("documents") + .and_then(Value::as_array) + .ok_or_else(|| "memory document list missing 'documents' array".to_string())?; + documents + .iter() + .cloned() + .map(|value| { + let raw: RawMemoryDocumentSummary = serde_json::from_value(value) + .map_err(|e| format!("decode memory document: {e}"))?; + Ok(MemoryDocumentSummary { + document_id: raw.document_id, + namespace: raw.namespace, + key: raw.key, + title: raw.title, + source_type: raw.source_type, + priority: raw.priority, + created_at: raw.created_at, + updated_at: raw.updated_at, + }) + }) + .collect() +} + +pub(crate) async fn query_limit_for_request( + client: &MemoryClient, + request: &QueryNamespaceRequest, +) -> Result { + let requested = request.resolved_limit(); + if request.document_ids.is_none() { + return Ok(requested); + } + + let raw = client.list_documents(Some(&request.namespace)).await?; + let documents = parse_memory_document_summaries(raw)?; + let total_documents = u32::try_from(documents.len()).unwrap_or(u32::MAX); + Ok(requested.max(total_documents)) +} diff --git a/src/openhuman/memory/ops/kv_graph.rs b/src/openhuman/memory/ops/kv_graph.rs new file mode 100644 index 000000000..3992288fb --- /dev/null +++ b/src/openhuman/memory/ops/kv_graph.rs @@ -0,0 +1,136 @@ +//! Key-value and knowledge-graph RPC handlers for the unified memory store. + +use serde::Deserialize; + +use crate::rpc::RpcOutcome; + +use super::helpers::active_memory_client; + +/// Parameters for the `kv_set` RPC method. +#[derive(Debug, Deserialize)] +pub struct KvSetParams { + /// The namespace for the key-value pair. + #[serde(default)] + pub namespace: Option, + /// The unique key. + pub key: String, + /// The value to store. + pub value: serde_json::Value, +} + +/// Parameters for `kv_get` and `kv_delete` RPC methods. +#[derive(Debug, Deserialize)] +pub struct KvGetDeleteParams { + /// The namespace containing the key. + #[serde(default)] + pub namespace: Option, + /// The unique key. + pub key: String, +} + +/// Parameters for the `graph_upsert` RPC method. +#[derive(Debug, Deserialize)] +pub struct GraphUpsertParams { + /// The namespace for the relation. + #[serde(default)] + pub namespace: Option, + /// The subject of the relation triple. + pub subject: String, + /// The predicate (relationship) of the triple. + pub predicate: String, + /// The object of the triple. + pub object: String, + /// Additional attributes for the relation. + #[serde(default)] + pub attrs: serde_json::Value, +} + +/// Parameters for the `graph_query` RPC method. +#[derive(Debug, Deserialize)] +pub struct GraphQueryParams { + /// The namespace to query. + #[serde(default)] + pub namespace: Option, + /// Optional subject filter. + #[serde(default)] + pub subject: Option, + /// Optional predicate filter. + #[serde(default)] + pub predicate: Option, +} + +// --------------------------------------------------------------------------- +// KV handlers +// --------------------------------------------------------------------------- + +/// Sets a key-value pair in the memory store. +pub async fn kv_set(params: KvSetParams) -> Result, String> { + let client = active_memory_client().await?; + client + .kv_set(params.namespace.as_deref(), ¶ms.key, ¶ms.value) + .await?; + Ok(RpcOutcome::single_log(true, "memory kv set")) +} + +/// Retrieves a value by key from the memory store. +pub async fn kv_get( + params: KvGetDeleteParams, +) -> Result>, String> { + let client = active_memory_client().await?; + let value = client + .kv_get(params.namespace.as_deref(), ¶ms.key) + .await?; + Ok(RpcOutcome::single_log(value, "memory kv get")) +} + +/// Deletes a key-value pair from the memory store. +pub async fn kv_delete(params: KvGetDeleteParams) -> Result, String> { + let client = active_memory_client().await?; + let deleted = client + .kv_delete(params.namespace.as_deref(), ¶ms.key) + .await?; + Ok(RpcOutcome::single_log(deleted, "memory kv delete")) +} + +/// Lists all key-value entries in a namespace. +pub async fn kv_list_namespace( + params: super::documents::NamespaceOnlyParams, +) -> Result>, String> { + let client = active_memory_client().await?; + let rows = client.kv_list_namespace(¶ms.namespace).await?; + Ok(RpcOutcome::single_log(rows, "memory namespace kv listed")) +} + +// --------------------------------------------------------------------------- +// Graph handlers +// --------------------------------------------------------------------------- + +/// Upserts a relation triple in the knowledge graph. +pub async fn graph_upsert(params: GraphUpsertParams) -> Result, String> { + let client = active_memory_client().await?; + client + .graph_upsert( + params.namespace.as_deref(), + ¶ms.subject, + ¶ms.predicate, + ¶ms.object, + ¶ms.attrs, + ) + .await?; + Ok(RpcOutcome::single_log(true, "memory graph upserted")) +} + +/// Queries relations from the knowledge graph. +pub async fn graph_query( + params: GraphQueryParams, +) -> Result>, String> { + let client = active_memory_client().await?; + let rows = client + .graph_query( + params.namespace.as_deref(), + params.subject.as_deref(), + params.predicate.as_deref(), + ) + .await?; + Ok(RpcOutcome::single_log(rows, "memory graph queried")) +} diff --git a/src/openhuman/memory/ops/learn.rs b/src/openhuman/memory/ops/learn.rs new file mode 100644 index 000000000..a20a8258f --- /dev/null +++ b/src/openhuman/memory/ops/learn.rs @@ -0,0 +1,152 @@ +//! `memory_learn_all` — runs the tree summarizer over namespaces sequentially. + +use std::collections::BTreeSet; + +use crate::rpc::RpcOutcome; + +use super::helpers::active_memory_client; + +/// Per-namespace outcome for `memory_learn_all`. +#[derive(Debug, serde::Serialize)] +pub struct NamespaceLearnResult { + pub namespace: String, + pub status: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +/// Result returned by `memory_learn_all`. +#[derive(Debug, serde::Serialize)] +pub struct LearnAllResult { + pub namespaces_processed: usize, + pub results: Vec, +} + +/// Parameters for `memory_learn_all`. +#[derive(Debug, serde::Deserialize)] +pub struct LearnAllParams { + /// Optional list of namespaces to constrain. Defaults to all namespaces. + #[serde(default)] + pub namespaces: Option>, +} + +/// Run the tree summarizer over all (or a constrained set of) namespaces. +/// +/// Enumerates namespaces via `namespace_list`, then for each runs +/// `tree_summarizer_run`. Results are collected per-namespace; a failing +/// namespace does not abort the rest. Runs sequentially to avoid saturating +/// the local AI provider. +pub async fn memory_learn_all( + params: LearnAllParams, +) -> Result, String> { + tracing::info!( + "[memory.learn] memory_learn_all: entry namespaces={:?}", + params.namespaces + ); + + // Resolve the target namespace list. + let client = active_memory_client().await?; + let all_ns = client.list_namespaces().await?; + tracing::debug!("[memory.learn] available namespaces: {:?}", all_ns); + + let target_ns: Vec = match ¶ms.namespaces { + Some(requested) if !requested.is_empty() => { + let mut seen = BTreeSet::new(); + let filtered: Vec<_> = requested + .iter() + .filter(|ns| all_ns.contains(ns)) + .filter(|ns| seen.insert((*ns).clone())) + .cloned() + .collect(); + tracing::debug!("[memory.learn] constrained to namespaces: {:?}", filtered); + filtered + } + Some(requested) => { + // Explicit empty list → no-op (don't fall back to all namespaces). + let mut seen = BTreeSet::new(); + let filtered: Vec<_> = requested + .iter() + .filter(|ns| all_ns.contains(ns)) + .filter(|ns| seen.insert((*ns).clone())) + .cloned() + .collect(); + tracing::debug!( + "[memory.learn] Some([]) empty request → no-op or filtered to {:?}", + filtered + ); + filtered + } + None => { + tracing::debug!("[memory.learn] using all {} namespaces", all_ns.len()); + all_ns + } + }; + + // Short-circuit when there are no namespaces to process — avoids loading + // config (and the local_ai.enabled guard) for an empty batch. + if target_ns.is_empty() { + tracing::info!( + "[memory.learn] memory_learn_all: no namespaces to process, returning early" + ); + return Ok(RpcOutcome::new( + LearnAllResult { + namespaces_processed: 0, + results: vec![], + }, + vec![], + )); + } + + let config = crate::openhuman::config::Config::load_or_init() + .await + .map_err(|e| format!("load config: {e}"))?; + + if !config.local_ai.enabled { + return Err("memory_learn_all requires local_ai.enabled=true".to_string()); + } + + let mut results = Vec::with_capacity(target_ns.len()); + for namespace in &target_ns { + tracing::info!( + "[memory.learn] running summarization for namespace='{}'", + namespace + ); + let outcome = + crate::openhuman::tree_summarizer::ops::tree_summarizer_run(&config, namespace).await; + match outcome { + Ok(_) => { + tracing::info!("[memory.learn] namespace='{}' ok", namespace); + results.push(NamespaceLearnResult { + namespace: namespace.clone(), + status: "ok".to_string(), + error: None, + }); + } + Err(e) => { + tracing::warn!("[memory.learn] namespace='{}' error: {}", namespace, e); + results.push(NamespaceLearnResult { + namespace: namespace.clone(), + status: "error".to_string(), + error: Some(e), + }); + } + } + } + + let namespaces_processed = results.len(); + tracing::info!( + "[memory.learn] memory_learn_all: done processed={} results={:?}", + namespaces_processed, + results + .iter() + .map(|r| (&r.namespace, &r.status)) + .collect::>() + ); + Ok(RpcOutcome::new( + LearnAllResult { + namespaces_processed, + results, + }, + vec![], + )) +} diff --git a/src/openhuman/memory/ops/mod.rs b/src/openhuman/memory/ops/mod.rs new file mode 100644 index 000000000..55c17999f --- /dev/null +++ b/src/openhuman/memory/ops/mod.rs @@ -0,0 +1,69 @@ +//! RPC operations for the memory system. +//! +//! This module implements the handlers for memory-related RPC requests, including +//! document management, semantic queries, key-value storage, and knowledge graph +//! operations. It manages the active memory client and provides utility functions +//! for formatting and filtering memory results. +//! +//! Internally the implementation is split across submodules by RPC family: +//! +//! - [`envelope`] — `ApiEnvelope`/`ApiError` wrapping helpers shared by every +//! envelope-style handler. +//! - [`helpers`] — formatting, default constants, path validators, and the +//! active memory-client lookup. +//! - [`documents`] — document/namespace direct API and the envelope-style +//! façade (`memory_init`, `memory_list_documents`, `memory_query_namespace`, +//! recall_*). +//! - [`kv_graph`] — key-value and knowledge-graph handlers. +//! - [`sync`] — `memory_sync_*` and `memory_ingestion_status`. +//! - [`learn`] — `memory_learn_all`. +//! - [`files`] — `ai_*_memory_file` handlers (use `tokio::fs`). + +pub mod documents; +pub mod envelope; +pub mod files; +pub mod helpers; +pub mod kv_graph; +pub mod learn; +pub mod sync; + +// --------------------------------------------------------------------------- +// Re-exports preserving the previous flat `memory::ops::*` surface. +// --------------------------------------------------------------------------- + +pub use documents::{ + clear_namespace, context_query, context_recall, doc_delete, doc_ingest, doc_list, doc_put, + memory_delete_document, memory_init, memory_list_documents, memory_list_namespaces, + memory_query_namespace, memory_recall_context, memory_recall_memories, namespace_list, + ClearNamespaceParams, ClearNamespaceResult, DeleteDocParams, IngestDocParams, + NamespaceOnlyParams, PutDocParams, PutDocResult, QueryNamespaceParams, RecallNamespaceParams, +}; +pub use files::{ai_list_memory_files, ai_read_memory_file, ai_write_memory_file}; +pub use kv_graph::{ + graph_query, graph_upsert, kv_delete, kv_get, kv_list_namespace, kv_set, GraphQueryParams, + GraphUpsertParams, KvGetDeleteParams, KvSetParams, +}; +pub use learn::{memory_learn_all, LearnAllParams, LearnAllResult, NamespaceLearnResult}; +pub use sync::{ + memory_ingestion_status, memory_sync_all, memory_sync_channel, IngestionStatusResult, + SyncAllResult, SyncChannelParams, SyncChannelResult, +}; + +// --------------------------------------------------------------------------- +// Test-only re-exports — keep the existing `ops_tests.rs` happy without +// changing the test file. The tests reference private helpers via `super::*`. +// --------------------------------------------------------------------------- + +#[cfg(test)] +pub(crate) use envelope::{error_envelope, memory_counts, memory_request_id}; +#[cfg(test)] +pub(crate) use helpers::{ + build_retrieval_context, chunk_metadata, default_category, default_priority, + default_source_type, extract_entity_type, filter_hits_by_document_ids, + format_llm_context_message, maybe_retrieval_context, memory_kind_label, relation_identity, + relation_metadata, timestamp_to_rfc3339, validate_memory_relative_path, +}; + +#[cfg(test)] +#[path = "../ops_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/ops/sync.rs b/src/openhuman/memory/ops/sync.rs new file mode 100644 index 000000000..c29542a60 --- /dev/null +++ b/src/openhuman/memory/ops/sync.rs @@ -0,0 +1,112 @@ +//! Memory-sync RPC handlers and ingestion-status reporting. +//! +//! Sync RPCs publish `DomainEvent::MemorySyncRequested` on the global event +//! bus — they are fire-and-forget hooks for future ingestion subscribers. + +use crate::rpc::RpcOutcome; + +/// Parameters for `memory_sync_channel`. +#[derive(Debug, serde::Deserialize)] +pub struct SyncChannelParams { + pub channel_id: String, +} + +/// Result returned by `memory_sync_channel`. +#[derive(Debug, serde::Serialize)] +pub struct SyncChannelResult { + pub requested: bool, + pub channel_id: String, +} + +/// Result returned by `memory_sync_all`. +#[derive(Debug, serde::Serialize)] +pub struct SyncAllResult { + pub requested: bool, +} + +/// Result returned by `memory_ingestion_status`. Mirrors +/// [`crate::openhuman::memory::IngestionStatusSnapshot`] but is the public RPC +/// shape — the indirection keeps internal renames from breaking the wire +/// contract. +#[derive(Debug, Clone, Default, serde::Serialize)] +pub struct IngestionStatusResult { + pub running: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub current_document_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub current_title: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub current_namespace: Option, + pub queue_depth: usize, + #[serde(skip_serializing_if = "Option::is_none")] + pub last_completed_at: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub last_document_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub last_success: Option, +} + +/// Request a memory sync for a specific channel. +/// +/// Ingestion in OpenHuman is listener/webhook-driven — there is no per-provider +/// pull mechanism yet. This RPC publishes `DomainEvent::MemorySyncRequested` so +/// that future ingestion subscribers can react to an explicit pull request. +/// The event is fire-and-forget; the caller receives confirmation that the +/// request was published, not that ingestion ran. +pub async fn memory_sync_channel( + params: SyncChannelParams, +) -> Result, String> { + // `channel_id` is a user/context identifier — keep it out of normal logs. + tracing::info!("[memory.sync] memory_sync_channel: entry"); + crate::core::event_bus::publish_global( + crate::core::event_bus::DomainEvent::MemorySyncRequested { + channel_id: Some(params.channel_id.clone()), + }, + ); + tracing::debug!("[memory.sync] memory_sync_channel: MemorySyncRequested published"); + Ok(RpcOutcome::new( + SyncChannelResult { + requested: true, + channel_id: params.channel_id, + }, + vec![], + )) +} + +/// Request a memory sync for all channels. +/// +/// Publishes `DomainEvent::MemorySyncRequested { channel_id: None }` on the +/// global event bus. No consumers exist yet — this is a hook for future +/// ingestion subscribers. +pub async fn memory_sync_all() -> Result, String> { + tracing::info!("[memory.sync] memory_sync_all: entry"); + crate::core::event_bus::publish_global( + crate::core::event_bus::DomainEvent::MemorySyncRequested { channel_id: None }, + ); + tracing::debug!("[memory.sync] memory_sync_all: MemorySyncRequested(all) published"); + Ok(RpcOutcome::new(SyncAllResult { requested: true }, vec![])) +} + +/// Returns the current memory-ingestion status: whether a job is running, the +/// in-flight document, queue depth, and the most recent completion. Read-only, +/// safe to poll. +pub async fn memory_ingestion_status() -> Result, String> { + let snapshot = match crate::openhuman::memory::global::client_if_ready() { + Some(c) => c.ingestion_state().snapshot(), + // Memory not yet initialised — report idle, no in-flight job. + None => Default::default(), + }; + Ok(RpcOutcome::new( + IngestionStatusResult { + running: snapshot.running, + current_document_id: snapshot.current_document_id, + current_title: snapshot.current_title, + current_namespace: snapshot.current_namespace, + queue_depth: snapshot.queue_depth, + last_completed_at: snapshot.last_completed_at, + last_document_id: snapshot.last_document_id, + last_success: snapshot.last_success, + }, + vec![], + )) +} diff --git a/src/openhuman/memory/ops_tests.rs b/src/openhuman/memory/ops_tests.rs index 2680a1844..75c47e6b7 100644 --- a/src/openhuman/memory/ops_tests.rs +++ b/src/openhuman/memory/ops_tests.rs @@ -1,3 +1,6 @@ +//! Unit tests for the memory `ops` helpers (retrieval context construction, +//! hit filtering, and LLM context message formatting). + use serde_json::json; use super::{build_retrieval_context, filter_hits_by_document_ids, format_llm_context_message}; @@ -302,7 +305,10 @@ fn default_constants_are_stable() { #[test] fn validate_memory_relative_path_rejects_empty_absolute_and_traversal() { - assert!(validate_memory_relative_path("").is_err()); + // Empty string is now allowed: it refers to the memory root + // (`/memory`) since the file-based RPCs resolve everything + // relative to that directory rather than the workspace root. + assert!(validate_memory_relative_path("").is_ok()); assert!(validate_memory_relative_path("/etc/passwd").is_err()); assert!(validate_memory_relative_path("../secrets").is_err()); assert!(validate_memory_relative_path("ok/subdir/file.md").is_ok()); diff --git a/src/openhuman/memory/response_cache.rs b/src/openhuman/memory/response_cache.rs deleted file mode 100644 index 7cca3b0e4..000000000 --- a/src/openhuman/memory/response_cache.rs +++ /dev/null @@ -1,236 +0,0 @@ -//! # Response Cache -//! -//! Avoid burning tokens and reduce latency on repeated LLM prompts by caching -//! responses. -//! -//! Stores LLM responses in a separate SQLite table keyed by a SHA-256 hash of -//! `(model, system_prompt, user_prompt)`. Entries expire after a -//! configurable TTL (Time To Live). The cache is optional and disabled by -//! default — users opt in via `[memory] response_cache_enabled = true`. -//! -//! The cache is stored in `response_cache.db` within the workspace's `memory` -//! directory. This separation from the main `brain.db` allows the cache to be -//! cleared or deleted without affecting permanent memories. - -use anyhow::Result; -use chrono::{Duration, Local}; -use parking_lot::Mutex; -use rusqlite::{params, Connection}; -use sha2::{Digest, Sha256}; -use std::path::{Path, PathBuf}; - -/// Response cache backed by a dedicated SQLite database. -/// -/// Handles storage, retrieval, and eviction (TTL and LRU) of LLM responses. -pub struct ResponseCache { - /// Thread-safe connection to the SQLite database. - conn: Mutex, - /// Path to the SQLite database file. - #[allow(dead_code)] - db_path: PathBuf, - /// Time-to-live for cache entries in minutes. - ttl_minutes: i64, - /// Maximum number of entries to keep in the cache. - max_entries: usize, -} - -impl ResponseCache { - /// Open (or create) the response cache database. - /// - /// # Arguments - /// - /// * `workspace_dir` - Path to the project workspace. - /// * `ttl_minutes` - How long each entry should remain valid. - /// * `max_entries` - The maximum number of entries to store before LRU eviction. - /// - /// # Errors - /// - /// Returns an error if the database directory cannot be created or if the - /// SQLite connection fails to open. - pub fn new(workspace_dir: &Path, ttl_minutes: u32, max_entries: usize) -> Result { - let db_dir = workspace_dir.join("memory"); - std::fs::create_dir_all(&db_dir)?; - let db_path = db_dir.join("response_cache.db"); - - let conn = Connection::open(&db_path)?; - - // Performance tuning for SQLite. - conn.execute_batch( - "PRAGMA journal_mode = WAL; - PRAGMA synchronous = NORMAL; - PRAGMA temp_store = MEMORY;", - )?; - - // Initialize the cache table and indices. - conn.execute_batch( - "CREATE TABLE IF NOT EXISTS response_cache ( - prompt_hash TEXT PRIMARY KEY, - model TEXT NOT NULL, - response TEXT NOT NULL, - token_count INTEGER NOT NULL DEFAULT 0, - created_at TEXT NOT NULL, - accessed_at TEXT NOT NULL, - hit_count INTEGER NOT NULL DEFAULT 0 - ); - CREATE INDEX IF NOT EXISTS idx_rc_accessed ON response_cache(accessed_at); - CREATE INDEX IF NOT EXISTS idx_rc_created ON response_cache(created_at);", - )?; - - Ok(Self { - conn: Mutex::new(conn), - db_path, - ttl_minutes: i64::from(ttl_minutes), - max_entries, - }) - } - - /// Build a deterministic cache key from model + system prompt + user prompt. - /// - /// Uses SHA-256 to create a unique identifier for the specific combination - /// of model and inputs. - /// - /// # Arguments - /// - /// * `model` - The name of the LLM model (e.g., "gpt-4o"). - /// * `system_prompt` - The optional system instruction prompt. - /// * `user_prompt` - The main user query/prompt. - pub fn cache_key(model: &str, system_prompt: Option<&str>, user_prompt: &str) -> String { - let mut hasher = Sha256::new(); - hasher.update(model.as_bytes()); - hasher.update(b"|"); - if let Some(sys) = system_prompt { - hasher.update(sys.as_bytes()); - } - hasher.update(b"|"); - hasher.update(user_prompt.as_bytes()); - let hash = hasher.finalize(); - format!("{:064x}", hash) - } - - /// Look up a cached response. Returns `None` on miss or expired entry. - /// - /// If a match is found, it updates the `accessed_at` timestamp and - /// increments the `hit_count`. - /// - /// # Arguments - /// - /// * `key` - The SHA-256 cache key generated by [`Self::cache_key`]. - pub fn get(&self, key: &str) -> Result> { - let conn = self.conn.lock(); - - let now = Local::now(); - let cutoff = (now - Duration::minutes(self.ttl_minutes)).to_rfc3339(); - - // Retrieve the response only if it hasn't expired according to the TTL. - let mut stmt = conn.prepare( - "SELECT response FROM response_cache - WHERE prompt_hash = ?1 AND created_at > ?2", - )?; - - let result: Option = stmt.query_row(params![key, cutoff], |row| row.get(0)).ok(); - - if result.is_some() { - // Bump hit count and accessed_at for LRU tracking and statistics. - let now_str = now.to_rfc3339(); - conn.execute( - "UPDATE response_cache - SET accessed_at = ?1, hit_count = hit_count + 1 - WHERE prompt_hash = ?2", - params![now_str, key], - )?; - } - - Ok(result) - } - - /// Store a response in the cache. - /// - /// This also triggers background maintenance: - /// 1. Evicts entries that have exceeded the TTL. - /// 2. Performs LRU (Least Recently Used) eviction if the cache size - /// exceeds `max_entries`. - /// - /// # Arguments - /// - /// * `key` - The unique SHA-256 key for this prompt. - /// * `model` - The model that generated the response. - /// * `response` - The generated text response. - /// * `token_count` - The number of tokens used (for savings tracking). - pub fn put(&self, key: &str, model: &str, response: &str, token_count: u32) -> Result<()> { - let conn = self.conn.lock(); - - let now = Local::now().to_rfc3339(); - - // Insert the new entry or replace an existing one. - conn.execute( - "INSERT OR REPLACE INTO response_cache - (prompt_hash, model, response, token_count, created_at, accessed_at, hit_count) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, 0)", - params![key, model, response, token_count, now, now], - )?; - - // 1. Evict expired entries (TTL-based) - let cutoff = (Local::now() - Duration::minutes(self.ttl_minutes)).to_rfc3339(); - conn.execute( - "DELETE FROM response_cache WHERE created_at <= ?1", - params![cutoff], - )?; - - // 2. LRU eviction (Size-based) - // If we have more than max_entries, delete the oldest entries based on accessed_at. - #[allow(clippy::cast_possible_wrap)] - let max = self.max_entries as i64; - conn.execute( - "DELETE FROM response_cache WHERE prompt_hash IN ( - SELECT prompt_hash FROM response_cache - ORDER BY accessed_at ASC - LIMIT MAX(0, (SELECT COUNT(*) FROM response_cache) - ?1) - )", - params![max], - )?; - - Ok(()) - } - - /// Return cache statistics. - /// - /// Returns a tuple of `(total_entries, total_hits, total_tokens_saved)`. - pub fn stats(&self) -> Result<(usize, u64, u64)> { - let conn = self.conn.lock(); - - // Count total active entries in the cache. - let count: i64 = - conn.query_row("SELECT COUNT(*) FROM response_cache", [], |row| row.get(0))?; - - // Sum up total cache hits. - let hits: i64 = conn.query_row( - "SELECT COALESCE(SUM(hit_count), 0) FROM response_cache", - [], - |row| row.get(0), - )?; - - // Calculate total tokens saved (token_count * hit_count for each entry). - let tokens_saved: i64 = conn.query_row( - "SELECT COALESCE(SUM(token_count * hit_count), 0) FROM response_cache", - [], - |row| row.get(0), - )?; - - #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] - Ok((count as usize, hits as u64, tokens_saved as u64)) - } - - /// Wipe the entire cache. - /// - /// Returns the number of entries deleted. - pub fn clear(&self) -> Result { - let conn = self.conn.lock(); - - let affected = conn.execute("DELETE FROM response_cache", [])?; - Ok(affected) - } -} - -#[cfg(test)] -mod tests { - // ... (tests remain unchanged) diff --git a/src/openhuman/memory/rpc_models.rs b/src/openhuman/memory/rpc_models.rs index d66f355e9..897ace38b 100644 --- a/src/openhuman/memory/rpc_models.rs +++ b/src/openhuman/memory/rpc_models.rs @@ -565,9 +565,11 @@ pub struct WriteMemoryFileResponse { pub bytes_written: usize, } -/// Default directory for memory operations. +/// Default directory for memory operations. Empty string means the memory +/// root itself (`/memory`); the file-based memory RPCs resolve all +/// relative paths under that directory. fn default_memory_relative_dir() -> String { - "memory".to_string() + String::new() } #[cfg(test)] diff --git a/src/openhuman/memory/rpc_models_tests.rs b/src/openhuman/memory/rpc_models_tests.rs index 8c0130fa6..ef7b38914 100644 --- a/src/openhuman/memory/rpc_models_tests.rs +++ b/src/openhuman/memory/rpc_models_tests.rs @@ -1,3 +1,6 @@ +//! Unit tests for the memory RPC request/response models, covering +//! deserialization compatibility and limit-resolution helpers. + use super::*; use serde_json::json; @@ -231,5 +234,6 @@ fn api_envelope_round_trip_preserves_data_and_meta() { #[test] fn default_memory_relative_dir_is_memory() { - assert_eq!(default_memory_relative_dir(), "memory"); + // Empty string == the memory root itself (`/memory`). + assert_eq!(default_memory_relative_dir(), ""); } diff --git a/src/openhuman/memory/schemas.rs b/src/openhuman/memory/schemas.rs deleted file mode 100644 index 902b76afa..000000000 --- a/src/openhuman/memory/schemas.rs +++ /dev/null @@ -1,1292 +0,0 @@ -//! RPC schemas and controller registration for the memory system. -//! -//! This module defines the metadata (schemas) for all memory-related RPC functions -//! and registers their corresponding handlers. It serves as the bridge between -//! the RPC system and the underlying memory operations. - -use serde::de::DeserializeOwned; -use serde_json::{Map, Value}; - -use crate::core::all::{ControllerFuture, RegisteredController}; -use crate::core::{ControllerSchema, FieldSchema, TypeSchema}; -use crate::openhuman::memory::rpc::{ - self, ClearNamespaceParams, DeleteDocParams, GraphQueryParams, GraphUpsertParams, - IngestDocParams, KvGetDeleteParams, KvSetParams, LearnAllParams, NamespaceOnlyParams, - PutDocParams, QueryNamespaceParams, RecallNamespaceParams, SyncChannelParams, -}; -use crate::openhuman::memory::{ - DeleteDocumentRequest, EmptyRequest, ListDocumentsRequest, ListMemoryFilesRequest, - MemoryInitRequest, QueryNamespaceRequest, ReadMemoryFileRequest, RecallContextRequest, - RecallMemoriesRequest, WriteMemoryFileRequest, -}; -use crate::rpc::RpcOutcome; - -// --------------------------------------------------------------------------- -// Public entry points -// --------------------------------------------------------------------------- - -/// Returns all controller schemas for the memory system. -pub fn all_controller_schemas() -> Vec { - vec![ - schemas("init"), - schemas("list_documents"), - schemas("list_namespaces"), - schemas("delete_document"), - schemas("query_namespace"), - schemas("recall_context"), - schemas("recall_memories"), - schemas("list_files"), - schemas("read_file"), - schemas("write_file"), - schemas("namespace_list"), - schemas("doc_put"), - schemas("doc_ingest"), - schemas("doc_list"), - schemas("doc_delete"), - schemas("context_query"), - schemas("context_recall"), - schemas("kv_set"), - schemas("kv_get"), - schemas("kv_delete"), - schemas("kv_list_namespace"), - schemas("graph_upsert"), - schemas("graph_query"), - schemas("clear_namespace"), - schemas("sync_channel"), - schemas("sync_all"), - schemas("learn_all"), - schemas("ingestion_status"), - ] -} - -/// Returns all registered controllers for the memory system, mapping schemas to handlers. -pub fn all_registered_controllers() -> Vec { - vec![ - RegisteredController { - schema: schemas("init"), - handler: handle_init, - }, - RegisteredController { - schema: schemas("list_documents"), - handler: handle_list_documents, - }, - RegisteredController { - schema: schemas("list_namespaces"), - handler: handle_list_namespaces, - }, - RegisteredController { - schema: schemas("delete_document"), - handler: handle_delete_document, - }, - RegisteredController { - schema: schemas("query_namespace"), - handler: handle_query_namespace, - }, - RegisteredController { - schema: schemas("recall_context"), - handler: handle_recall_context, - }, - RegisteredController { - schema: schemas("recall_memories"), - handler: handle_recall_memories, - }, - RegisteredController { - schema: schemas("list_files"), - handler: handle_list_files, - }, - RegisteredController { - schema: schemas("read_file"), - handler: handle_read_file, - }, - RegisteredController { - schema: schemas("write_file"), - handler: handle_write_file, - }, - RegisteredController { - schema: schemas("namespace_list"), - handler: handle_namespace_list, - }, - RegisteredController { - schema: schemas("doc_put"), - handler: handle_doc_put, - }, - RegisteredController { - schema: schemas("doc_ingest"), - handler: handle_doc_ingest, - }, - RegisteredController { - schema: schemas("doc_list"), - handler: handle_doc_list, - }, - RegisteredController { - schema: schemas("doc_delete"), - handler: handle_doc_delete, - }, - RegisteredController { - schema: schemas("context_query"), - handler: handle_context_query, - }, - RegisteredController { - schema: schemas("context_recall"), - handler: handle_context_recall, - }, - RegisteredController { - schema: schemas("kv_set"), - handler: handle_kv_set, - }, - RegisteredController { - schema: schemas("kv_get"), - handler: handle_kv_get, - }, - RegisteredController { - schema: schemas("kv_delete"), - handler: handle_kv_delete, - }, - RegisteredController { - schema: schemas("kv_list_namespace"), - handler: handle_kv_list_namespace, - }, - RegisteredController { - schema: schemas("graph_upsert"), - handler: handle_graph_upsert, - }, - RegisteredController { - schema: schemas("graph_query"), - handler: handle_graph_query, - }, - RegisteredController { - schema: schemas("clear_namespace"), - handler: handle_clear_namespace, - }, - RegisteredController { - schema: schemas("sync_channel"), - handler: handle_sync_channel, - }, - RegisteredController { - schema: schemas("sync_all"), - handler: handle_sync_all, - }, - RegisteredController { - schema: schemas("learn_all"), - handler: handle_learn_all, - }, - RegisteredController { - schema: schemas("ingestion_status"), - handler: handle_ingestion_status, - }, - ] -} - -// --------------------------------------------------------------------------- -// Schema definitions -// --------------------------------------------------------------------------- - -/// Defines the schema for a specific memory controller function. -pub fn schemas(function: &str) -> ControllerSchema { - match function { - // ----- legacy envelope-style methods ----- - "init" => ControllerSchema { - namespace: "memory", - function: "init", - description: "Initialise the local-only (SQLite) memory subsystem for the current workspace. The jwt_token parameter is accepted for backward compatibility but ignored — memory is entirely local.", - inputs: vec![FieldSchema { - name: "jwt_token", - ty: TypeSchema::Option(Box::new(TypeSchema::String)), - comment: "Accepted for backward compatibility but ignored — memory is local-only. Remote sync is a future consideration.", - required: false, - }], - outputs: vec![FieldSchema { - name: "result", - ty: TypeSchema::Json, - comment: "Envelope with initialisation status, workspace and memory paths.", - required: true, - }], - }, - "list_documents" => ControllerSchema { - namespace: "memory", - function: "list_documents", - description: "List documents stored in memory, optionally filtered by namespace.", - inputs: vec![FieldSchema { - name: "namespace", - ty: TypeSchema::Option(Box::new(TypeSchema::String)), - comment: "Namespace to filter documents by.", - required: false, - }], - outputs: vec![FieldSchema { - name: "result", - ty: TypeSchema::Json, - comment: "Envelope with documents array and count.", - required: true, - }], - }, - "list_namespaces" => ControllerSchema { - namespace: "memory", - function: "list_namespaces", - description: "List all namespaces that contain memory documents.", - inputs: vec![], - outputs: vec![FieldSchema { - name: "result", - ty: TypeSchema::Json, - comment: "Envelope with namespaces array and count.", - required: true, - }], - }, - "delete_document" => ControllerSchema { - namespace: "memory", - function: "delete_document", - description: "Delete a specific document from a namespace.", - inputs: vec![ - FieldSchema { - name: "namespace", - ty: TypeSchema::String, - comment: "Namespace containing the document.", - required: true, - }, - FieldSchema { - name: "document_id", - ty: TypeSchema::String, - comment: "Identifier of the document to delete.", - required: true, - }, - ], - outputs: vec![FieldSchema { - name: "result", - ty: TypeSchema::Json, - comment: "Envelope with deletion status.", - required: true, - }], - }, - "query_namespace" => ControllerSchema { - namespace: "memory", - function: "query_namespace", - description: "Semantic query against a namespace with optional reference data.", - inputs: vec![ - FieldSchema { - name: "namespace", - ty: TypeSchema::String, - comment: "Namespace to query.", - required: true, - }, - FieldSchema { - name: "query", - ty: TypeSchema::String, - comment: "Natural-language query string.", - required: true, - }, - FieldSchema { - name: "include_references", - ty: TypeSchema::Option(Box::new(TypeSchema::Bool)), - comment: "Whether to include entity/relation context in the response.", - required: false, - }, - FieldSchema { - name: "document_ids", - ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new( - TypeSchema::String, - )))), - comment: "Restrict results to these document IDs.", - required: false, - }, - FieldSchema { - name: "limit", - ty: TypeSchema::Option(Box::new(TypeSchema::U64)), - comment: "Maximum number of results to return.", - required: false, - }, - FieldSchema { - name: "max_chunks", - ty: TypeSchema::Option(Box::new(TypeSchema::U64)), - comment: "Maximum number of chunks to return (alias for limit).", - required: false, - }, - ], - outputs: vec![FieldSchema { - name: "result", - ty: TypeSchema::Json, - comment: "Envelope with retrieval context and LLM context message.", - required: true, - }], - }, - "recall_context" => ControllerSchema { - namespace: "memory", - function: "recall_context", - description: "Recall contextual data from a namespace without a specific query.", - inputs: vec![ - FieldSchema { - name: "namespace", - ty: TypeSchema::String, - comment: "Namespace to recall from.", - required: true, - }, - FieldSchema { - name: "include_references", - ty: TypeSchema::Option(Box::new(TypeSchema::Bool)), - comment: "Whether to include entity/relation context.", - required: false, - }, - FieldSchema { - name: "limit", - ty: TypeSchema::Option(Box::new(TypeSchema::U64)), - comment: "Maximum number of results.", - required: false, - }, - FieldSchema { - name: "max_chunks", - ty: TypeSchema::Option(Box::new(TypeSchema::U64)), - comment: "Maximum number of chunks (alias for limit).", - required: false, - }, - ], - outputs: vec![FieldSchema { - name: "result", - ty: TypeSchema::Json, - comment: "Envelope with retrieval context and LLM context message.", - required: true, - }], - }, - "recall_memories" => ControllerSchema { - namespace: "memory", - function: "recall_memories", - description: "Recall memory items from a namespace with optional retention filtering.", - inputs: vec![ - FieldSchema { - name: "namespace", - ty: TypeSchema::String, - comment: "Namespace to recall memories from.", - required: true, - }, - FieldSchema { - name: "min_retention", - ty: TypeSchema::Option(Box::new(TypeSchema::F64)), - comment: "Minimum retention score (forward-compat, currently ignored).", - required: false, - }, - FieldSchema { - name: "as_of", - ty: TypeSchema::Option(Box::new(TypeSchema::F64)), - comment: "Temporal recall timestamp (forward-compat, currently ignored).", - required: false, - }, - FieldSchema { - name: "limit", - ty: TypeSchema::Option(Box::new(TypeSchema::U64)), - comment: "Maximum number of results.", - required: false, - }, - FieldSchema { - name: "max_chunks", - ty: TypeSchema::Option(Box::new(TypeSchema::U64)), - comment: "Maximum number of chunks (alias for limit).", - required: false, - }, - FieldSchema { - name: "top_k", - ty: TypeSchema::Option(Box::new(TypeSchema::U64)), - comment: "Top-k override (alias for limit).", - required: false, - }, - ], - outputs: vec![FieldSchema { - name: "result", - ty: TypeSchema::Json, - comment: "Envelope with recalled memory items.", - required: true, - }], - }, - - // ----- file-based memory methods ----- - "list_files" => ControllerSchema { - namespace: "memory", - function: "list_files", - description: "List files in a memory directory.", - inputs: vec![FieldSchema { - name: "relative_dir", - ty: TypeSchema::String, - comment: "Relative directory path under the workspace (default: \"memory\").", - required: false, - }], - outputs: vec![FieldSchema { - name: "result", - ty: TypeSchema::Json, - comment: "Envelope with file listing.", - required: true, - }], - }, - "read_file" => ControllerSchema { - namespace: "memory", - function: "read_file", - description: "Read the contents of a memory file.", - inputs: vec![FieldSchema { - name: "relative_path", - ty: TypeSchema::String, - comment: "Relative path to the file under the workspace memory directory.", - required: true, - }], - outputs: vec![FieldSchema { - name: "result", - ty: TypeSchema::Json, - comment: "Envelope with file content.", - required: true, - }], - }, - "write_file" => ControllerSchema { - namespace: "memory", - function: "write_file", - description: "Write content to a memory file.", - inputs: vec![ - FieldSchema { - name: "relative_path", - ty: TypeSchema::String, - comment: "Relative path to the file under the workspace memory directory.", - required: true, - }, - FieldSchema { - name: "content", - ty: TypeSchema::String, - comment: "Content to write to the file.", - required: true, - }, - ], - outputs: vec![FieldSchema { - name: "result", - ty: TypeSchema::Json, - comment: "Envelope with write confirmation and bytes written.", - required: true, - }], - }, - // ----- unified memory API methods ----- - "namespace_list" => ControllerSchema { - namespace: "memory", - function: "namespace_list", - description: "List all namespaces in the unified memory store.", - inputs: vec![], - outputs: vec![FieldSchema { - name: "namespaces", - ty: TypeSchema::Array(Box::new(TypeSchema::String)), - comment: "Namespace names.", - required: true, - }], - }, - "doc_put" => ControllerSchema { - namespace: "memory", - function: "doc_put", - description: "Upsert a document into a namespace.", - inputs: vec![ - FieldSchema { - name: "namespace", - ty: TypeSchema::String, - comment: "Target namespace.", - required: true, - }, - FieldSchema { - name: "key", - ty: TypeSchema::String, - comment: "Document key for upsert deduplication.", - required: true, - }, - FieldSchema { - name: "title", - ty: TypeSchema::String, - comment: "Human-readable title.", - required: true, - }, - FieldSchema { - name: "content", - ty: TypeSchema::String, - comment: "Document body content.", - required: true, - }, - FieldSchema { - name: "source_type", - ty: TypeSchema::String, - comment: "Source type label (default: \"doc\").", - required: false, - }, - FieldSchema { - name: "priority", - ty: TypeSchema::String, - comment: "Priority level (default: \"medium\").", - required: false, - }, - FieldSchema { - name: "tags", - ty: TypeSchema::Array(Box::new(TypeSchema::String)), - comment: "Tags for categorisation (default: []).", - required: false, - }, - FieldSchema { - name: "metadata", - ty: TypeSchema::Json, - comment: "Arbitrary metadata (default: {}).", - required: false, - }, - FieldSchema { - name: "category", - ty: TypeSchema::String, - comment: "Memory category (default: \"core\").", - required: false, - }, - FieldSchema { - name: "session_id", - ty: TypeSchema::Option(Box::new(TypeSchema::String)), - comment: "Optional session ID for provenance tracking.", - required: false, - }, - FieldSchema { - name: "document_id", - ty: TypeSchema::Option(Box::new(TypeSchema::String)), - comment: "Optional explicit document ID; generated if omitted.", - required: false, - }, - ], - outputs: vec![FieldSchema { - name: "document_id", - ty: TypeSchema::String, - comment: "ID of the upserted document.", - required: true, - }], - }, - "doc_ingest" => ControllerSchema { - namespace: "memory", - function: "doc_ingest", - description: "Ingest a document with entity/relation extraction and chunk embedding.", - inputs: vec![ - FieldSchema { - name: "namespace", - ty: TypeSchema::String, - comment: "Target namespace.", - required: true, - }, - FieldSchema { - name: "key", - ty: TypeSchema::String, - comment: "Document key.", - required: true, - }, - FieldSchema { - name: "title", - ty: TypeSchema::String, - comment: "Human-readable title.", - required: true, - }, - FieldSchema { - name: "content", - ty: TypeSchema::String, - comment: "Document body content.", - required: true, - }, - FieldSchema { - name: "source_type", - ty: TypeSchema::String, - comment: "Source type label (default: \"doc\").", - required: false, - }, - FieldSchema { - name: "priority", - ty: TypeSchema::String, - comment: "Priority level (default: \"medium\").", - required: false, - }, - FieldSchema { - name: "tags", - ty: TypeSchema::Array(Box::new(TypeSchema::String)), - comment: "Tags for categorisation (default: []).", - required: false, - }, - FieldSchema { - name: "metadata", - ty: TypeSchema::Json, - comment: "Arbitrary metadata (default: {}).", - required: false, - }, - FieldSchema { - name: "category", - ty: TypeSchema::String, - comment: "Memory category (default: \"core\").", - required: false, - }, - FieldSchema { - name: "session_id", - ty: TypeSchema::Option(Box::new(TypeSchema::String)), - comment: "Optional session ID.", - required: false, - }, - FieldSchema { - name: "document_id", - ty: TypeSchema::Option(Box::new(TypeSchema::String)), - comment: "Optional explicit document ID.", - required: false, - }, - FieldSchema { - name: "config", - ty: TypeSchema::Option(Box::new(TypeSchema::Json)), - comment: "Optional ingestion configuration overrides.", - required: false, - }, - ], - outputs: vec![FieldSchema { - name: "result", - ty: TypeSchema::Json, - comment: "Ingestion result with entity, relation and chunk counts.", - required: true, - }], - }, - "doc_list" => ControllerSchema { - namespace: "memory", - function: "doc_list", - description: "List documents in the unified memory store, optionally by namespace.", - inputs: vec![FieldSchema { - name: "namespace", - ty: TypeSchema::Option(Box::new(TypeSchema::String)), - comment: "Optional namespace filter.", - required: false, - }], - outputs: vec![FieldSchema { - name: "result", - ty: TypeSchema::Json, - comment: "Document listing.", - required: true, - }], - }, - "doc_delete" => ControllerSchema { - namespace: "memory", - function: "doc_delete", - description: "Delete a document from the unified memory store.", - inputs: vec![ - FieldSchema { - name: "namespace", - ty: TypeSchema::String, - comment: "Namespace containing the document.", - required: true, - }, - FieldSchema { - name: "document_id", - ty: TypeSchema::String, - comment: "Document identifier to delete.", - required: true, - }, - ], - outputs: vec![FieldSchema { - name: "result", - ty: TypeSchema::Json, - comment: "Deletion result.", - required: true, - }], - }, - "context_query" => ControllerSchema { - namespace: "memory", - function: "context_query", - description: "Query a namespace for contextual information.", - inputs: vec![ - FieldSchema { - name: "namespace", - ty: TypeSchema::String, - comment: "Namespace to query.", - required: true, - }, - FieldSchema { - name: "query", - ty: TypeSchema::String, - comment: "Natural-language query string.", - required: true, - }, - FieldSchema { - name: "limit", - ty: TypeSchema::Option(Box::new(TypeSchema::U64)), - comment: "Maximum number of results.", - required: false, - }, - ], - outputs: vec![FieldSchema { - name: "result", - ty: TypeSchema::String, - comment: "Contextual query result string.", - required: true, - }], - }, - "context_recall" => ControllerSchema { - namespace: "memory", - function: "context_recall", - description: "Recall context from a namespace.", - inputs: vec![ - FieldSchema { - name: "namespace", - ty: TypeSchema::String, - comment: "Namespace to recall from.", - required: true, - }, - FieldSchema { - name: "limit", - ty: TypeSchema::Option(Box::new(TypeSchema::U64)), - comment: "Maximum number of results.", - required: false, - }, - ], - outputs: vec![FieldSchema { - name: "result", - ty: TypeSchema::Json, - comment: "Recalled context (may be null if empty).", - required: true, - }], - }, - - // ----- key-value methods ----- - "kv_set" => ControllerSchema { - namespace: "memory", - function: "kv_set", - description: "Set a key-value pair in the memory store.", - inputs: vec![ - FieldSchema { - name: "namespace", - ty: TypeSchema::Option(Box::new(TypeSchema::String)), - comment: "Optional namespace scope.", - required: false, - }, - FieldSchema { - name: "key", - ty: TypeSchema::String, - comment: "Key to set.", - required: true, - }, - FieldSchema { - name: "value", - ty: TypeSchema::Json, - comment: "JSON value to store.", - required: true, - }, - ], - outputs: vec![FieldSchema { - name: "result", - ty: TypeSchema::Bool, - comment: "True when the value was stored.", - required: true, - }], - }, - "kv_get" => ControllerSchema { - namespace: "memory", - function: "kv_get", - description: "Get a value by key from the memory store.", - inputs: vec![ - FieldSchema { - name: "namespace", - ty: TypeSchema::Option(Box::new(TypeSchema::String)), - comment: "Optional namespace scope.", - required: false, - }, - FieldSchema { - name: "key", - ty: TypeSchema::String, - comment: "Key to retrieve.", - required: true, - }, - ], - outputs: vec![FieldSchema { - name: "result", - ty: TypeSchema::Json, - comment: "Stored value or null if not found.", - required: true, - }], - }, - "kv_delete" => ControllerSchema { - namespace: "memory", - function: "kv_delete", - description: "Delete a key-value pair from the memory store.", - inputs: vec![ - FieldSchema { - name: "namespace", - ty: TypeSchema::Option(Box::new(TypeSchema::String)), - comment: "Optional namespace scope.", - required: false, - }, - FieldSchema { - name: "key", - ty: TypeSchema::String, - comment: "Key to delete.", - required: true, - }, - ], - outputs: vec![FieldSchema { - name: "result", - ty: TypeSchema::Bool, - comment: "True when the key was deleted.", - required: true, - }], - }, - "kv_list_namespace" => ControllerSchema { - namespace: "memory", - function: "kv_list_namespace", - description: "List all key-value entries in a namespace.", - inputs: vec![FieldSchema { - name: "namespace", - ty: TypeSchema::String, - comment: "Namespace to list.", - required: true, - }], - outputs: vec![FieldSchema { - name: "result", - ty: TypeSchema::Json, - comment: "Array of key-value entries.", - required: true, - }], - }, - - // ----- graph methods ----- - "graph_upsert" => ControllerSchema { - namespace: "memory", - function: "graph_upsert", - description: "Upsert a relation triple in the knowledge graph.", - inputs: vec![ - FieldSchema { - name: "namespace", - ty: TypeSchema::Option(Box::new(TypeSchema::String)), - comment: "Optional namespace scope.", - required: false, - }, - FieldSchema { - name: "subject", - ty: TypeSchema::String, - comment: "Subject entity of the relation.", - required: true, - }, - FieldSchema { - name: "predicate", - ty: TypeSchema::String, - comment: "Relation predicate.", - required: true, - }, - FieldSchema { - name: "object", - ty: TypeSchema::String, - comment: "Object entity of the relation.", - required: true, - }, - FieldSchema { - name: "attrs", - ty: TypeSchema::Json, - comment: "Extra attributes on the relation (default: {}).", - required: false, - }, - ], - outputs: vec![FieldSchema { - name: "result", - ty: TypeSchema::Bool, - comment: "True when the relation was upserted.", - required: true, - }], - }, - "graph_query" => ControllerSchema { - namespace: "memory", - function: "graph_query", - description: "Query relations from the knowledge graph.", - inputs: vec![ - FieldSchema { - name: "namespace", - ty: TypeSchema::Option(Box::new(TypeSchema::String)), - comment: "Optional namespace scope.", - required: false, - }, - FieldSchema { - name: "subject", - ty: TypeSchema::Option(Box::new(TypeSchema::String)), - comment: "Filter by subject entity.", - required: false, - }, - FieldSchema { - name: "predicate", - ty: TypeSchema::Option(Box::new(TypeSchema::String)), - comment: "Filter by relation predicate.", - required: false, - }, - ], - outputs: vec![FieldSchema { - name: "result", - ty: TypeSchema::Json, - comment: "Array of matching relation records.", - required: true, - }], - }, - - // ----- bulk operations ----- - "clear_namespace" => ControllerSchema { - namespace: "memory", - function: "clear_namespace", - description: - "Delete all documents, vector chunks, KV entries, and graph relations for a namespace.", - inputs: vec![FieldSchema { - name: "namespace", - ty: TypeSchema::String, - comment: "Namespace to clear completely.", - required: true, - }], - outputs: vec![ - FieldSchema { - name: "cleared", - ty: TypeSchema::Bool, - comment: "True when the namespace was cleared.", - required: true, - }, - FieldSchema { - name: "namespace", - ty: TypeSchema::String, - comment: "The namespace that was cleared.", - required: true, - }, - ], - }, - - // ----- sync / learn ----- - "sync_channel" => ControllerSchema { - namespace: "memory", - function: "sync_channel", - description: "Request a memory sync for a specific channel. Publishes MemorySyncRequested on the event bus. No ingestion consumers exist yet; this is a hook for future pull-based subscribers.", - inputs: vec![FieldSchema { - name: "channel_id", - ty: TypeSchema::String, - comment: "ID of the channel to sync.", - required: true, - }], - outputs: vec![ - FieldSchema { - name: "requested", - ty: TypeSchema::Bool, - comment: "Always true when the event was published.", - required: true, - }, - FieldSchema { - name: "channel_id", - ty: TypeSchema::String, - comment: "Echo of the channel_id that was requested.", - required: true, - }, - ], - }, - "sync_all" => ControllerSchema { - namespace: "memory", - function: "sync_all", - description: "Request a memory sync for all channels. Publishes MemorySyncRequested { channel_id: None } on the event bus.", - inputs: vec![], - outputs: vec![FieldSchema { - name: "requested", - ty: TypeSchema::Bool, - comment: "Always true when the event was published.", - required: true, - }], - }, - "learn_all" => ControllerSchema { - namespace: "memory", - function: "learn_all", - description: "Run the tree summarizer over all memory namespaces (or a constrained subset). Processes namespaces sequentially; a failing namespace is recorded but does not abort the rest.", - inputs: vec![FieldSchema { - name: "namespaces", - ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new(TypeSchema::String)))), - comment: "Optional list of namespaces to constrain. Defaults to all namespaces.", - required: false, - }], - outputs: vec![ - FieldSchema { - name: "namespaces_processed", - ty: TypeSchema::U64, - comment: "Total number of namespaces processed.", - required: true, - }, - FieldSchema { - name: "results", - ty: TypeSchema::Json, - comment: "Per-namespace outcomes: [{ namespace, status: 'ok'|'skipped'|'error', error? }].", - required: true, - }, - ], - }, - - "ingestion_status" => ControllerSchema { - namespace: "memory", - function: "ingestion_status", - description: "Returns the current memory-ingestion status (whether a job is running, the in-flight document, queue depth, and most recent completion). Safe to poll.", - inputs: vec![], - outputs: vec![ - FieldSchema { - name: "running", - ty: TypeSchema::Bool, - comment: "True while an ingestion job is running on the local extraction LLM.", - required: true, - }, - FieldSchema { - name: "current_document_id", - ty: TypeSchema::Option(Box::new(TypeSchema::String)), - comment: "Document id of the in-flight job.", - required: false, - }, - FieldSchema { - name: "current_title", - ty: TypeSchema::Option(Box::new(TypeSchema::String)), - comment: "Title of the in-flight document.", - required: false, - }, - FieldSchema { - name: "current_namespace", - ty: TypeSchema::Option(Box::new(TypeSchema::String)), - comment: "Namespace of the in-flight document.", - required: false, - }, - FieldSchema { - name: "queue_depth", - ty: TypeSchema::U64, - comment: "Number of jobs waiting behind the current one.", - required: true, - }, - FieldSchema { - name: "last_completed_at", - ty: TypeSchema::Option(Box::new(TypeSchema::I64)), - comment: "Unix-ms timestamp of the most recent completion.", - required: false, - }, - FieldSchema { - name: "last_document_id", - ty: TypeSchema::Option(Box::new(TypeSchema::String)), - comment: "Document id of the most recent completed job.", - required: false, - }, - FieldSchema { - name: "last_success", - ty: TypeSchema::Option(Box::new(TypeSchema::Bool)), - comment: "Whether the most recent job succeeded.", - required: false, - }, - ], - }, - - // ----- fallback ----- - _other => ControllerSchema { - namespace: "memory", - function: "unknown", - description: "Unknown memory controller function.", - inputs: vec![FieldSchema { - name: "function", - ty: TypeSchema::String, - comment: "Unknown function requested for schema lookup.", - required: true, - }], - outputs: vec![FieldSchema { - name: "error", - ty: TypeSchema::String, - comment: "Lookup error details.", - required: true, - }], - }, - } -} - -// --------------------------------------------------------------------------- -// Handlers -// --------------------------------------------------------------------------- - -fn handle_init(params: Map) -> ControllerFuture { - Box::pin(async move { - let payload = parse_params::(params)?; - to_json(rpc::memory_init(payload).await?) - }) -} - -fn handle_list_documents(params: Map) -> ControllerFuture { - Box::pin(async move { - let payload = parse_params::(params)?; - to_json(rpc::memory_list_documents(payload).await?) - }) -} - -fn handle_list_namespaces(_params: Map) -> ControllerFuture { - Box::pin(async move { to_json(rpc::memory_list_namespaces(EmptyRequest {}).await?) }) -} - -fn handle_delete_document(params: Map) -> ControllerFuture { - Box::pin(async move { - let payload = parse_params::(params)?; - to_json(rpc::memory_delete_document(payload).await?) - }) -} - -fn handle_query_namespace(params: Map) -> ControllerFuture { - Box::pin(async move { - let payload = parse_params::(params)?; - to_json(rpc::memory_query_namespace(payload).await?) - }) -} - -fn handle_recall_context(params: Map) -> ControllerFuture { - Box::pin(async move { - let payload = parse_params::(params)?; - to_json(rpc::memory_recall_context(payload).await?) - }) -} - -fn handle_recall_memories(params: Map) -> ControllerFuture { - Box::pin(async move { - let payload = parse_params::(params)?; - to_json(rpc::memory_recall_memories(payload).await?) - }) -} - -fn handle_list_files(params: Map) -> ControllerFuture { - Box::pin(async move { - let relative_dir = params - .get("relative_dir") - .and_then(|v| v.as_str()) - .unwrap_or("memory") - .to_string(); - let payload = ListMemoryFilesRequest { relative_dir }; - to_json(rpc::ai_list_memory_files(payload).await?) - }) -} - -fn handle_read_file(params: Map) -> ControllerFuture { - Box::pin(async move { - let payload = parse_params::(params)?; - to_json(rpc::ai_read_memory_file(payload).await?) - }) -} - -fn handle_write_file(params: Map) -> ControllerFuture { - Box::pin(async move { - let payload = parse_params::(params)?; - to_json(rpc::ai_write_memory_file(payload).await?) - }) -} - -fn handle_namespace_list(_params: Map) -> ControllerFuture { - Box::pin(async move { to_json(rpc::namespace_list().await?) }) -} - -fn handle_doc_put(params: Map) -> ControllerFuture { - Box::pin(async move { - let payload = parse_params::(params)?; - to_json(rpc::doc_put(payload).await?) - }) -} - -fn handle_doc_ingest(params: Map) -> ControllerFuture { - Box::pin(async move { - let payload = parse_params::(params)?; - to_json(rpc::doc_ingest(payload).await?) - }) -} - -fn handle_doc_list(params: Map) -> ControllerFuture { - Box::pin(async move { - let namespace = - params - .get("namespace") - .and_then(|v| v.as_str()) - .map(|ns| NamespaceOnlyParams { - namespace: ns.to_string(), - }); - to_json(rpc::doc_list(namespace).await?) - }) -} - -fn handle_doc_delete(params: Map) -> ControllerFuture { - Box::pin(async move { - let payload = parse_params::(params)?; - to_json(rpc::doc_delete(payload).await?) - }) -} - -fn handle_context_query(params: Map) -> ControllerFuture { - Box::pin(async move { - let payload = parse_params::(params)?; - to_json(rpc::context_query(payload).await?) - }) -} - -fn handle_context_recall(params: Map) -> ControllerFuture { - Box::pin(async move { - let payload = parse_params::(params)?; - to_json(rpc::context_recall(payload).await?) - }) -} - -fn handle_kv_set(params: Map) -> ControllerFuture { - Box::pin(async move { - let payload = parse_params::(params)?; - to_json(rpc::kv_set(payload).await?) - }) -} - -fn handle_kv_get(params: Map) -> ControllerFuture { - Box::pin(async move { - let payload = parse_params::(params)?; - to_json(rpc::kv_get(payload).await?) - }) -} - -fn handle_kv_delete(params: Map) -> ControllerFuture { - Box::pin(async move { - let payload = parse_params::(params)?; - to_json(rpc::kv_delete(payload).await?) - }) -} - -fn handle_kv_list_namespace(params: Map) -> ControllerFuture { - Box::pin(async move { - let payload = parse_params::(params)?; - to_json(rpc::kv_list_namespace(payload).await?) - }) -} - -fn handle_graph_upsert(params: Map) -> ControllerFuture { - Box::pin(async move { - let payload = parse_params::(params)?; - to_json(rpc::graph_upsert(payload).await?) - }) -} - -fn handle_graph_query(params: Map) -> ControllerFuture { - Box::pin(async move { - let payload = parse_params::(params)?; - to_json(rpc::graph_query(payload).await?) - }) -} - -fn handle_clear_namespace(params: Map) -> ControllerFuture { - Box::pin(async move { - let payload = parse_params::(params)?; - to_json(rpc::clear_namespace(payload).await?) - }) -} - -fn handle_sync_channel(params: Map) -> ControllerFuture { - Box::pin(async move { - let payload = parse_params::(params)?; - to_json(rpc::memory_sync_channel(payload).await?) - }) -} - -fn handle_sync_all(_params: Map) -> ControllerFuture { - Box::pin(async move { to_json(rpc::memory_sync_all().await?) }) -} - -fn handle_learn_all(params: Map) -> ControllerFuture { - Box::pin(async move { - let payload = parse_params::(params)?; - to_json(rpc::memory_learn_all(payload).await?) - }) -} - -fn handle_ingestion_status(_params: Map) -> ControllerFuture { - Box::pin(async move { to_json(rpc::memory_ingestion_status().await?) }) -} - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -fn parse_params(params: Map) -> Result { - serde_json::from_value(Value::Object(params)).map_err(|e| format!("invalid params: {e}")) -} - -fn to_json(outcome: RpcOutcome) -> Result { - outcome.into_cli_compatible_json() -} - -#[cfg(test)] -#[path = "schemas_tests.rs"] -mod tests; diff --git a/src/openhuman/memory/schemas/documents.rs b/src/openhuman/memory/schemas/documents.rs new file mode 100644 index 000000000..6399bb22c --- /dev/null +++ b/src/openhuman/memory/schemas/documents.rs @@ -0,0 +1,537 @@ +//! Schemas and handlers for document, namespace, recall, and clear-namespace +//! RPC methods. + +use serde_json::{Map, Value}; + +use crate::core::all::{ControllerFuture, RegisteredController}; +use crate::core::{ControllerSchema, FieldSchema, TypeSchema}; +use crate::openhuman::memory::rpc::{ + self, ClearNamespaceParams, DeleteDocParams, IngestDocParams, NamespaceOnlyParams, + PutDocParams, QueryNamespaceParams, RecallNamespaceParams, +}; +use crate::openhuman::memory::{ + DeleteDocumentRequest, EmptyRequest, ListDocumentsRequest, MemoryInitRequest, + QueryNamespaceRequest, RecallContextRequest, RecallMemoriesRequest, +}; + +use super::{parse_params, to_json}; + +pub(super) const FUNCTIONS: &[&str] = &[ + "init", + "list_documents", + "list_namespaces", + "delete_document", + "query_namespace", + "recall_context", + "recall_memories", + "namespace_list", + "doc_put", + "doc_ingest", + "doc_list", + "doc_delete", + "context_query", + "context_recall", + "clear_namespace", +]; + +pub(super) fn controllers() -> Vec { + vec![ + RegisteredController { + schema: schema("init").unwrap(), + handler: handle_init, + }, + RegisteredController { + schema: schema("list_documents").unwrap(), + handler: handle_list_documents, + }, + RegisteredController { + schema: schema("list_namespaces").unwrap(), + handler: handle_list_namespaces, + }, + RegisteredController { + schema: schema("delete_document").unwrap(), + handler: handle_delete_document, + }, + RegisteredController { + schema: schema("query_namespace").unwrap(), + handler: handle_query_namespace, + }, + RegisteredController { + schema: schema("recall_context").unwrap(), + handler: handle_recall_context, + }, + RegisteredController { + schema: schema("recall_memories").unwrap(), + handler: handle_recall_memories, + }, + RegisteredController { + schema: schema("namespace_list").unwrap(), + handler: handle_namespace_list, + }, + RegisteredController { + schema: schema("doc_put").unwrap(), + handler: handle_doc_put, + }, + RegisteredController { + schema: schema("doc_ingest").unwrap(), + handler: handle_doc_ingest, + }, + RegisteredController { + schema: schema("doc_list").unwrap(), + handler: handle_doc_list, + }, + RegisteredController { + schema: schema("doc_delete").unwrap(), + handler: handle_doc_delete, + }, + RegisteredController { + schema: schema("context_query").unwrap(), + handler: handle_context_query, + }, + RegisteredController { + schema: schema("context_recall").unwrap(), + handler: handle_context_recall, + }, + RegisteredController { + schema: schema("clear_namespace").unwrap(), + handler: handle_clear_namespace, + }, + ] +} + +pub(super) fn schema(function: &str) -> Option { + Some(match function { + "init" => ControllerSchema { + namespace: "memory", + function: "init", + description: "Initialise the local-only (SQLite) memory subsystem for the current workspace. The jwt_token parameter is accepted for backward compatibility but ignored — memory is entirely local.", + inputs: vec![FieldSchema { + name: "jwt_token", + ty: TypeSchema::Option(Box::new(TypeSchema::String)), + comment: "Accepted for backward compatibility but ignored — memory is local-only. Remote sync is a future consideration.", + required: false, + }], + outputs: vec![FieldSchema { + name: "result", + ty: TypeSchema::Json, + comment: "Envelope with initialisation status, workspace and memory paths.", + required: true, + }], + }, + "list_documents" => ControllerSchema { + namespace: "memory", + function: "list_documents", + description: "List documents stored in memory, optionally filtered by namespace.", + inputs: vec![FieldSchema { + name: "namespace", + ty: TypeSchema::Option(Box::new(TypeSchema::String)), + comment: "Namespace to filter documents by.", + required: false, + }], + outputs: vec![FieldSchema { + name: "result", + ty: TypeSchema::Json, + comment: "Envelope with documents array and count.", + required: true, + }], + }, + "list_namespaces" => ControllerSchema { + namespace: "memory", + function: "list_namespaces", + description: "List all namespaces that contain memory documents.", + inputs: vec![], + outputs: vec![FieldSchema { + name: "result", + ty: TypeSchema::Json, + comment: "Envelope with namespaces array and count.", + required: true, + }], + }, + "delete_document" => ControllerSchema { + namespace: "memory", + function: "delete_document", + description: "Delete a specific document from a namespace.", + inputs: vec![ + FieldSchema { + name: "namespace", + ty: TypeSchema::String, + comment: "Namespace containing the document.", + required: true, + }, + FieldSchema { + name: "document_id", + ty: TypeSchema::String, + comment: "Identifier of the document to delete.", + required: true, + }, + ], + outputs: vec![FieldSchema { + name: "result", + ty: TypeSchema::Json, + comment: "Envelope with deletion status.", + required: true, + }], + }, + "query_namespace" => ControllerSchema { + namespace: "memory", + function: "query_namespace", + description: "Semantic query against a namespace with optional reference data.", + inputs: vec![ + FieldSchema { + name: "namespace", + ty: TypeSchema::String, + comment: "Namespace to query.", + required: true, + }, + FieldSchema { + name: "query", + ty: TypeSchema::String, + comment: "Natural-language query string.", + required: true, + }, + FieldSchema { + name: "include_references", + ty: TypeSchema::Option(Box::new(TypeSchema::Bool)), + comment: "Whether to include entity/relation context in the response.", + required: false, + }, + FieldSchema { + name: "document_ids", + ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new( + TypeSchema::String, + )))), + comment: "Restrict results to these document IDs.", + required: false, + }, + FieldSchema { + name: "limit", + ty: TypeSchema::Option(Box::new(TypeSchema::U64)), + comment: "Maximum number of results to return.", + required: false, + }, + FieldSchema { + name: "max_chunks", + ty: TypeSchema::Option(Box::new(TypeSchema::U64)), + comment: "Maximum number of chunks to return (alias for limit).", + required: false, + }, + ], + outputs: vec![FieldSchema { + name: "result", + ty: TypeSchema::Json, + comment: "Envelope with retrieval context and LLM context message.", + required: true, + }], + }, + "recall_context" => ControllerSchema { + namespace: "memory", + function: "recall_context", + description: "Recall contextual data from a namespace without a specific query.", + inputs: vec![ + FieldSchema { + name: "namespace", + ty: TypeSchema::String, + comment: "Namespace to recall from.", + required: true, + }, + FieldSchema { + name: "include_references", + ty: TypeSchema::Option(Box::new(TypeSchema::Bool)), + comment: "Whether to include entity/relation context.", + required: false, + }, + FieldSchema { + name: "limit", + ty: TypeSchema::Option(Box::new(TypeSchema::U64)), + comment: "Maximum number of results.", + required: false, + }, + FieldSchema { + name: "max_chunks", + ty: TypeSchema::Option(Box::new(TypeSchema::U64)), + comment: "Maximum number of chunks (alias for limit).", + required: false, + }, + ], + outputs: vec![FieldSchema { + name: "result", + ty: TypeSchema::Json, + comment: "Envelope with retrieval context and LLM context message.", + required: true, + }], + }, + "recall_memories" => ControllerSchema { + namespace: "memory", + function: "recall_memories", + description: "Recall memory items from a namespace with optional retention filtering.", + inputs: vec![ + FieldSchema { + name: "namespace", + ty: TypeSchema::String, + comment: "Namespace to recall memories from.", + required: true, + }, + FieldSchema { + name: "min_retention", + ty: TypeSchema::Option(Box::new(TypeSchema::F64)), + comment: "Minimum retention score (forward-compat, currently ignored).", + required: false, + }, + FieldSchema { + name: "as_of", + ty: TypeSchema::Option(Box::new(TypeSchema::F64)), + comment: "Temporal recall timestamp (forward-compat, currently ignored).", + required: false, + }, + FieldSchema { + name: "limit", + ty: TypeSchema::Option(Box::new(TypeSchema::U64)), + comment: "Maximum number of results.", + required: false, + }, + FieldSchema { + name: "max_chunks", + ty: TypeSchema::Option(Box::new(TypeSchema::U64)), + comment: "Maximum number of chunks (alias for limit).", + required: false, + }, + FieldSchema { + name: "top_k", + ty: TypeSchema::Option(Box::new(TypeSchema::U64)), + comment: "Top-k override (alias for limit).", + required: false, + }, + ], + outputs: vec![FieldSchema { + name: "result", + ty: TypeSchema::Json, + comment: "Envelope with recalled memory items.", + required: true, + }], + }, + "namespace_list" => ControllerSchema { + namespace: "memory", + function: "namespace_list", + description: "List all namespaces in the unified memory store.", + inputs: vec![], + outputs: vec![FieldSchema { + name: "namespaces", + ty: TypeSchema::Array(Box::new(TypeSchema::String)), + comment: "Namespace names.", + required: true, + }], + }, + "doc_put" => ControllerSchema { + namespace: "memory", + function: "doc_put", + description: "Upsert a document into a namespace.", + inputs: vec![ + FieldSchema { name: "namespace", ty: TypeSchema::String, comment: "Target namespace.", required: true }, + FieldSchema { name: "key", ty: TypeSchema::String, comment: "Document key for upsert deduplication.", required: true }, + FieldSchema { name: "title", ty: TypeSchema::String, comment: "Human-readable title.", required: true }, + FieldSchema { name: "content", ty: TypeSchema::String, comment: "Document body content.", required: true }, + FieldSchema { name: "source_type", ty: TypeSchema::String, comment: "Source type label (default: \"doc\").", required: false }, + FieldSchema { name: "priority", ty: TypeSchema::String, comment: "Priority level (default: \"medium\").", required: false }, + FieldSchema { name: "tags", ty: TypeSchema::Array(Box::new(TypeSchema::String)), comment: "Tags for categorisation (default: []).", required: false }, + FieldSchema { name: "metadata", ty: TypeSchema::Json, comment: "Arbitrary metadata (default: {}).", required: false }, + FieldSchema { name: "category", ty: TypeSchema::String, comment: "Memory category (default: \"core\").", required: false }, + FieldSchema { name: "session_id", ty: TypeSchema::Option(Box::new(TypeSchema::String)), comment: "Optional session ID for provenance tracking.", required: false }, + FieldSchema { name: "document_id", ty: TypeSchema::Option(Box::new(TypeSchema::String)), comment: "Optional explicit document ID; generated if omitted.", required: false }, + ], + outputs: vec![FieldSchema { name: "document_id", ty: TypeSchema::String, comment: "ID of the upserted document.", required: true }], + }, + "doc_ingest" => ControllerSchema { + namespace: "memory", + function: "doc_ingest", + description: "Ingest a document with entity/relation extraction and chunk embedding.", + inputs: vec![ + FieldSchema { name: "namespace", ty: TypeSchema::String, comment: "Target namespace.", required: true }, + FieldSchema { name: "key", ty: TypeSchema::String, comment: "Document key.", required: true }, + FieldSchema { name: "title", ty: TypeSchema::String, comment: "Human-readable title.", required: true }, + FieldSchema { name: "content", ty: TypeSchema::String, comment: "Document body content.", required: true }, + FieldSchema { name: "source_type", ty: TypeSchema::String, comment: "Source type label (default: \"doc\").", required: false }, + FieldSchema { name: "priority", ty: TypeSchema::String, comment: "Priority level (default: \"medium\").", required: false }, + FieldSchema { name: "tags", ty: TypeSchema::Array(Box::new(TypeSchema::String)), comment: "Tags for categorisation (default: []).", required: false }, + FieldSchema { name: "metadata", ty: TypeSchema::Json, comment: "Arbitrary metadata (default: {}).", required: false }, + FieldSchema { name: "category", ty: TypeSchema::String, comment: "Memory category (default: \"core\").", required: false }, + FieldSchema { name: "session_id", ty: TypeSchema::Option(Box::new(TypeSchema::String)), comment: "Optional session ID.", required: false }, + FieldSchema { name: "document_id", ty: TypeSchema::Option(Box::new(TypeSchema::String)), comment: "Optional explicit document ID.", required: false }, + FieldSchema { name: "config", ty: TypeSchema::Option(Box::new(TypeSchema::Json)), comment: "Optional ingestion configuration overrides.", required: false }, + ], + outputs: vec![FieldSchema { name: "result", ty: TypeSchema::Json, comment: "Ingestion result with entity, relation and chunk counts.", required: true }], + }, + "doc_list" => ControllerSchema { + namespace: "memory", + function: "doc_list", + description: "List documents in the unified memory store, optionally by namespace.", + inputs: vec![FieldSchema { + name: "namespace", + ty: TypeSchema::Option(Box::new(TypeSchema::String)), + comment: "Optional namespace filter.", + required: false, + }], + outputs: vec![FieldSchema { name: "result", ty: TypeSchema::Json, comment: "Document listing.", required: true }], + }, + "doc_delete" => ControllerSchema { + namespace: "memory", + function: "doc_delete", + description: "Delete a document from the unified memory store.", + inputs: vec![ + FieldSchema { name: "namespace", ty: TypeSchema::String, comment: "Namespace containing the document.", required: true }, + FieldSchema { name: "document_id", ty: TypeSchema::String, comment: "Document identifier to delete.", required: true }, + ], + outputs: vec![FieldSchema { name: "result", ty: TypeSchema::Json, comment: "Deletion result.", required: true }], + }, + "context_query" => ControllerSchema { + namespace: "memory", + function: "context_query", + description: "Query a namespace for contextual information.", + inputs: vec![ + FieldSchema { name: "namespace", ty: TypeSchema::String, comment: "Namespace to query.", required: true }, + FieldSchema { name: "query", ty: TypeSchema::String, comment: "Natural-language query string.", required: true }, + FieldSchema { name: "limit", ty: TypeSchema::Option(Box::new(TypeSchema::U64)), comment: "Maximum number of results.", required: false }, + ], + outputs: vec![FieldSchema { name: "result", ty: TypeSchema::String, comment: "Contextual query result string.", required: true }], + }, + "context_recall" => ControllerSchema { + namespace: "memory", + function: "context_recall", + description: "Recall context from a namespace.", + inputs: vec![ + FieldSchema { name: "namespace", ty: TypeSchema::String, comment: "Namespace to recall from.", required: true }, + FieldSchema { name: "limit", ty: TypeSchema::Option(Box::new(TypeSchema::U64)), comment: "Maximum number of results.", required: false }, + ], + outputs: vec![FieldSchema { name: "result", ty: TypeSchema::Json, comment: "Recalled context (may be null if empty).", required: true }], + }, + "clear_namespace" => ControllerSchema { + namespace: "memory", + function: "clear_namespace", + description: "Delete all documents, vector chunks, KV entries, and graph relations for a namespace.", + inputs: vec![FieldSchema { + name: "namespace", + ty: TypeSchema::String, + comment: "Namespace to clear completely.", + required: true, + }], + outputs: vec![ + FieldSchema { name: "cleared", ty: TypeSchema::Bool, comment: "True when the namespace was cleared.", required: true }, + FieldSchema { name: "namespace", ty: TypeSchema::String, comment: "The namespace that was cleared.", required: true }, + ], + }, + _ => return None, + }) +} + +// --------------------------------------------------------------------------- +// Handlers +// --------------------------------------------------------------------------- + +fn handle_init(params: Map) -> ControllerFuture { + Box::pin(async move { + let payload = parse_params::(params)?; + to_json(rpc::memory_init(payload).await?) + }) +} + +fn handle_list_documents(params: Map) -> ControllerFuture { + Box::pin(async move { + let payload = parse_params::(params)?; + to_json(rpc::memory_list_documents(payload).await?) + }) +} + +fn handle_list_namespaces(_params: Map) -> ControllerFuture { + Box::pin(async move { to_json(rpc::memory_list_namespaces(EmptyRequest {}).await?) }) +} + +fn handle_delete_document(params: Map) -> ControllerFuture { + Box::pin(async move { + let payload = parse_params::(params)?; + to_json(rpc::memory_delete_document(payload).await?) + }) +} + +fn handle_query_namespace(params: Map) -> ControllerFuture { + Box::pin(async move { + let payload = parse_params::(params)?; + to_json(rpc::memory_query_namespace(payload).await?) + }) +} + +fn handle_recall_context(params: Map) -> ControllerFuture { + Box::pin(async move { + let payload = parse_params::(params)?; + to_json(rpc::memory_recall_context(payload).await?) + }) +} + +fn handle_recall_memories(params: Map) -> ControllerFuture { + Box::pin(async move { + let payload = parse_params::(params)?; + to_json(rpc::memory_recall_memories(payload).await?) + }) +} + +fn handle_namespace_list(_params: Map) -> ControllerFuture { + Box::pin(async move { to_json(rpc::namespace_list().await?) }) +} + +fn handle_doc_put(params: Map) -> ControllerFuture { + Box::pin(async move { + let payload = parse_params::(params)?; + to_json(rpc::doc_put(payload).await?) + }) +} + +fn handle_doc_ingest(params: Map) -> ControllerFuture { + Box::pin(async move { + let payload = parse_params::(params)?; + to_json(rpc::doc_ingest(payload).await?) + }) +} + +#[derive(serde::Deserialize)] +struct DocListParams { + #[serde(default)] + namespace: Option, +} + +fn handle_doc_list(params: Map) -> ControllerFuture { + Box::pin(async move { + // Reject invalid `namespace` types (e.g. `123`, `["x"]`) instead of + // silently coercing to `None` and returning an unscoped document list. + let parsed: DocListParams = parse_params(params)?; + let namespace = parsed + .namespace + .map(|namespace| NamespaceOnlyParams { namespace }); + to_json(rpc::doc_list(namespace).await?) + }) +} + +fn handle_doc_delete(params: Map) -> ControllerFuture { + Box::pin(async move { + let payload = parse_params::(params)?; + to_json(rpc::doc_delete(payload).await?) + }) +} + +fn handle_context_query(params: Map) -> ControllerFuture { + Box::pin(async move { + let payload = parse_params::(params)?; + to_json(rpc::context_query(payload).await?) + }) +} + +fn handle_context_recall(params: Map) -> ControllerFuture { + Box::pin(async move { + let payload = parse_params::(params)?; + to_json(rpc::context_recall(payload).await?) + }) +} + +fn handle_clear_namespace(params: Map) -> ControllerFuture { + Box::pin(async move { + let payload = parse_params::(params)?; + to_json(rpc::clear_namespace(payload).await?) + }) +} diff --git a/src/openhuman/memory/schemas/files.rs b/src/openhuman/memory/schemas/files.rs new file mode 100644 index 000000000..53525e7b2 --- /dev/null +++ b/src/openhuman/memory/schemas/files.rs @@ -0,0 +1,128 @@ +//! Schemas and handlers for file-based memory RPC methods. + +use serde_json::{Map, Value}; + +use crate::core::all::{ControllerFuture, RegisteredController}; +use crate::core::{ControllerSchema, FieldSchema, TypeSchema}; +use crate::openhuman::memory::rpc; +use crate::openhuman::memory::{ + ListMemoryFilesRequest, ReadMemoryFileRequest, WriteMemoryFileRequest, +}; + +use super::{parse_params, to_json}; + +pub(super) const FUNCTIONS: &[&str] = &["list_files", "read_file", "write_file"]; + +pub(super) fn controllers() -> Vec { + vec![ + RegisteredController { + schema: schema("list_files").unwrap(), + handler: handle_list_files, + }, + RegisteredController { + schema: schema("read_file").unwrap(), + handler: handle_read_file, + }, + RegisteredController { + schema: schema("write_file").unwrap(), + handler: handle_write_file, + }, + ] +} + +pub(super) fn schema(function: &str) -> Option { + Some(match function { + "list_files" => ControllerSchema { + namespace: "memory", + function: "list_files", + description: "List files in a memory directory.", + inputs: vec![FieldSchema { + name: "relative_dir", + ty: TypeSchema::String, + comment: "Relative directory path under the workspace (default: \"memory\").", + required: false, + }], + outputs: vec![FieldSchema { + name: "result", + ty: TypeSchema::Json, + comment: "Envelope with file listing.", + required: true, + }], + }, + "read_file" => ControllerSchema { + namespace: "memory", + function: "read_file", + description: "Read the contents of a memory file.", + inputs: vec![FieldSchema { + name: "relative_path", + ty: TypeSchema::String, + comment: "Relative path to the file under the workspace memory directory.", + required: true, + }], + outputs: vec![FieldSchema { + name: "result", + ty: TypeSchema::Json, + comment: "Envelope with file content.", + required: true, + }], + }, + "write_file" => ControllerSchema { + namespace: "memory", + function: "write_file", + description: "Write content to a memory file.", + inputs: vec![ + FieldSchema { + name: "relative_path", + ty: TypeSchema::String, + comment: "Relative path to the file under the workspace memory directory.", + required: true, + }, + FieldSchema { + name: "content", + ty: TypeSchema::String, + comment: "Content to write to the file.", + required: true, + }, + ], + outputs: vec![FieldSchema { + name: "result", + ty: TypeSchema::Json, + comment: "Envelope with write confirmation and bytes written.", + required: true, + }], + }, + _ => return None, + }) +} + +#[derive(serde::Deserialize)] +struct ListFilesParams { + #[serde(default)] + relative_dir: Option, +} + +fn handle_list_files(params: Map) -> ControllerFuture { + Box::pin(async move { + // Reject invalid `relative_dir` types (e.g. `123`, `["x"]`) instead of + // silently defaulting and masking client errors. + let parsed: ListFilesParams = parse_params(params)?; + // Empty string == the memory root itself (`/memory`). + let relative_dir = parsed.relative_dir.unwrap_or_default(); + let payload = ListMemoryFilesRequest { relative_dir }; + to_json(rpc::ai_list_memory_files(payload).await?) + }) +} + +fn handle_read_file(params: Map) -> ControllerFuture { + Box::pin(async move { + let payload = parse_params::(params)?; + to_json(rpc::ai_read_memory_file(payload).await?) + }) +} + +fn handle_write_file(params: Map) -> ControllerFuture { + Box::pin(async move { + let payload = parse_params::(params)?; + to_json(rpc::ai_write_memory_file(payload).await?) + }) +} diff --git a/src/openhuman/memory/schemas/kv_graph.rs b/src/openhuman/memory/schemas/kv_graph.rs new file mode 100644 index 000000000..0fa77eb05 --- /dev/null +++ b/src/openhuman/memory/schemas/kv_graph.rs @@ -0,0 +1,269 @@ +//! Schemas and handlers for key-value and knowledge-graph RPC methods. + +use serde_json::{Map, Value}; + +use crate::core::all::{ControllerFuture, RegisteredController}; +use crate::core::{ControllerSchema, FieldSchema, TypeSchema}; +use crate::openhuman::memory::rpc::{ + self, GraphQueryParams, GraphUpsertParams, KvGetDeleteParams, KvSetParams, NamespaceOnlyParams, +}; + +use super::{parse_params, to_json}; + +pub(super) const FUNCTIONS: &[&str] = &[ + "kv_set", + "kv_get", + "kv_delete", + "kv_list_namespace", + "graph_upsert", + "graph_query", +]; + +pub(super) fn controllers() -> Vec { + vec![ + RegisteredController { + schema: schema("kv_set").unwrap(), + handler: handle_kv_set, + }, + RegisteredController { + schema: schema("kv_get").unwrap(), + handler: handle_kv_get, + }, + RegisteredController { + schema: schema("kv_delete").unwrap(), + handler: handle_kv_delete, + }, + RegisteredController { + schema: schema("kv_list_namespace").unwrap(), + handler: handle_kv_list_namespace, + }, + RegisteredController { + schema: schema("graph_upsert").unwrap(), + handler: handle_graph_upsert, + }, + RegisteredController { + schema: schema("graph_query").unwrap(), + handler: handle_graph_query, + }, + ] +} + +pub(super) fn schema(function: &str) -> Option { + Some(match function { + "kv_set" => ControllerSchema { + namespace: "memory", + function: "kv_set", + description: "Set a key-value pair in the memory store.", + inputs: vec![ + FieldSchema { + name: "namespace", + ty: TypeSchema::Option(Box::new(TypeSchema::String)), + comment: "Optional namespace scope.", + required: false, + }, + FieldSchema { + name: "key", + ty: TypeSchema::String, + comment: "Key to set.", + required: true, + }, + FieldSchema { + name: "value", + ty: TypeSchema::Json, + comment: "JSON value to store.", + required: true, + }, + ], + outputs: vec![FieldSchema { + name: "result", + ty: TypeSchema::Bool, + comment: "True when the value was stored.", + required: true, + }], + }, + "kv_get" => ControllerSchema { + namespace: "memory", + function: "kv_get", + description: "Get a value by key from the memory store.", + inputs: vec![ + FieldSchema { + name: "namespace", + ty: TypeSchema::Option(Box::new(TypeSchema::String)), + comment: "Optional namespace scope.", + required: false, + }, + FieldSchema { + name: "key", + ty: TypeSchema::String, + comment: "Key to retrieve.", + required: true, + }, + ], + outputs: vec![FieldSchema { + name: "result", + ty: TypeSchema::Json, + comment: "Stored value or null if not found.", + required: true, + }], + }, + "kv_delete" => ControllerSchema { + namespace: "memory", + function: "kv_delete", + description: "Delete a key-value pair from the memory store.", + inputs: vec![ + FieldSchema { + name: "namespace", + ty: TypeSchema::Option(Box::new(TypeSchema::String)), + comment: "Optional namespace scope.", + required: false, + }, + FieldSchema { + name: "key", + ty: TypeSchema::String, + comment: "Key to delete.", + required: true, + }, + ], + outputs: vec![FieldSchema { + name: "result", + ty: TypeSchema::Bool, + comment: "True when the key was deleted.", + required: true, + }], + }, + "kv_list_namespace" => ControllerSchema { + namespace: "memory", + function: "kv_list_namespace", + description: "List all key-value entries in a namespace.", + inputs: vec![FieldSchema { + name: "namespace", + ty: TypeSchema::String, + comment: "Namespace to list.", + required: true, + }], + outputs: vec![FieldSchema { + name: "result", + ty: TypeSchema::Json, + comment: "Array of key-value entries.", + required: true, + }], + }, + "graph_upsert" => ControllerSchema { + namespace: "memory", + function: "graph_upsert", + description: "Upsert a relation triple in the knowledge graph.", + inputs: vec![ + FieldSchema { + name: "namespace", + ty: TypeSchema::Option(Box::new(TypeSchema::String)), + comment: "Optional namespace scope.", + required: false, + }, + FieldSchema { + name: "subject", + ty: TypeSchema::String, + comment: "Subject entity of the relation.", + required: true, + }, + FieldSchema { + name: "predicate", + ty: TypeSchema::String, + comment: "Relation predicate.", + required: true, + }, + FieldSchema { + name: "object", + ty: TypeSchema::String, + comment: "Object entity of the relation.", + required: true, + }, + FieldSchema { + name: "attrs", + ty: TypeSchema::Json, + comment: "Extra attributes on the relation (default: {}).", + required: false, + }, + ], + outputs: vec![FieldSchema { + name: "result", + ty: TypeSchema::Bool, + comment: "True when the relation was upserted.", + required: true, + }], + }, + "graph_query" => ControllerSchema { + namespace: "memory", + function: "graph_query", + description: "Query relations from the knowledge graph.", + inputs: vec![ + FieldSchema { + name: "namespace", + ty: TypeSchema::Option(Box::new(TypeSchema::String)), + comment: "Optional namespace scope.", + required: false, + }, + FieldSchema { + name: "subject", + ty: TypeSchema::Option(Box::new(TypeSchema::String)), + comment: "Filter by subject entity.", + required: false, + }, + FieldSchema { + name: "predicate", + ty: TypeSchema::Option(Box::new(TypeSchema::String)), + comment: "Filter by relation predicate.", + required: false, + }, + ], + outputs: vec![FieldSchema { + name: "result", + ty: TypeSchema::Json, + comment: "Array of matching relation records.", + required: true, + }], + }, + _ => return None, + }) +} + +fn handle_kv_set(params: Map) -> ControllerFuture { + Box::pin(async move { + let payload = parse_params::(params)?; + to_json(rpc::kv_set(payload).await?) + }) +} + +fn handle_kv_get(params: Map) -> ControllerFuture { + Box::pin(async move { + let payload = parse_params::(params)?; + to_json(rpc::kv_get(payload).await?) + }) +} + +fn handle_kv_delete(params: Map) -> ControllerFuture { + Box::pin(async move { + let payload = parse_params::(params)?; + to_json(rpc::kv_delete(payload).await?) + }) +} + +fn handle_kv_list_namespace(params: Map) -> ControllerFuture { + Box::pin(async move { + let payload = parse_params::(params)?; + to_json(rpc::kv_list_namespace(payload).await?) + }) +} + +fn handle_graph_upsert(params: Map) -> ControllerFuture { + Box::pin(async move { + let payload = parse_params::(params)?; + to_json(rpc::graph_upsert(payload).await?) + }) +} + +fn handle_graph_query(params: Map) -> ControllerFuture { + Box::pin(async move { + let payload = parse_params::(params)?; + to_json(rpc::graph_query(payload).await?) + }) +} diff --git a/src/openhuman/memory/schemas/learn.rs b/src/openhuman/memory/schemas/learn.rs new file mode 100644 index 000000000..666f8356c --- /dev/null +++ b/src/openhuman/memory/schemas/learn.rs @@ -0,0 +1,56 @@ +//! Schema and handler for the `memory.learn_all` RPC method. + +use serde_json::{Map, Value}; + +use crate::core::all::{ControllerFuture, RegisteredController}; +use crate::core::{ControllerSchema, FieldSchema, TypeSchema}; +use crate::openhuman::memory::rpc::{self, LearnAllParams}; + +use super::{parse_params, to_json}; + +pub(super) const FUNCTIONS: &[&str] = &["learn_all"]; + +pub(super) fn controllers() -> Vec { + vec![RegisteredController { + schema: schema("learn_all").unwrap(), + handler: handle_learn_all, + }] +} + +pub(super) fn schema(function: &str) -> Option { + Some(match function { + "learn_all" => ControllerSchema { + namespace: "memory", + function: "learn_all", + description: "Run the tree summarizer over all memory namespaces (or a constrained subset). Processes namespaces sequentially; a failing namespace is recorded but does not abort the rest.", + inputs: vec![FieldSchema { + name: "namespaces", + ty: TypeSchema::Option(Box::new(TypeSchema::Array(Box::new(TypeSchema::String)))), + comment: "Optional list of namespaces to constrain. Defaults to all namespaces.", + required: false, + }], + outputs: vec![ + FieldSchema { + name: "namespaces_processed", + ty: TypeSchema::U64, + comment: "Total number of namespaces processed.", + required: true, + }, + FieldSchema { + name: "results", + ty: TypeSchema::Json, + comment: "Per-namespace outcomes: [{ namespace, status: 'ok'|'skipped'|'error', error? }].", + required: true, + }, + ], + }, + _ => return None, + }) +} + +fn handle_learn_all(params: Map) -> ControllerFuture { + Box::pin(async move { + let payload = parse_params::(params)?; + to_json(rpc::memory_learn_all(payload).await?) + }) +} diff --git a/src/openhuman/memory/schemas/mod.rs b/src/openhuman/memory/schemas/mod.rs new file mode 100644 index 000000000..e7c775b4b --- /dev/null +++ b/src/openhuman/memory/schemas/mod.rs @@ -0,0 +1,109 @@ +//! RPC schemas and controller registration for the memory system. +//! +//! This module defines the metadata (schemas) for all memory-related RPC +//! functions and registers their corresponding handlers. It serves as the +//! bridge between the RPC system and the underlying memory operations. +//! +//! Internally the schemas are organised into family submodules that mirror +//! [`crate::openhuman::memory::ops`]: +//! +//! - [`documents`] — doc/namespace/recall/clear schemas + handlers. +//! - [`kv_graph`] — key-value and knowledge-graph schemas + handlers. +//! - [`sync`] — `sync_channel`, `sync_all`, `ingestion_status`. +//! - [`learn`] — `learn_all`. +//! - [`files`] — file-based memory schemas + handlers. + +use serde::de::DeserializeOwned; +use serde_json::{Map, Value}; + +use crate::core::all::RegisteredController; +use crate::core::{ControllerSchema, FieldSchema, TypeSchema}; +use crate::rpc::RpcOutcome; + +mod documents; +mod files; +mod kv_graph; +mod learn; +mod sync; + +// --------------------------------------------------------------------------- +// Public entry points +// --------------------------------------------------------------------------- + +/// Returns all controller schemas for the memory system. +pub fn all_controller_schemas() -> Vec { + let mut out = Vec::new(); + out.extend(documents::FUNCTIONS.iter().map(|f| schemas(f))); + out.extend(files::FUNCTIONS.iter().map(|f| schemas(f))); + out.extend(kv_graph::FUNCTIONS.iter().map(|f| schemas(f))); + out.extend(sync::FUNCTIONS.iter().map(|f| schemas(f))); + out.extend(learn::FUNCTIONS.iter().map(|f| schemas(f))); + out +} + +/// Returns all registered controllers for the memory system, mapping schemas to handlers. +pub fn all_registered_controllers() -> Vec { + let mut out = Vec::new(); + out.extend(documents::controllers()); + out.extend(files::controllers()); + out.extend(kv_graph::controllers()); + out.extend(sync::controllers()); + out.extend(learn::controllers()); + out +} + +/// Defines the schema for a specific memory controller function. +pub fn schemas(function: &str) -> ControllerSchema { + if let Some(schema) = documents::schema(function) { + return schema; + } + if let Some(schema) = files::schema(function) { + return schema; + } + if let Some(schema) = kv_graph::schema(function) { + return schema; + } + if let Some(schema) = sync::schema(function) { + return schema; + } + if let Some(schema) = learn::schema(function) { + return schema; + } + unknown_schema() +} + +fn unknown_schema() -> ControllerSchema { + ControllerSchema { + namespace: "memory", + function: "unknown", + description: "Unknown memory controller function.", + inputs: vec![FieldSchema { + name: "function", + ty: TypeSchema::String, + comment: "Unknown function requested for schema lookup.", + required: true, + }], + outputs: vec![FieldSchema { + name: "error", + ty: TypeSchema::String, + comment: "Lookup error details.", + required: true, + }], + } +} + +// --------------------------------------------------------------------------- +// Helpers shared by every handler submodule +// --------------------------------------------------------------------------- + +pub(super) fn parse_params(params: Map) -> Result { + serde_json::from_value(Value::Object(params)).map_err(|e| format!("invalid params: {e}")) +} + +pub(super) fn to_json(outcome: RpcOutcome) -> Result { + outcome.into_cli_compatible_json() +} + +#[cfg(test)] +#[path = "../schemas_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/schemas/sync.rs b/src/openhuman/memory/schemas/sync.rs new file mode 100644 index 000000000..8ba74c7f3 --- /dev/null +++ b/src/openhuman/memory/schemas/sync.rs @@ -0,0 +1,87 @@ +//! Schemas and handlers for memory-sync and ingestion-status RPC methods. + +use serde_json::{Map, Value}; + +use crate::core::all::{ControllerFuture, RegisteredController}; +use crate::core::{ControllerSchema, FieldSchema, TypeSchema}; +use crate::openhuman::memory::rpc::{self, SyncChannelParams}; + +use super::{parse_params, to_json}; + +pub(super) const FUNCTIONS: &[&str] = &["sync_channel", "sync_all", "ingestion_status"]; + +pub(super) fn controllers() -> Vec { + vec![ + RegisteredController { + schema: schema("sync_channel").unwrap(), + handler: handle_sync_channel, + }, + RegisteredController { + schema: schema("sync_all").unwrap(), + handler: handle_sync_all, + }, + RegisteredController { + schema: schema("ingestion_status").unwrap(), + handler: handle_ingestion_status, + }, + ] +} + +pub(super) fn schema(function: &str) -> Option { + Some(match function { + "sync_channel" => ControllerSchema { + namespace: "memory", + function: "sync_channel", + description: "Request a memory sync for a specific channel. Publishes MemorySyncRequested on the event bus. No ingestion consumers exist yet; this is a hook for future pull-based subscribers.", + inputs: vec![FieldSchema { + name: "channel_id", + ty: TypeSchema::String, + comment: "ID of the channel to sync.", + required: true, + }], + outputs: vec![ + FieldSchema { name: "requested", ty: TypeSchema::Bool, comment: "Always true when the event was published.", required: true }, + FieldSchema { name: "channel_id", ty: TypeSchema::String, comment: "Echo of the channel_id that was requested.", required: true }, + ], + }, + "sync_all" => ControllerSchema { + namespace: "memory", + function: "sync_all", + description: "Request a memory sync for all channels. Publishes MemorySyncRequested { channel_id: None } on the event bus.", + inputs: vec![], + outputs: vec![FieldSchema { name: "requested", ty: TypeSchema::Bool, comment: "Always true when the event was published.", required: true }], + }, + "ingestion_status" => ControllerSchema { + namespace: "memory", + function: "ingestion_status", + description: "Returns the current memory-ingestion status (whether a job is running, the in-flight document, queue depth, and most recent completion). Safe to poll.", + inputs: vec![], + outputs: vec![ + FieldSchema { name: "running", ty: TypeSchema::Bool, comment: "True while an ingestion job is running on the local extraction LLM.", required: true }, + FieldSchema { name: "current_document_id", ty: TypeSchema::Option(Box::new(TypeSchema::String)), comment: "Document id of the in-flight job.", required: false }, + FieldSchema { name: "current_title", ty: TypeSchema::Option(Box::new(TypeSchema::String)), comment: "Title of the in-flight document.", required: false }, + FieldSchema { name: "current_namespace", ty: TypeSchema::Option(Box::new(TypeSchema::String)), comment: "Namespace of the in-flight document.", required: false }, + FieldSchema { name: "queue_depth", ty: TypeSchema::U64, comment: "Number of jobs waiting behind the current one.", required: true }, + FieldSchema { name: "last_completed_at", ty: TypeSchema::Option(Box::new(TypeSchema::I64)), comment: "Unix-ms timestamp of the most recent completion.", required: false }, + FieldSchema { name: "last_document_id", ty: TypeSchema::Option(Box::new(TypeSchema::String)), comment: "Document id of the most recent completed job.", required: false }, + FieldSchema { name: "last_success", ty: TypeSchema::Option(Box::new(TypeSchema::Bool)), comment: "Whether the most recent job succeeded.", required: false }, + ], + }, + _ => return None, + }) +} + +fn handle_sync_channel(params: Map) -> ControllerFuture { + Box::pin(async move { + let payload = parse_params::(params)?; + to_json(rpc::memory_sync_channel(payload).await?) + }) +} + +fn handle_sync_all(_params: Map) -> ControllerFuture { + Box::pin(async move { to_json(rpc::memory_sync_all().await?) }) +} + +fn handle_ingestion_status(_params: Map) -> ControllerFuture { + Box::pin(async move { to_json(rpc::memory_ingestion_status().await?) }) +} diff --git a/src/openhuman/memory/schemas_tests.rs b/src/openhuman/memory/schemas_tests.rs index 292b0b05e..d7e410768 100644 --- a/src/openhuman/memory/schemas_tests.rs +++ b/src/openhuman/memory/schemas_tests.rs @@ -1,3 +1,6 @@ +//! Unit tests for memory RPC schema registration and parameter parsing, +//! validating that every advertised function name has a registered controller. + use super::*; use serde_json::json; diff --git a/src/openhuman/memory/slack_ingestion/README.md b/src/openhuman/memory/slack_ingestion/README.md new file mode 100644 index 000000000..fc2f45d98 --- /dev/null +++ b/src/openhuman/memory/slack_ingestion/README.md @@ -0,0 +1,37 @@ +# slack_ingestion + +Slack-specific plumbing that feeds the memory tree. Auth and scheduling +live elsewhere — OAuth via Composio, periodic ticks via +`composio::periodic` — this folder converts Slack messages into +`tree::ingest::ingest_chat` calls. + +## Files + +- **`mod.rs`** — module re-exports + (`all_slack_ingestion_controller_schemas`, + `all_slack_ingestion_registered_controllers`). +- **`types.rs`** — internal `SlackMessage`, `SlackChannel`, `Bucket`. + Decoupled from raw Slack Web API payloads; the Composio provider + parses into these. +- **`bucketer.rs`** — 6-hour UTC-aligned windowing + (`bucket_start_for`, `bucket_end_for`), grace-period extraction + (`split_closed`), and stable `source_id_for` generation. Bucket + width is a schema constant — changing it invalidates all existing + source IDs. +- **`ops.rs`** — bucket-to-`ChatBatch` canonicalisation + (`bucket_to_chat_batch`) and the `ingest_bucket` wrapper around + `tree::ingest::ingest_chat`. Free of HTTP and timers so it is + easy to unit-test. +- **`rpc.rs`** — handler bodies for `slack_memory_sync_trigger` + (manual sync run) and `slack_memory_sync_status` (per-connection + cursor + budget snapshot). +- **`schemas.rs`** — controller schemas + dispatch wiring under the + `slack_memory` JSON-RPC namespace. + +## Where it fits + +The Composio-backed `SlackProvider` (in `composio/providers/slack/`) +calls into `bucketer` + `ops` to produce closed-bucket +`ChatBatch`es; those flow into `memory::tree::ingest::ingest_chat` and +become source-tree leaves. State (cursors, dedup set, daily budget) +lives in the Composio sync-state KV — not here. diff --git a/src/openhuman/memory/slack_ingestion/bucketer.rs b/src/openhuman/memory/slack_ingestion/bucketer.rs index bae1f5fd9..f5eb1a285 100644 --- a/src/openhuman/memory/slack_ingestion/bucketer.rs +++ b/src/openhuman/memory/slack_ingestion/bucketer.rs @@ -9,7 +9,7 @@ //! ## Why closed-bucket-only? //! //! Each ingested chunk becomes a *leaf* in the source tree via -//! `append_leaf` (see `memory::tree::source_tree::bucket_seal`). Leaves +//! `append_leaf` (see `memory::tree::tree_source::bucket_seal`). Leaves //! participate in a token-budget seal cascade — once appended, they //! cannot be cleanly retracted from already-sealed summary nodes //! upstream. So we only ingest a bucket once, *after* it has closed diff --git a/src/openhuman/memory/slack_ingestion/rpc.rs b/src/openhuman/memory/slack_ingestion/rpc.rs index 148b8fde2..4c4dffa93 100644 --- a/src/openhuman/memory/slack_ingestion/rpc.rs +++ b/src/openhuman/memory/slack_ingestion/rpc.rs @@ -27,6 +27,8 @@ pub struct SyncTriggerRequest { pub connection_id: Option, } +/// Result of `slack_memory_sync_trigger` — per-connection [`SyncOutcome`]s +/// plus aggregate counters. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct SyncTriggerResponse { pub outcomes: Vec, @@ -105,14 +107,18 @@ pub async fn sync_trigger_rpc( )) } +/// Request body for `slack_memory_sync_status` — no parameters. #[derive(Clone, Debug, Serialize, Deserialize, Default)] pub struct SyncStatusRequest {} +/// Response body for `slack_memory_sync_status` — one row per active +/// Slack Composio connection. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct SyncStatusResponse { pub connections: Vec, } +/// Per-connection sync state snapshot pulled from the Composio sync-state KV. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct ConnectionStatus { pub connection_id: String, diff --git a/src/openhuman/memory/slack_ingestion/schemas.rs b/src/openhuman/memory/slack_ingestion/schemas.rs index c1e9b6acc..457ab9774 100644 --- a/src/openhuman/memory/slack_ingestion/schemas.rs +++ b/src/openhuman/memory/slack_ingestion/schemas.rs @@ -18,10 +18,12 @@ use crate::rpc::RpcOutcome; const NAMESPACE: &str = "slack_memory"; +/// Returns every schema published by the Slack-ingestion namespace. pub fn all_controller_schemas() -> Vec { vec![schemas("sync_trigger"), schemas("sync_status")] } +/// Returns every controller (schema + handler pair) for the Slack-ingestion namespace. pub fn all_registered_controllers() -> Vec { vec![ RegisteredController { @@ -35,6 +37,7 @@ pub fn all_registered_controllers() -> Vec { ] } +/// Build the [`ControllerSchema`] for one named function in this namespace. pub fn schemas(function: &str) -> ControllerSchema { match function { "sync_trigger" => ControllerSchema { diff --git a/src/openhuman/memory/store/README.md b/src/openhuman/memory/store/README.md new file mode 100644 index 000000000..872fd3d3c --- /dev/null +++ b/src/openhuman/memory/store/README.md @@ -0,0 +1,17 @@ +# Memory store + +Storage backend for the memory subsystem. Houses the SQLite + FTS5 + vector + graph implementation (`UnifiedMemory`), the async client handle used by RPC controllers (`MemoryClient`), the `Memory` trait impl bridging both, and the factory functions used to bootstrap a memory instance. + +## Files + +- **`mod.rs`** — module root; re-exports `UnifiedMemory`, `MemoryClient`, factory functions, and the public types from `types.rs`. +- **`types.rs`** — public input/output structs (`NamespaceDocumentInput`, `NamespaceMemoryHit`, `NamespaceRetrievalContext`, `RetrievalScoreBreakdown`, `MemoryItemKind`, `StoredMemoryDocument`, `MemoryKvRecord`, `GraphRelationRecord`) plus the `GLOBAL_NAMESPACE` sentinel. +- **`client.rs`** — `MemoryClient` / `MemoryClientRef` / `MemoryState`. Async wrapper around `UnifiedMemory` that owns the singleton ingestion queue and exposes the surface called by RPC handlers (`put_doc`, `ingest_doc`, `query_namespace`, `recall_namespace_*`, `kv_*`, `graph_*`, skill-sync helpers). Always local — no remote sync. +- **`client_tests.rs`** — coverage for the client-facing storage and graph round-trips against a fresh temp workspace. +- **`factories.rs`** — `create_memory*` constructors that select the embedding provider from `MemoryConfig` and instantiate `UnifiedMemory`. `effective_memory_backend_name` always reports `"namespace"`. +- **`memory_trait.rs`** — `impl Memory for UnifiedMemory`, mapping the generic trait surface (`store`, `recall`, `get`, `list`, `forget`, `namespace_summaries`) onto the unified store. Includes namespace normalisation and episodic-session augmentation. +- **`unified/`** — the SQLite implementation, broken into per-table submodules. See `unified/README.md`. + +## How it fits + +Callers (RPC controllers, the agent harness, learning pipelines) interact with `MemoryClient`. The client delegates persistence to `UnifiedMemory` and offloads heavier work (chunk + embed + graph extraction) to the singleton `IngestionQueue` defined in `../ingestion/`. Generic consumers that just need `Memory` trait behaviour go through `memory_trait.rs`. diff --git a/src/openhuman/memory/store/client.rs b/src/openhuman/memory/store/client.rs index eae62d308..b6d9b6f15 100644 --- a/src/openhuman/memory/store/client.rs +++ b/src/openhuman/memory/store/client.rs @@ -11,12 +11,12 @@ use serde_json::json; use std::path::PathBuf; use std::sync::Arc; -use crate::openhuman::memory::embeddings::{self, EmbeddingProvider}; +use crate::openhuman::embeddings::{self, EmbeddingProvider}; +use crate::openhuman::memory::ingestion::queue as ingestion_queue; use crate::openhuman::memory::ingestion::{ - MemoryIngestionConfig, MemoryIngestionRequest, MemoryIngestionResult, + IngestionJob, IngestionQueue, IngestionState, MemoryIngestionConfig, MemoryIngestionRequest, + MemoryIngestionResult, }; -use crate::openhuman::memory::ingestion_queue::{self, IngestionJob, IngestionQueue}; -use crate::openhuman::memory::ingestion_state::IngestionState; use crate::openhuman::memory::store::types::{ NamespaceDocumentInput, NamespaceMemoryHit, NamespaceRetrievalContext, }; diff --git a/src/openhuman/memory/store/client_tests.rs b/src/openhuman/memory/store/client_tests.rs index 653391b0d..927c4709b 100644 --- a/src/openhuman/memory/store/client_tests.rs +++ b/src/openhuman/memory/store/client_tests.rs @@ -1,3 +1,6 @@ +//! Tests for `MemoryClient` — exercise the sync storage surface (upsert, list, +//! kv, graph) against a fresh temp workspace. + use super::*; use tempfile::TempDir; diff --git a/src/openhuman/memory/store/factories.rs b/src/openhuman/memory/store/factories.rs index c7d2921be..79e333b5e 100644 --- a/src/openhuman/memory/store/factories.rs +++ b/src/openhuman/memory/store/factories.rs @@ -12,7 +12,7 @@ use std::path::Path; use std::sync::Arc; use crate::openhuman::config::{EmbeddingRouteConfig, MemoryConfig, StorageProviderConfig}; -use crate::openhuman::memory::embeddings::{self, EmbeddingProvider}; +use crate::openhuman::embeddings::{self, EmbeddingProvider}; use crate::openhuman::memory::store::unified::UnifiedMemory; use crate::openhuman::memory::traits::Memory; diff --git a/src/openhuman/memory/store/memory_trait.rs b/src/openhuman/memory/store/memory_trait.rs index cf139798a..3d64bd657 100644 --- a/src/openhuman/memory/store/memory_trait.rs +++ b/src/openhuman/memory/store/memory_trait.rs @@ -331,7 +331,7 @@ impl Memory for UnifiedMemory { #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory::embeddings::NoopEmbedding; + use crate::openhuman::embeddings::NoopEmbedding; use std::sync::Arc; use tempfile::TempDir; diff --git a/src/openhuman/memory/store/types.rs b/src/openhuman/memory/store/types.rs index 6d4576457..dc65e8677 100644 --- a/src/openhuman/memory/store/types.rs +++ b/src/openhuman/memory/store/types.rs @@ -4,6 +4,11 @@ use serde::{Deserialize, Serialize}; pub(crate) const GLOBAL_NAMESPACE: &str = "global"; +/// Input payload for upserting a namespace-scoped memory document. +/// +/// Used by `MemoryClient::put_doc` and the ingestion pipeline. `document_id` +/// is optional — when omitted, an existing row keyed by `(namespace, key)` is +/// reused, otherwise a new id is generated. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct NamespaceDocumentInput { pub namespace: String, @@ -23,6 +28,7 @@ pub struct NamespaceDocumentInput { pub document_id: Option, } +/// One ranked retrieval result for a namespace text query. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct NamespaceQueryResult { pub key: String, @@ -32,6 +38,7 @@ pub struct NamespaceQueryResult { pub category: String, } +/// Discriminator for the kind of stored memory item a hit refers to. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum MemoryItemKind { @@ -41,6 +48,8 @@ pub enum MemoryItemKind { Event, } +/// Persisted form of a memory document as stored in `memory_docs`, +/// including timestamps and the markdown sidecar path. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct StoredMemoryDocument { pub document_id: String, @@ -59,6 +68,7 @@ pub struct StoredMemoryDocument { pub markdown_rel_path: String, } +/// A single KV row, namespace-scoped or global (when `namespace` is `None`). #[derive(Debug, Clone, Serialize, Deserialize)] pub struct MemoryKvRecord { pub namespace: Option, @@ -67,6 +77,10 @@ pub struct MemoryKvRecord { pub updated_at: f64, } +/// A graph edge (subject — predicate → object) plus accumulated evidence. +/// +/// `document_ids` and `chunk_ids` track every source that contributed to this +/// relation; `evidence_count` is the merged count after de-duplication. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GraphRelationRecord { pub namespace: Option, @@ -81,6 +95,8 @@ pub struct GraphRelationRecord { pub chunk_ids: Vec, } +/// Per-signal contribution to a hit's final score, surfaced for debugging +/// and UI ranking explainers. #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct RetrievalScoreBreakdown { pub keyword_relevance: f64, @@ -91,6 +107,8 @@ pub struct RetrievalScoreBreakdown { pub final_score: f64, } +/// A single ranked retrieval hit returned from `query_namespace_hits` / +/// `recall_namespace_memories`. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct NamespaceMemoryHit { pub id: String, @@ -112,6 +130,8 @@ pub struct NamespaceMemoryHit { pub supporting_relations: Vec, } +/// Aggregated retrieval result for a namespace: rendered context text plus +/// the underlying hits. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct NamespaceRetrievalContext { pub namespace: String, diff --git a/src/openhuman/memory/store/unified/README.md b/src/openhuman/memory/store/unified/README.md new file mode 100644 index 000000000..1a0a491ec --- /dev/null +++ b/src/openhuman/memory/store/unified/README.md @@ -0,0 +1,22 @@ +# Unified memory store + +SQLite-backed implementation of the memory store. One `UnifiedMemory` struct owns a WAL-mode connection plus the on-disk markdown sidecar tree and vector storage path; the rest of this directory adds capabilities to it via per-domain `impl` blocks. + +## Files + +- **`mod.rs`** — declares the `UnifiedMemory` struct (connection + paths + embedder) and wires the submodules. +- **`init.rs`** — constructor, `CREATE TABLE` bootstrap (docs, kv, graph, vector chunks, episodic FTS5, segments, events, profile), idempotent legacy-namespace migrations, plus path / namespace helpers (`sanitize_namespace`, `now_ts`, `namespace_dir`). +- **`documents.rs`** — `memory_docs` CRUD: `upsert_document` (chunks + embeds + writes markdown sidecar), `upsert_document_metadata_only` (light path), `list_documents`, `list_namespaces`, `delete_document`, `clear_namespace`. +- **`kv.rs`** — global and namespace-scoped get/set/delete/list against `kv_global` / `kv_namespace`. +- **`graph.rs`** — `graph_namespace` / `graph_global` upserts with attribute merging and evidence accumulation, plus namespace / global / cross-namespace queries and document-scoped relation removal. +- **`query.rs`** — hybrid retrieval. Combines graph relevance, vector similarity, keyword overlap, episodic signal and freshness; exposes `query_namespace_*` (with query) and `recall_namespace_*` (query-less) entry points used by `MemoryClient`. +- **`helpers.rs`** — shared utilities: f32-vector byte codecs, cosine similarity, markdown chunking, text/graph normalisation, JSON attribute merging, recency scoring. +- **`fts5.rs`** — FTS5 episodic memory (`episodic_log` + `episodic_fts`). `EpisodicEntry` plus `episodic_insert` / `episodic_search` / `episodic_session_entries` for the Archivist and `search_memory` tool. +- **`segments.rs`** — conversation segmentation (`conversation_segments`). Boundary detection (time gap, embedding drift, explicit markers, turn count), segment lifecycle (open → closed → summarised), and the `BoundaryConfig` knobs. +- **`events.rs`** — event extraction (`event_log` + `event_fts`). Stores typed atomic events (Fact / Decision / Commitment / Preference / Question / Foresight) extracted from closed segments via heuristic pattern matching. +- **`profile.rs`** — user profile facets (`user_profile`). Evidence-backed `FacetType` rows that accumulate across sessions; on conflict, evidence count is bumped and the value is overwritten only if confidence improves. +- **`*_tests.rs`** — module-local tests for documents, events, profile, query, segments. + +## How it fits + +`MemoryClient` (in `../client.rs`) and the `impl Memory for UnifiedMemory` in `../memory_trait.rs` are the only things that should hold a `UnifiedMemory` directly. The ingestion pipeline (`../../ingestion/`) calls `upsert_document` and `graph_upsert_namespace` after parsing; the agent harness reads via `query_namespace_*` and `recall_namespace_*`; the Archivist writes episodic turns via `fts5::episodic_insert` and segments / events / profile facets via the dedicated submodules. diff --git a/src/openhuman/memory/store/unified/documents.rs b/src/openhuman/memory/store/unified/documents.rs index e459442a1..fbbe01e70 100644 --- a/src/openhuman/memory/store/unified/documents.rs +++ b/src/openhuman/memory/store/unified/documents.rs @@ -1,3 +1,9 @@ +//! Document CRUD against the `memory_docs` table. +//! +//! Owns the upsert pipeline (with chunking + embedding), metadata-only writes +//! for high-frequency callers, list/delete/clear-namespace operations, and the +//! markdown sidecar files in `memory/namespaces//docs/`. + use rusqlite::{params, OptionalExtension}; use serde_json::{json, Value}; use std::collections::BTreeSet; @@ -8,6 +14,9 @@ use crate::openhuman::memory::store::types::{NamespaceDocumentInput, StoredMemor use super::UnifiedMemory; impl UnifiedMemory { + /// Insert or update a document by `(namespace, key)`. Writes the markdown + /// sidecar, replaces vector chunks, and embeds them with the configured + /// provider. pub async fn upsert_document(&self, input: NamespaceDocumentInput) -> Result { let namespace = Self::sanitize_namespace(&input.namespace); let key = input.key.trim().to_string(); @@ -57,6 +66,7 @@ impl UnifiedMemory { updated_at, &input.content, ) + .await .map_err(|e| e.to_string())?; let tags_json = serde_json::to_string(&input.tags).map_err(|e| e.to_string())?; @@ -196,6 +206,7 @@ impl UnifiedMemory { updated_at, &input.content, ) + .await .map_err(|e| e.to_string())?; let tags_json = serde_json::to_string(&input.tags).map_err(|e| e.to_string())?; @@ -300,6 +311,8 @@ impl UnifiedMemory { Ok(docs) } + /// List documents in a namespace, or across all namespaces when `None`. + /// Returns `{ "documents": [...], "count": N }` JSON. pub async fn list_documents(&self, namespace: Option<&str>) -> Result { let conn = self.conn.lock(); let mut docs = Vec::new(); @@ -357,6 +370,7 @@ impl UnifiedMemory { Ok(json!({ "documents": docs, "count": docs.len() })) } + /// Return every distinct namespace that has at least one document. pub async fn list_namespaces(&self) -> Result, String> { let conn = self.conn.lock(); let mut stmt = conn @@ -432,7 +446,7 @@ impl UnifiedMemory { // Remove on-disk markdown files for this namespace. let docs_dir = self.namespace_dir(&ns).join("docs"); if docs_dir.exists() { - std::fs::remove_dir_all(&docs_dir).map_err(|e| { + tokio::fs::remove_dir_all(&docs_dir).await.map_err(|e| { format!( "clear_namespace remove docs dir {}: {e}", docs_dir.display() @@ -448,6 +462,8 @@ impl UnifiedMemory { Ok(()) } + /// Delete a single document plus its vector chunks, graph relations, and + /// markdown sidecar. Returns `{ "deleted": bool, "namespace", "documentId" }`. pub async fn delete_document( &self, namespace: &str, @@ -487,7 +503,13 @@ impl UnifiedMemory { if let Some(rel) = rel_path { let abs = self.workspace_dir.join(rel); - let _ = std::fs::remove_file(abs); + // Surface non-NotFound failures so storage drift between the DB + // row and the markdown sidecar is diagnosable. + if let Err(e) = tokio::fs::remove_file(&abs).await { + if e.kind() != std::io::ErrorKind::NotFound { + log::warn!("[memory] failed to remove sidecar {}: {e}", abs.display()); + } + } } Ok(json!({"deleted": deleted, "namespace": ns, "documentId": document_id })) } diff --git a/src/openhuman/memory/store/unified/documents_tests.rs b/src/openhuman/memory/store/unified/documents_tests.rs index 23fe25682..6a6b8d630 100644 --- a/src/openhuman/memory/store/unified/documents_tests.rs +++ b/src/openhuman/memory/store/unified/documents_tests.rs @@ -1,9 +1,12 @@ +//! Tests for the `documents` module — upsert / list / delete / clear-namespace. + use std::sync::Arc; use serde_json::json; use tempfile::TempDir; -use crate::openhuman::memory::{embeddings::NoopEmbedding, NamespaceDocumentInput, UnifiedMemory}; +use crate::openhuman::embeddings::NoopEmbedding; +use crate::openhuman::memory::{NamespaceDocumentInput, UnifiedMemory}; fn make_doc_input( namespace: &str, diff --git a/src/openhuman/memory/store/unified/events.rs b/src/openhuman/memory/store/unified/events.rs index 77f00d5f5..46c294c68 100644 --- a/src/openhuman/memory/store/unified/events.rs +++ b/src/openhuman/memory/store/unified/events.rs @@ -77,6 +77,7 @@ pub enum EventType { } impl EventType { + /// Stable lowercase identifier persisted in the `event_log` table. pub fn as_str(&self) -> &'static str { match self { Self::Fact => "fact", @@ -88,6 +89,8 @@ impl EventType { } } + /// Parse a stored string back to an `EventType`; unknown values fall back + /// to `Fact`. pub fn parse_or_default(s: &str) -> Self { match s { "decision" => Self::Decision, diff --git a/src/openhuman/memory/store/unified/events_tests.rs b/src/openhuman/memory/store/unified/events_tests.rs index 20c951e33..ce700b3b0 100644 --- a/src/openhuman/memory/store/unified/events_tests.rs +++ b/src/openhuman/memory/store/unified/events_tests.rs @@ -1,3 +1,5 @@ +//! Tests for the `events` module — heuristic extraction and FTS5 storage. + use super::*; fn setup_db() -> Arc> { diff --git a/src/openhuman/memory/store/unified/graph.rs b/src/openhuman/memory/store/unified/graph.rs index 014a15e4d..ed4737cf4 100644 --- a/src/openhuman/memory/store/unified/graph.rs +++ b/src/openhuman/memory/store/unified/graph.rs @@ -1,3 +1,9 @@ +//! Knowledge-graph relations stored in `graph_namespace` and `graph_global`. +//! +//! Provides upsert (with attribute merging + evidence accumulation), namespace +//! / global / cross-namespace queries, and the document-scoped removal used +//! when a source document is deleted or re-ingested. + use rusqlite::{params, OptionalExtension}; use serde_json::{json, Map, Value}; @@ -94,6 +100,7 @@ impl UnifiedMemory { Ok(()) } + /// Upsert a relation into the cross-namespace `graph_global` table. pub async fn graph_upsert_global( &self, subject: &str, @@ -105,6 +112,9 @@ impl UnifiedMemory { .await } + /// Upsert a relation into the namespace-scoped `graph_namespace` table, + /// merging attributes (evidence count, document/chunk ids) with any + /// existing edge. pub async fn graph_upsert_namespace( &self, namespace: &str, @@ -117,6 +127,7 @@ impl UnifiedMemory { .await } + /// Query relations in the global graph with optional subject/predicate filters. pub async fn graph_query_global( &self, subject: Option<&str>, @@ -153,6 +164,7 @@ impl UnifiedMemory { .collect::>()) } + /// Query relations within a single namespace with optional subject/predicate filters. pub async fn graph_query_namespace( &self, namespace: &str, diff --git a/src/openhuman/memory/store/unified/helpers.rs b/src/openhuman/memory/store/unified/helpers.rs index 3e219b932..0f4cbd813 100644 --- a/src/openhuman/memory/store/unified/helpers.rs +++ b/src/openhuman/memory/store/unified/helpers.rs @@ -1,10 +1,14 @@ +//! Shared helpers used across the unified store: byte/float vector codecs, +//! cosine similarity, markdown chunking, text/predicate normalization, JSON +//! attribute merging, and recency scoring. + use crate::openhuman::memory::chunker::chunk_markdown; use super::UnifiedMemory; impl UnifiedMemory { #[allow(clippy::too_many_arguments)] - pub(crate) fn write_markdown_doc( + pub(crate) async fn write_markdown_doc( &self, namespace: &str, doc_id: &str, @@ -17,7 +21,7 @@ impl UnifiedMemory { content: &str, ) -> anyhow::Result { let docs_dir = self.namespace_dir(namespace).join("docs"); - std::fs::create_dir_all(&docs_dir)?; + tokio::fs::create_dir_all(&docs_dir).await?; let rel_path = format!( "memory/namespaces/{}/docs/{doc_id}.md", Self::sanitize_namespace(namespace) @@ -34,7 +38,7 @@ impl UnifiedMemory { created_at, updated_at ); - std::fs::write(abs_path, format!("{header}{content}\n"))?; + tokio::fs::write(abs_path, format!("{header}{content}\n")).await?; Ok(rel_path) } diff --git a/src/openhuman/memory/store/unified/init.rs b/src/openhuman/memory/store/unified/init.rs index 0d2de2986..d25bf9bec 100644 --- a/src/openhuman/memory/store/unified/init.rs +++ b/src/openhuman/memory/store/unified/init.rs @@ -1,15 +1,28 @@ +//! `UnifiedMemory` constructor + schema bootstrap. +//! +//! Creates the workspace directories, opens the SQLite connection in WAL mode, +//! materialises every table the unified store owns (docs, kv, graph, vector +//! chunks, episodic FTS5, segments, events, profile), and runs idempotent +//! legacy-namespace migrations. Also exposes path / namespace helpers shared +//! by the rest of the unified module. + use std::path::{Path, PathBuf}; use std::sync::Arc; use parking_lot::Mutex; use rusqlite::Connection; -use crate::openhuman::memory::embeddings::EmbeddingProvider; +use crate::openhuman::embeddings::EmbeddingProvider; use crate::openhuman::memory::store::types::GLOBAL_NAMESPACE; use super::UnifiedMemory; impl UnifiedMemory { + /// Open (or create) the unified store rooted at `workspace_dir`. + /// + /// Creates the on-disk layout, runs all `CREATE TABLE` statements, and + /// applies idempotent legacy-namespace migrations. Safe to call on every + /// boot. pub fn new( workspace_dir: &Path, embedder: Arc, @@ -179,14 +192,17 @@ impl UnifiedMemory { }) } + /// Root workspace directory holding `memory/` and its subtrees. pub fn workspace_dir(&self) -> &Path { &self.workspace_dir } + /// Filesystem path of the SQLite database file. pub fn db_path(&self) -> &Path { &self.db_path } + /// Directory used for vector-related sidecar files. pub fn vectors_dir(&self) -> &Path { &self.vectors_dir } diff --git a/src/openhuman/memory/store/unified/kv.rs b/src/openhuman/memory/store/unified/kv.rs index 72f365475..381f5359e 100644 --- a/src/openhuman/memory/store/unified/kv.rs +++ b/src/openhuman/memory/store/unified/kv.rs @@ -1,3 +1,8 @@ +//! Key-value storage backed by the `kv_global` and `kv_namespace` tables. +//! +//! Provides global and namespace-scoped get/set/delete/list, plus internal +//! record loaders used by the retrieval pipeline. + use rusqlite::{params, OptionalExtension}; use serde_json::json; @@ -6,6 +11,7 @@ use crate::openhuman::memory::store::types::MemoryKvRecord; use super::UnifiedMemory; impl UnifiedMemory { + /// Insert or update a global key-value pair. pub async fn kv_set_global(&self, key: &str, value: &serde_json::Value) -> Result<(), String> { let conn = self.conn.lock(); conn.execute( @@ -18,6 +24,7 @@ impl UnifiedMemory { Ok(()) } + /// Read a global key, returning `None` if absent. pub async fn kv_get_global(&self, key: &str) -> Result, String> { let conn = self.conn.lock(); let value: Option = conn @@ -31,6 +38,7 @@ impl UnifiedMemory { Ok(value.and_then(|v| serde_json::from_str(&v).ok())) } + /// Insert or update a namespace-scoped key-value pair. pub async fn kv_set_namespace( &self, namespace: &str, @@ -48,6 +56,7 @@ impl UnifiedMemory { Ok(()) } + /// Read a namespace-scoped key, returning `None` if absent. pub async fn kv_get_namespace( &self, namespace: &str, @@ -65,6 +74,7 @@ impl UnifiedMemory { Ok(value.and_then(|v| serde_json::from_str(&v).ok())) } + /// Delete a global key. Returns `true` if a row was removed. pub async fn kv_delete_global(&self, key: &str) -> Result { let conn = self.conn.lock(); let changed = conn @@ -73,6 +83,7 @@ impl UnifiedMemory { Ok(changed > 0) } + /// Delete a namespace-scoped key. Returns `true` if a row was removed. pub async fn kv_delete_namespace(&self, namespace: &str, key: &str) -> Result { let conn = self.conn.lock(); let changed = conn @@ -84,6 +95,7 @@ impl UnifiedMemory { Ok(changed > 0) } + /// List all keys in a namespace, most recently updated first. pub async fn kv_list_namespace( &self, namespace: &str, diff --git a/src/openhuman/memory/store/unified/mod.rs b/src/openhuman/memory/store/unified/mod.rs index 6f99f00a9..5d292c03f 100644 --- a/src/openhuman/memory/store/unified/mod.rs +++ b/src/openhuman/memory/store/unified/mod.rs @@ -5,8 +5,13 @@ use rusqlite::Connection; use std::path::PathBuf; use std::sync::Arc; -use crate::openhuman::memory::embeddings::EmbeddingProvider; +use crate::openhuman::embeddings::EmbeddingProvider; +/// SQLite-backed unified memory store. +/// +/// Owns a single connection (WAL-mode) plus the on-disk markdown sidecar +/// directory and vector storage path. Methods are added across the sibling +/// modules (`documents`, `kv`, `graph`, `query`, …) via `impl` blocks. pub struct UnifiedMemory { pub(crate) workspace_dir: PathBuf, pub(crate) db_path: PathBuf, diff --git a/src/openhuman/memory/store/unified/profile.rs b/src/openhuman/memory/store/unified/profile.rs index ac35e6e05..bb3dbae4b 100644 --- a/src/openhuman/memory/store/unified/profile.rs +++ b/src/openhuman/memory/store/unified/profile.rs @@ -43,6 +43,7 @@ pub enum FacetType { } impl FacetType { + /// Stable lowercase identifier persisted in the `user_profile` table. pub fn as_str(&self) -> &'static str { match self { Self::Preference => "preference", @@ -53,6 +54,8 @@ impl FacetType { } } + /// Parse a stored string back to a `FacetType`; unknown values fall back + /// to `Preference`. pub fn parse_or_default(s: &str) -> Self { match s { "skill" => Self::Skill, diff --git a/src/openhuman/memory/store/unified/profile_tests.rs b/src/openhuman/memory/store/unified/profile_tests.rs index 3060a7804..6583f6327 100644 --- a/src/openhuman/memory/store/unified/profile_tests.rs +++ b/src/openhuman/memory/store/unified/profile_tests.rs @@ -1,3 +1,5 @@ +//! Tests for the `profile` module — facet upsert with confidence merging. + use super::*; fn setup_db() -> Arc> { diff --git a/src/openhuman/memory/store/unified/query.rs b/src/openhuman/memory/store/unified/query.rs index 2cb126f7b..2217d2891 100644 --- a/src/openhuman/memory/store/unified/query.rs +++ b/src/openhuman/memory/store/unified/query.rs @@ -1,3 +1,11 @@ +//! Hybrid retrieval over the unified store. +//! +//! Combines graph relevance, vector similarity, keyword overlap, episodic +//! signal, and freshness into a single score per hit. Owns the query planner +//! (`build_retrieval_plan`), per-document score composition, and the +//! `query_namespace_hits` / `query_namespace_ranked` / `recall_namespace_*` +//! entry points used by `MemoryClient`. + use rusqlite::params; use std::collections::{HashMap, HashSet}; @@ -85,6 +93,9 @@ impl UnifiedMemory { Ok(out) } + /// Hybrid retrieval: returns ranked hits across documents and KV records, + /// scored by graph relevance + vector similarity + keyword overlap + + /// freshness. pub async fn query_namespace_hits( &self, namespace: &str, @@ -322,6 +333,7 @@ impl UnifiedMemory { Ok(hits) } + /// Run a hybrid query and return only the rendered context text. pub async fn query_namespace_context( &self, namespace: &str, @@ -334,6 +346,8 @@ impl UnifiedMemory { Ok(context.context_text) } + /// Run a hybrid query and return both the rendered context text and the + /// underlying ranked hits. pub async fn query_namespace_context_data( &self, namespace: &str, @@ -350,6 +364,8 @@ impl UnifiedMemory { }) } + /// Query-less recall: rank documents and KV records by priority + graph + /// relevance + freshness without a search query. pub async fn recall_namespace_memories( &self, namespace: &str, @@ -454,6 +470,8 @@ impl UnifiedMemory { Ok(hits) } + /// Query-less recall returning only rendered context text. `None` when + /// the namespace is empty. pub async fn recall_namespace_context( &self, namespace: &str, @@ -468,6 +486,7 @@ impl UnifiedMemory { Ok(Some(Self::format_context_text(&hits, None))) } + /// Query-less recall returning both rendered text and ranked hits. pub async fn recall_namespace_context_data( &self, namespace: &str, diff --git a/src/openhuman/memory/store/unified/query_tests.rs b/src/openhuman/memory/store/unified/query_tests.rs index 1f4c5958e..a5ff3b88c 100644 --- a/src/openhuman/memory/store/unified/query_tests.rs +++ b/src/openhuman/memory/store/unified/query_tests.rs @@ -1,9 +1,12 @@ +//! Tests for the `query` module — hybrid retrieval scoring. + use std::sync::Arc; use serde_json::json; use tempfile::TempDir; -use crate::openhuman::memory::{embeddings::NoopEmbedding, NamespaceDocumentInput, UnifiedMemory}; +use crate::openhuman::embeddings::NoopEmbedding; +use crate::openhuman::memory::{NamespaceDocumentInput, UnifiedMemory}; #[tokio::test] async fn graph_duplicate_upsert_aggregates_evidence_count() { diff --git a/src/openhuman/memory/store/unified/segments.rs b/src/openhuman/memory/store/unified/segments.rs index 08b5cbb4f..4ff9fc332 100644 --- a/src/openhuman/memory/store/unified/segments.rs +++ b/src/openhuman/memory/store/unified/segments.rs @@ -49,6 +49,7 @@ pub enum SegmentStatus { } impl SegmentStatus { + /// Stable lowercase identifier persisted in the `conversation_segments` table. pub fn as_str(&self) -> &'static str { match self { Self::Open => "open", @@ -57,6 +58,8 @@ impl SegmentStatus { } } + /// Parse a stored string back to a `SegmentStatus`; unknown values fall + /// back to `Open`. pub fn parse_or_default(s: &str) -> Self { match s { "closed" => Self::Closed, @@ -116,6 +119,7 @@ pub enum BoundaryDecision { Boundary(BoundaryReason), } +/// Reason a new segment boundary was triggered. #[derive(Debug, Clone)] pub enum BoundaryReason { TimeGap, diff --git a/src/openhuman/memory/store/unified/segments_tests.rs b/src/openhuman/memory/store/unified/segments_tests.rs index ebe619813..abd356811 100644 --- a/src/openhuman/memory/store/unified/segments_tests.rs +++ b/src/openhuman/memory/store/unified/segments_tests.rs @@ -1,3 +1,5 @@ +//! Tests for the `segments` module — boundary detection and segment lifecycle. + use super::*; fn setup_db() -> Arc> { diff --git a/src/openhuman/memory/tree/README.md b/src/openhuman/memory/tree/README.md new file mode 100644 index 000000000..e619c9903 --- /dev/null +++ b/src/openhuman/memory/tree/README.md @@ -0,0 +1,57 @@ +# Memory tree + +Bucket-seal-ready local memory architecture (Phase 1 of issue #707; the LLD design doc `docs/MEMORY_ARCHITECTURE_LLD.md` is referenced by the in-tree module headers but is not checked into this repo). Coexists with the legacy `store/` backend until full replacement. + +## Pipeline + +```text +source adapters (chat / email / document) + │ + ▼ +canonicalize/ ── normalised Markdown + provenance Metadata + │ + ▼ +chunker.rs ── deterministic IDs, ≤3k-token bounded segments + │ + ▼ +content_store/── atomic .md files on disk (body + tags) + │ + ▼ +store.rs ── SQLite persistence (chunks, scores, summaries, jobs, hotness) + │ + ▼ +score/ ── signals + embeddings + entity extraction + │ + ▼ +tree_source/ tree_topic/ tree_global/ ── per-scope summary trees + │ + ▼ +retrieval/ ── search / drill_down / topic / global / fetch + │ + ▼ +jobs/ ── background workers + scheduler (extract, admit, seal, digest) +``` + +## Files at this level + +- [`mod.rs`](mod.rs) — Phase 1 module banner; re-exports controller registries (`all_memory_tree_*`, `all_retrieval_*`). +- [`chunker.rs`](chunker.rs) — slice canonical Markdown into ≤`DEFAULT_CHUNK_MAX_TOKENS` chunks; chat/email split at message boundaries, document at paragraphs. +- [`ingest.rs`](ingest.rs) — orchestrator: `canonicalize -> chunk -> stage_chunks -> fast score -> persist -> enqueue extract jobs`. Hot path; heavy work runs out of `jobs/`. +- [`rpc.rs`](rpc.rs) — JSON-RPC handlers for `memory_tree_ingest`, `list_chunks`, `get_chunk`, `trigger_digest`. Delegates to `ingest`/`store`/`jobs`. +- [`schemas.rs`](schemas.rs) — `ControllerSchema` definitions + `RegisteredController` wiring for the four `memory_tree_*` RPC methods. +- [`store.rs`](store.rs) — SQLite schema (chunks, score, entity index, trees, summaries, buffers, hotness, jobs) and accessors. Lazily initialised at `/memory_tree/chunks.db`. +- [`store_tests.rs`](store_tests.rs) — store-layer unit tests. +- [`types.rs`](types.rs) — `Chunk`, `Metadata`, `SourceKind`, `DataSource`, `SourceRef`; deterministic `chunk_id` hash; `approx_token_count` heuristic. + +## Subdirectories + +- [`canonicalize/`](canonicalize/README.md) — chat / email / document → canonical Markdown + email body cleaner. +- [`chunker.rs`](chunker.rs) — see above. +- [`content_store/`](content_store/README.md) — on-disk `.md` files (atomic writes, paths, YAML compose, read+verify, tag rewrites). +- [`jobs/`](jobs/) — async job queue (extract / admit / seal / topic / digest workers). +- [`retrieval/`](retrieval/) — search and drill-down RPC surface. +- [`score/`](score/) — fast scorer, embeddings, entity extraction, score persistence. +- [`tree_source/`](tree_source/) — per-source summary trees (L0 buffer → L1 seal → cascade). +- [`tree_topic/`](tree_topic/) — per-entity topic trees, materialised lazily by hotness. +- [`tree_global/`](tree_global/) — daily global digest tree. +- [`util/`](util/README.md) — shared helpers (`redact` for log PII). diff --git a/src/openhuman/memory/tree/canonicalize/README.md b/src/openhuman/memory/tree/canonicalize/README.md new file mode 100644 index 000000000..9ed564430 --- /dev/null +++ b/src/openhuman/memory/tree/canonicalize/README.md @@ -0,0 +1,17 @@ +# canonicalize/ + +Source-specific adapters that normalise upstream payloads (chat batches, email threads, documents) into a single shape — `CanonicalisedSource { markdown, metadata }` — that the chunker downstream slices into bounded chunks. + +Adapters do not interpret content semantically; they only normalise shape and capture provenance. Scoring / extraction / summarisation happen later in the pipeline. + +## Files + +- [`mod.rs`](mod.rs) — `CanonicalisedSource` struct, generic `CanonicaliseRequest

` envelope, and `normalize_source_ref` helper shared by all adapters. +- [`chat.rs`](chat.rs) — chat transcripts (Slack / Discord / Telegram / WhatsApp) → Markdown of `## — \n` blocks. Sorts messages and captures `time_range`. Produces empty-input `Ok(None)`. +- [`document.rs`](document.rs) — single documents (Notion page, Drive doc, meeting note, uploaded file) → trimmed body Markdown. `time_range` collapses to a single point at `modified_at`. +- [`email.rs`](email.rs) — email threads (Gmail + generic) → per-message `---\nFrom: …\nSubject: …\nDate: …\n\n` blocks. Bodies pass through `email_clean::clean_body` first. +- [`email_clean.rs`](email_clean.rs) — pure-string helpers: `clean_body` (strip reply chains + footer/legal boilerplate), `truncate_body`, `md_escape`, `extract_email`, `parse_message_date`. Used by both the email canonicaliser and the `gmail-fetch-emails` bin. + +## Output contract + +The canonicalised Markdown carries no leading `# Header` line — provider/title metadata lives in YAML front-matter written by `content_store/compose.rs`. The chunker relies on the `##` prefix followed by a space (chat) and `---\nFrom:` (email) boundaries to split at message granularity. diff --git a/src/openhuman/memory/tree/canonicalize/document.rs b/src/openhuman/memory/tree/canonicalize/document.rs index a3e171dd4..da39d454f 100644 --- a/src/openhuman/memory/tree/canonicalize/document.rs +++ b/src/openhuman/memory/tree/canonicalize/document.rs @@ -28,6 +28,9 @@ pub struct DocumentInput { pub source_ref: Option, } +/// Canonicalise a single document into a [`CanonicalisedSource`]. Returns +/// `Ok(None)` if both the title and body are empty — caller treats as nothing +/// to ingest. pub fn canonicalise( source_id: &str, owner: &str, diff --git a/src/openhuman/memory/tree/canonicalize/email.rs b/src/openhuman/memory/tree/canonicalize/email.rs index 925026ac7..c764242d7 100644 --- a/src/openhuman/memory/tree/canonicalize/email.rs +++ b/src/openhuman/memory/tree/canonicalize/email.rs @@ -42,6 +42,9 @@ pub struct EmailThread { pub messages: Vec, } +/// Canonicalise an email thread into a [`CanonicalisedSource`]. Bodies are +/// passed through [`email_clean::clean_body`] to strip reply chains and footer +/// boilerplate. Returns `Ok(None)` when the thread has no messages. pub fn canonicalise( source_id: &str, owner: &str, diff --git a/src/openhuman/memory/tree/canonicalize/mod.rs b/src/openhuman/memory/tree/canonicalize/mod.rs index 168fafaec..71f476138 100644 --- a/src/openhuman/memory/tree/canonicalize/mod.rs +++ b/src/openhuman/memory/tree/canonicalize/mod.rs @@ -22,7 +22,9 @@ use crate::openhuman::memory::tree::types::{Metadata, SourceRef}; /// (a chat batch, an email, a document). #[derive(Clone, Debug)] pub struct CanonicalisedSource { + /// Canonical Markdown blob produced by the adapter. pub markdown: String, + /// Provenance the chunker will clone onto each emitted [`Chunk`]. pub metadata: Metadata, } diff --git a/src/openhuman/memory/tree/chunker.rs b/src/openhuman/memory/tree/chunker.rs index 62ffc04d0..3dfe8a248 100644 --- a/src/openhuman/memory/tree/chunker.rs +++ b/src/openhuman/memory/tree/chunker.rs @@ -20,7 +20,7 @@ use crate::openhuman::memory::tree::util::redact::redact; /// Default upper bound on per-chunk tokens. /// -/// Sized below the L0 seal budget (`source_tree::types::TOKEN_BUDGET = 4_500`) +/// Sized below the L0 seal budget (`tree_source::types::TOKEN_BUDGET = 4_500`) /// so each seal accumulates roughly 1–3 chunks before firing — natural pacing /// for the local 1B summariser, which produces noticeably better summaries /// with smaller (≤4–5k) inputs than at the previous 10k cap. diff --git a/src/openhuman/memory/tree/content_store/README.md b/src/openhuman/memory/tree/content_store/README.md new file mode 100644 index 000000000..0aba3d865 --- /dev/null +++ b/src/openhuman/memory/tree/content_store/README.md @@ -0,0 +1,18 @@ +# content_store/ + +On-disk `.md` storage for chunk and summary bodies (Phase MD-content). SQLite holds `content_path` (relative, forward-slash) and `content_sha256` (over body bytes only) as pointers + integrity tokens; the body itself lives at `/`. + +The body is **immutable** once written — only the YAML front-matter `tags:` block may be rewritten post-extraction. + +## Files + +- [`mod.rs`](mod.rs) — public surface: `StagedChunk`, `stage_chunks` (write all chunks atomically before SQLite upsert), `update_summary_tags` re-export. +- [`atomic.rs`](atomic.rs) — `write_if_new` (tempfile + fsync + rename, parent dir fsync on Unix), `stage_summary` (idempotent re-stage with on-disk SHA check + auto-rewrite on mismatch), `sha256_hex`, `StagedSummary`. +- [`compose.rs`](compose.rs) — YAML front-matter + body composition. `compose_chunk_file` for chunks (with email-only `participants:` / `aliases:` fields parsed from `gmail:{addr1|addr2|…}` source ids), `compose_summary_md` for summary nodes. `rewrite_tags` / `rewrite_summary_tags` swap the `tags:` block in place. `split_front_matter` parses `---\n…\n---\n`. +- [`paths.rs`](paths.rs) — path generators. `chunk_rel_path` (`email//.md`, `chat//.md`, `document//.md`); `summary_rel_path` (`summaries/{source,global,topic}/…`). `slugify_source_id` is the canonical filesystem-safe slug. +- [`read.rs`](read.rs) — `read_chunk_file` / `read_summary_file` parse front-matter and return body+SHA. `verify_*` compares against an expected SHA. `read_chunk_body` / `read_summary_body` resolve the path via SQLite and verify the integrity hash; this is the authoritative entry-point for callers that need the **full** body (LLM extractor, summariser, embedder, retrieval API). +- [`tags.rs`](tags.rs) — post-extraction tag rewrites. `update_chunk_tags` (atomic tempfile rewrite of the `tags:` block) and `update_summary_tags` (fetches entities from `mem_tree_entity_index`, builds Obsidian `kind/Value` tags, rewrites, verifies body SHA is unchanged). `slugify_tag_kind`, `slugify_tag_value`, `entity_tag` build the tag strings. + +## Integrity contract + +The body bytes never change after the first write. The SHA-256 stored in SQLite is computed over body bytes only — front-matter (including `tags:`) can be rewritten without invalidating the hash. Read paths verify SHA on every fetch and fail loudly on mismatch rather than serve corrupt data into the extractor or summariser. diff --git a/src/openhuman/memory/tree/content_store/atomic.rs b/src/openhuman/memory/tree/content_store/atomic.rs index ef1798bb9..7234b895f 100644 --- a/src/openhuman/memory/tree/content_store/atomic.rs +++ b/src/openhuman/memory/tree/content_store/atomic.rs @@ -100,6 +100,7 @@ pub fn write_if_new(abs_path: &Path, bytes: &[u8]) -> anyhow::Result { /// A summary that has been written to disk and is ready for SQLite upsert. #[derive(Debug, Clone)] pub struct StagedSummary { + /// Identifier of the summary that was staged. pub summary_id: String, /// Relative content path (forward-slash, e.g. `"summaries/source/slug/L1/id.md"`). pub content_path: String, diff --git a/src/openhuman/memory/tree/content_store/compose.rs b/src/openhuman/memory/tree/content_store/compose.rs index 75e98969f..cc8b32087 100644 --- a/src/openhuman/memory/tree/content_store/compose.rs +++ b/src/openhuman/memory/tree/content_store/compose.rs @@ -238,24 +238,33 @@ fn replace_tags_in_front_matter(fm: &str, new_tags: &[String]) -> Result { + /// Stable id of the summary node (also used to derive the filename). pub summary_id: &'a str, + /// Which tree (source / global / topic) this summary belongs to. pub tree_kind: SummaryTreeKind, + /// Owning tree id (FK into `mem_tree_trees`). pub tree_id: &'a str, /// Raw tree scope string, e.g. `"gmail:alice@x.com|bob@y.com"` or `"global"`. pub tree_scope: &'a str, + /// Level in the tree (L0 = leaves, L1+ = summaries). pub level: u32, /// Child ids (chunk_ids at L0 → L1, summary_ids for cascades). pub child_ids: &'a [String], /// Total child count (== child_ids.len() unless truncated). pub child_count: usize, + /// Start of the time range covered by this summary's children. pub time_range_start: DateTime, + /// End of the time range covered by this summary's children. pub time_range_end: DateTime, + /// When the buffer was sealed into this summary node. pub sealed_at: DateTime, /// Raw summariser output text — the body written to disk. pub body: &'a str, } /// The composed front-matter, body, and full file content for a summary. +/// +/// `body` is what the SHA-256 integrity hash is computed over. pub struct ComposedSummary { /// The YAML front-matter block (including `---` delimiters), UTF-8 string. pub front_matter: String, diff --git a/src/openhuman/memory/tree/ingest.rs b/src/openhuman/memory/tree/ingest.rs index 07b3c54bb..fb2054ccc 100644 --- a/src/openhuman/memory/tree/ingest.rs +++ b/src/openhuman/memory/tree/ingest.rs @@ -46,6 +46,8 @@ impl IngestResult { } } +/// Ingest a batch of chat messages: canonicalise → chunk → fast-score → persist +/// → enqueue async extract jobs. Returns a noop [`IngestResult`] on an empty batch. pub async fn ingest_chat( config: &Config, source_id: &str, @@ -61,6 +63,8 @@ pub async fn ingest_chat( persist(config, source_id, canonical).await } +/// Ingest an email thread: canonicalise → chunk → fast-score → persist → enqueue +/// async extract jobs. Returns a noop [`IngestResult`] on an empty thread. pub async fn ingest_email( config: &Config, source_id: &str, @@ -76,6 +80,8 @@ pub async fn ingest_email( persist(config, source_id, canonical).await } +/// Ingest a single document: canonicalise → chunk → fast-score → persist → +/// enqueue async extract jobs. Returns a noop [`IngestResult`] on empty input. pub async fn ingest_document( config: &Config, source_id: &str, diff --git a/src/openhuman/memory/tree/jobs/README.md b/src/openhuman/memory/tree/jobs/README.md new file mode 100644 index 000000000..08a549493 --- /dev/null +++ b/src/openhuman/memory/tree/jobs/README.md @@ -0,0 +1,36 @@ +# Memory tree — jobs + +Async job pipeline driving extraction, scoring, summarisation, and digesting off the ingest hot path. Replaces the previous synchronous `append_leaf → cascade_seal → LLM summarise` chain with a SQLite-backed queue (`mem_tree_jobs`) and a worker pool. Producers commit side-effect + follow-up job atomically inside one transaction via `enqueue_tx`. + +## Pipeline shape + +```text +ingest::persist → enqueues `extract_chunk` +worker pool (3 tasks): + extract_chunk → LLM extraction → admission → enqueue `append_buffer` + `topic_route` + append_buffer → push to L0 → enqueue `seal` if gate met + seal → seal one level → enqueue parent seal if cascading + topic_route → match topics → enqueue per-topic `append_buffer` + digest_daily → call `tree_global::digest::end_of_day_digest` + flush_stale → enqueue seals for time-stale buffers +scheduler (1 task) → daily wall-clock tick → `digest_daily(yesterday)` + `flush_stale(today)` +``` + +## Public surface + +- `pub fn enqueue` / `enqueue_tx` / `claim_next` / `mark_done` / `mark_failed` / `recover_stale_locks` / `get_job` / `count_by_status` / `count_total` — `store.rs` — queue persistence. +- `pub fn start` / `wake_workers` — `worker.rs` — spawn the worker pool (idempotent) and notify idle workers. +- `pub fn trigger_digest` / `backfill_missing_digests` — `scheduler.rs` — manual digest enqueues. +- `pub fn drain_until_idle` — `testing.rs` — deterministic test runner that processes all eligible jobs. +- `pub enum JobKind` / `JobStatus` / `pub struct Job` / `NewJob` / payload structs (`ExtractChunkPayload`, `AppendBufferPayload`, `SealPayload`, `TopicRoutePayload`, `DigestDailyPayload`, `FlushStalePayload`) and `NodeRef` / `AppendTarget` — `types.rs`. +- `pub const DEFAULT_LOCK_DURATION_MS` — `store.rs` — claim lease window (5 min). + +## Files + +- `mod.rs` — module surface and re-exports. +- `types.rs` — `JobKind`, `JobStatus`, payload structs, `NewJob` builders. Each payload owns its `dedupe_key()` so duplicates in flight are silently suppressed. +- `store.rs` — SQLite persistence: `INSERT OR IGNORE` + partial unique index on `dedupe_key WHERE status IN ('ready','running')` for at-most-one-active dedupe; `claim_next` is a single `UPDATE ... RETURNING`; `mark_done`/`mark_failed` are claim-token gated to make stale-worker settlements no-ops. +- `worker.rs` — three worker tasks plus startup `recover_stale_locks` and a 3-permit semaphore around LLM-bound jobs. Calls into `crate::openhuman::scheduler_gate::wait_for_capacity()` before claiming so Throttled / Paused modes back off without holding DB leases. +- `scheduler.rs` — daily tick at UTC 00:05 that enqueues `digest_daily(yesterday)` + `flush_stale(today)`; `trigger_digest` and `backfill_missing_digests` are manual catch-up helpers. +- `handlers/` — per-`JobKind` handler implementations. +- `testing.rs` — `drain_until_idle` for tests that need the pipeline to settle synchronously. diff --git a/src/openhuman/memory/tree/jobs/handlers/README.md b/src/openhuman/memory/tree/jobs/handlers/README.md new file mode 100644 index 000000000..8f38bcdcc --- /dev/null +++ b/src/openhuman/memory/tree/jobs/handlers/README.md @@ -0,0 +1,20 @@ +# Memory tree — jobs handlers + +Per-`JobKind` handler implementations dispatched by `worker::run_once_with_semaphore`. Each handler parses its payload, performs side effects, and enqueues any follow-up work (typically inside the same SQLite transaction as its primary write so a crash doesn't lose downstream jobs). + +## Public surface + +- `pub async fn handle_job(config, job)` — `mod.rs` — branches on `job.kind` and invokes the matching handler. + +## Handlers (private to the module) + +- `handle_extract` — runs the scorer + LLM extractor over one chunk, packs the embedding, writes `mem_tree_score` + entity-index rows + chunk lifecycle in one tx, and enqueues the follow-up `append_buffer` and `topic_route` jobs. Also rewrites Obsidian-style `tags:` in the on-disk chunk markdown (best-effort, post-tx). +- `handle_append_buffer` — hydrates a `LeafRef` (chunk or summary), pushes into the target tree's L0 buffer, and enqueues a `seal` job if the buffer crosses its budget. Updates chunk lifecycle (`buffered`) for source-tree appends. All in one tx. +- `handle_seal` — seals exactly one buffer level via `bucket_seal::seal_one_level` (which atomically inserts the parent-cascade seal and summary-side `topic_route` for source trees). Topic-tree seals are sinks and do not enqueue further routing. Rewrites tags on the sealed summary's `.md` post-commit. +- `handle_topic_route` — for each canonical entity associated with the node, asks the topic curator whether to spawn a topic tree, and enqueues an `append_buffer` per matched topic tree. +- `handle_digest_daily` — invokes `tree_global::digest::end_of_day_digest` for the requested UTC date; idempotent via the digest's own `find_existing_daily` check. +- `handle_flush_stale` — walks `list_stale_buffers` and enqueues a forced `seal` per buffer over the configured `DEFAULT_FLUSH_AGE_SECS` cap. + +## Files + +- `mod.rs` — `handle_job` dispatch and all handler bodies. diff --git a/src/openhuman/memory/tree/jobs/handlers/mod.rs b/src/openhuman/memory/tree/jobs/handlers/mod.rs index 2de3d5e1b..eef71a7dd 100644 --- a/src/openhuman/memory/tree/jobs/handlers/mod.rs +++ b/src/openhuman/memory/tree/jobs/handlers/mod.rs @@ -1,10 +1,16 @@ +//! Per-`JobKind` handler implementations dispatched by the worker pool. +//! +//! Each handler parses its payload from `Job::payload_json`, performs its +//! side effects (DB writes, LLM calls, follow-up enqueues), and returns +//! `Ok(())` on success or an `anyhow::Error` on retryable failure. +//! [`handle_job`] fans out to the handler matching the row's `kind`. + use anyhow::{Context, Result}; use crate::openhuman::config::Config; use crate::openhuman::memory::tree::content_store::{ self as content_store, read as content_read, tags as content_tags, }; -use crate::openhuman::memory::tree::global_tree::digest::{self, DigestOutcome}; use crate::openhuman::memory::tree::jobs::store; use crate::openhuman::memory::tree::jobs::types::{ AppendBufferPayload, AppendTarget, DigestDailyPayload, ExtractChunkPayload, FlushStalePayload, @@ -14,12 +20,14 @@ use crate::openhuman::memory::tree::score; use crate::openhuman::memory::tree::score::embed::{build_embedder_from_config, pack_checked}; use crate::openhuman::memory::tree::score::extract::build_summary_extractor; use crate::openhuman::memory::tree::score::store as score_store; -use crate::openhuman::memory::tree::source_tree::{ +use crate::openhuman::memory::tree::store as chunk_store; +use crate::openhuman::memory::tree::tree_global::digest::{self, DigestOutcome}; +use crate::openhuman::memory::tree::tree_source::{ build_summariser, get_or_create_source_tree, LabelStrategy, LeafRef, }; -use crate::openhuman::memory::tree::store as chunk_store; -use crate::openhuman::memory::tree::topic_tree::curator; +use crate::openhuman::memory::tree::tree_topic::curator; +/// Dispatch a claimed job to the matching per-kind handler. pub async fn handle_job(config: &Config, job: &Job) -> Result<()> { match job.kind { JobKind::ExtractChunk => handle_extract(config, job).await, @@ -200,8 +208,8 @@ async fn handle_extract(config: &Config, job: &Job) -> Result<()> { } async fn handle_append_buffer(config: &Config, job: &Job) -> Result<()> { - use crate::openhuman::memory::tree::source_tree::bucket_seal::should_seal; - use crate::openhuman::memory::tree::source_tree::store as src_store; + use crate::openhuman::memory::tree::tree_source::bucket_seal::should_seal; + use crate::openhuman::memory::tree::tree_source::store as src_store; let payload: AppendBufferPayload = serde_json::from_str(&job.payload_json).context("parse AppendBuffer payload")?; @@ -338,9 +346,9 @@ async fn handle_append_buffer(config: &Config, job: &Job) -> Result<()> { } async fn handle_seal(config: &Config, job: &Job) -> Result<()> { - use crate::openhuman::memory::tree::source_tree::bucket_seal::{seal_one_level, should_seal}; - use crate::openhuman::memory::tree::source_tree::store as src_store; - use crate::openhuman::memory::tree::source_tree::types::TreeKind; + use crate::openhuman::memory::tree::tree_source::bucket_seal::{seal_one_level, should_seal}; + use crate::openhuman::memory::tree::tree_source::store as src_store; + use crate::openhuman::memory::tree::tree_source::types::TreeKind; let payload: SealPayload = serde_json::from_str(&job.payload_json).context("parse Seal payload")?; @@ -428,7 +436,7 @@ async fn handle_topic_route(config: &Config, job: &Job) -> Result<()> { chunk_id.clone() } NodeRef::Summary { summary_id } => { - if crate::openhuman::memory::tree::source_tree::store::get_summary(config, summary_id)? + if crate::openhuman::memory::tree::tree_source::store::get_summary(config, summary_id)? .is_none() { log::warn!( @@ -449,9 +457,9 @@ async fn handle_topic_route(config: &Config, job: &Job) -> Result<()> { let summariser = build_summariser(config); for entity_id in entity_ids { let _ = curator::maybe_spawn_topic_tree(config, &entity_id, summariser.as_ref()).await?; - if let Some(tree) = crate::openhuman::memory::tree::source_tree::store::get_tree_by_scope( + if let Some(tree) = crate::openhuman::memory::tree::tree_source::store::get_tree_by_scope( config, - crate::openhuman::memory::tree::source_tree::types::TreeKind::Topic, + crate::openhuman::memory::tree::tree_source::types::TreeKind::Topic, &entity_id, )? { let job = NewJob::append_buffer(&AppendBufferPayload { @@ -491,10 +499,10 @@ async fn handle_flush_stale(config: &Config, job: &Job) -> Result<()> { serde_json::from_str(&job.payload_json).context("parse FlushStale payload")?; let age_secs = payload .max_age_secs - .unwrap_or(crate::openhuman::memory::tree::source_tree::types::DEFAULT_FLUSH_AGE_SECS); + .unwrap_or(crate::openhuman::memory::tree::tree_source::types::DEFAULT_FLUSH_AGE_SECS); let cutoff = chrono::Utc::now() - chrono::Duration::seconds(age_secs); let buffers = - crate::openhuman::memory::tree::source_tree::store::list_stale_buffers(config, cutoff)?; + crate::openhuman::memory::tree::tree_source::store::list_stale_buffers(config, cutoff)?; for buf in buffers { let seal = SealPayload { tree_id: buf.tree_id.clone(), @@ -514,10 +522,10 @@ mod tests { use crate::openhuman::memory::tree::content_store; use crate::openhuman::memory::tree::jobs::store::{count_by_status, count_total}; use crate::openhuman::memory::tree::jobs::types::JobStatus; - use crate::openhuman::memory::tree::source_tree::bucket_seal::{append_leaf_deferred, LeafRef}; - use crate::openhuman::memory::tree::source_tree::registry::get_or_create_source_tree; - use crate::openhuman::memory::tree::source_tree::store as src_store; use crate::openhuman::memory::tree::store::with_connection; + use crate::openhuman::memory::tree::tree_source::bucket_seal::{append_leaf_deferred, LeafRef}; + use crate::openhuman::memory::tree::tree_source::registry::get_or_create_source_tree; + use crate::openhuman::memory::tree::tree_source::store as src_store; use chrono::TimeZone; use rusqlite::params; use tempfile::TempDir; @@ -571,7 +579,7 @@ mod tests { /// fire `handle_seal` and inspect the result. async fn seed_source_tree_ready_to_seal( cfg: &Config, - ) -> crate::openhuman::memory::tree::source_tree::types::Tree { + ) -> crate::openhuman::memory::tree::tree_source::types::Tree { use crate::openhuman::memory::tree::store::upsert_chunks; use crate::openhuman::memory::tree::types::{ chunk_id, Chunk, Metadata, SourceKind, SourceRef, @@ -676,7 +684,7 @@ mod tests { // Spawn a topic tree directly via the registry (skipping curator's // hotness gate — we just need a TreeKind::Topic with leaves). let topic_tree = - crate::openhuman::memory::tree::topic_tree::registry::get_or_create_topic_tree( + crate::openhuman::memory::tree::tree_topic::registry::get_or_create_topic_tree( &cfg, "topic:phoenix-migration", ) @@ -753,7 +761,7 @@ mod tests { // 1. Create a target topic tree with a clean L0 buffer. let topic_tree = - crate::openhuman::memory::tree::topic_tree::registry::get_or_create_topic_tree( + crate::openhuman::memory::tree::tree_topic::registry::get_or_create_topic_tree( &cfg, "email:alice@example.com", ) @@ -765,8 +773,8 @@ mod tests { // is to create a separate source tree, push two 6k leaves into // it, and let the seal produce a summary we can address. let source_tree = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); - use crate::openhuman::memory::tree::source_tree::bucket_seal::seal_one_level; use crate::openhuman::memory::tree::store::upsert_chunks; + use crate::openhuman::memory::tree::tree_source::bucket_seal::seal_one_level; use crate::openhuman::memory::tree::types::{ chunk_id, Chunk, Metadata, SourceKind, SourceRef, }; @@ -821,7 +829,7 @@ mod tests { &source_tree, &buf, summariser.as_ref(), - &crate::openhuman::memory::tree::source_tree::bucket_seal::LabelStrategy::Empty, + &crate::openhuman::memory::tree::tree_source::bucket_seal::LabelStrategy::Empty, // No follow-up enqueues — the test scopes assertions to the // append_buffer handler, not seal-side fan-out. false, diff --git a/src/openhuman/memory/tree/jobs/mod.rs b/src/openhuman/memory/tree/jobs/mod.rs index bcc5aec95..8ce89ab2a 100644 --- a/src/openhuman/memory/tree/jobs/mod.rs +++ b/src/openhuman/memory/tree/jobs/mod.rs @@ -14,7 +14,7 @@ //! append_buffer → push to L0 → enqueue seal if gate met → enqueue topic_route //! seal → seal one level → enqueue parent seal if cascading //! topic_route → match topics → enqueue per-topic append_buffer -//! digest_daily → call global_tree::digest::end_of_day_digest +//! digest_daily → call tree_global::digest::end_of_day_digest //! flush_stale → enqueue seals for time-stale buffers //! //! scheduler (1 task) ──► daily wall-clock tick: diff --git a/src/openhuman/memory/tree/jobs/scheduler.rs b/src/openhuman/memory/tree/jobs/scheduler.rs index ea77035d2..d38324193 100644 --- a/src/openhuman/memory/tree/jobs/scheduler.rs +++ b/src/openhuman/memory/tree/jobs/scheduler.rs @@ -1,3 +1,8 @@ +//! Wall-clock scheduler that wakes once a day shortly after UTC midnight to +//! enqueue the global [`JobKind::DigestDaily`] for yesterday and a +//! [`JobKind::FlushStale`] for today. Also exposes manual-trigger helpers +//! for catch-up and testing. + use std::time::Duration; use anyhow::Result; diff --git a/src/openhuman/memory/tree/jobs/testing.rs b/src/openhuman/memory/tree/jobs/testing.rs index e8211fb78..c739443fa 100644 --- a/src/openhuman/memory/tree/jobs/testing.rs +++ b/src/openhuman/memory/tree/jobs/testing.rs @@ -1,3 +1,5 @@ +//! Test helpers for the jobs runtime — not used in production code paths. + use anyhow::Result; use crate::openhuman::config::Config; diff --git a/src/openhuman/memory/tree/jobs/types.rs b/src/openhuman/memory/tree/jobs/types.rs index e6bf044f2..676ea4d95 100644 --- a/src/openhuman/memory/tree/jobs/types.rs +++ b/src/openhuman/memory/tree/jobs/types.rs @@ -27,6 +27,7 @@ pub enum JobKind { } impl JobKind { + /// Snake-case wire string written to `mem_tree_jobs.kind`. pub fn as_str(&self) -> &'static str { match self { JobKind::ExtractChunk => "extract_chunk", @@ -38,6 +39,7 @@ impl JobKind { } } + /// Inverse of [`Self::as_str`]; returns `Err` for unknown kinds. pub fn parse(s: &str) -> Result { Ok(match s { "extract_chunk" => JobKind::ExtractChunk, @@ -76,6 +78,7 @@ pub enum JobStatus { } impl JobStatus { + /// Snake-case wire string written to `mem_tree_jobs.status`. pub fn as_str(&self) -> &'static str { match self { JobStatus::Ready => "ready", @@ -86,6 +89,7 @@ impl JobStatus { } } + /// Inverse of [`Self::as_str`]; returns `Err` for unknown values. pub fn parse(s: &str) -> Result { Ok(match s { "ready" => JobStatus::Ready, @@ -97,6 +101,8 @@ impl JobStatus { }) } + /// True for `Done`, `Failed`, `Cancelled` — i.e. no further worker + /// transitions are expected. pub fn is_terminal(&self) -> bool { matches!( self, @@ -118,7 +124,8 @@ pub enum NodeRef { } impl NodeRef { - /// Stringified id with kind prefix, suitable for dedupe-key composition. + /// Stringified id with kind prefix (`leaf:` or `summary:`), suitable + /// for dedupe-key composition. pub fn dedupe_fragment(&self) -> String { match self { NodeRef::Leaf { chunk_id } => format!("leaf:{chunk_id}"), @@ -133,6 +140,8 @@ pub struct ExtractChunkPayload { } impl ExtractChunkPayload { + /// Stable dedupe key written to `mem_tree_jobs.dedupe_key` so a partial + /// unique index can suppress in-flight duplicates. pub fn dedupe_key(&self) -> String { format!("extract:{}", self.chunk_id) } @@ -155,6 +164,8 @@ pub struct AppendBufferPayload { } impl AppendBufferPayload { + /// Stable dedupe key written to `mem_tree_jobs.dedupe_key` so a partial + /// unique index can suppress in-flight duplicates. pub fn dedupe_key(&self) -> String { let node_part = self.node.dedupe_fragment(); match &self.target { @@ -179,6 +190,8 @@ pub struct SealPayload { } impl SealPayload { + /// Stable dedupe key written to `mem_tree_jobs.dedupe_key` so a partial + /// unique index can suppress in-flight duplicates. pub fn dedupe_key(&self) -> String { // Active seal-job uniqueness is enforced per (tree, level): a seal // already in flight suppresses duplicate enqueues. Once the job @@ -194,6 +207,8 @@ pub struct TopicRoutePayload { } impl TopicRoutePayload { + /// Stable dedupe key written to `mem_tree_jobs.dedupe_key` so a partial + /// unique index can suppress in-flight duplicates. pub fn dedupe_key(&self) -> String { format!("topic_route:{}", self.node.dedupe_fragment()) } @@ -207,6 +222,8 @@ pub struct DigestDailyPayload { } impl DigestDailyPayload { + /// Stable dedupe key written to `mem_tree_jobs.dedupe_key` so a partial + /// unique index can suppress in-flight duplicates. pub fn dedupe_key(&self) -> String { format!("digest_daily:{}", self.date_iso) } @@ -221,6 +238,8 @@ pub struct FlushStalePayload { } impl FlushStalePayload { + /// Stable dedupe key. `date_iso` scopes one flush per UTC day so the + /// scheduler can re-enqueue safely without duplicating work. pub fn dedupe_key(&self, date_iso: &str) -> String { format!("flush_stale:{date_iso}") } @@ -259,6 +278,7 @@ pub struct NewJob { } impl NewJob { + /// Build an [`JobKind::ExtractChunk`] enqueue request. pub fn extract_chunk(p: &ExtractChunkPayload) -> Result { Ok(Self { kind: JobKind::ExtractChunk, @@ -269,6 +289,7 @@ impl NewJob { }) } + /// Build an [`JobKind::AppendBuffer`] enqueue request. pub fn append_buffer(p: &AppendBufferPayload) -> Result { Ok(Self { kind: JobKind::AppendBuffer, @@ -279,6 +300,7 @@ impl NewJob { }) } + /// Build an [`JobKind::Seal`] enqueue request. pub fn seal(p: &SealPayload) -> Result { Ok(Self { kind: JobKind::Seal, @@ -289,6 +311,7 @@ impl NewJob { }) } + /// Build an [`JobKind::TopicRoute`] enqueue request. pub fn topic_route(p: &TopicRoutePayload) -> Result { Ok(Self { kind: JobKind::TopicRoute, @@ -299,6 +322,7 @@ impl NewJob { }) } + /// Build an [`JobKind::DigestDaily`] enqueue request. pub fn digest_daily(p: &DigestDailyPayload) -> Result { Ok(Self { kind: JobKind::DigestDaily, @@ -309,6 +333,7 @@ impl NewJob { }) } + /// Build an [`JobKind::FlushStale`] enqueue request scoped to `date_iso`. pub fn flush_stale(p: &FlushStalePayload, date_iso: &str) -> Result { Ok(Self { kind: JobKind::FlushStale, diff --git a/src/openhuman/memory/tree/jobs/worker.rs b/src/openhuman/memory/tree/jobs/worker.rs index 58c9aa7a4..571e59a3b 100644 --- a/src/openhuman/memory/tree/jobs/worker.rs +++ b/src/openhuman/memory/tree/jobs/worker.rs @@ -1,3 +1,7 @@ +//! Worker pool: claims jobs from `mem_tree_jobs`, dispatches them through +//! [`handlers::handle_job`], and settles the row. A small global semaphore +//! caps concurrent LLM-bound work; non-LLM jobs run unrestricted. + use std::sync::{Arc, OnceLock}; use std::time::Duration; @@ -16,6 +20,8 @@ const POLL_INTERVAL: Duration = Duration::from_secs(5); static WORKER_NOTIFY: OnceLock> = OnceLock::new(); static STARTED: std::sync::Once = std::sync::Once::new(); +/// Notify any idle workers so they re-poll immediately instead of waiting +/// out [`POLL_INTERVAL`]. Cheap no-op before [`start`] has run. pub fn wake_workers() { if let Some(notify) = WORKER_NOTIFY.get() { notify.notify_waiters(); @@ -67,6 +73,9 @@ pub fn start(config: Config) { }); } +/// Claim and run a single job. Returns `true` when work was processed, +/// `false` when no eligible row was available. Test entry point — the +/// production worker loop calls [`run_once_with_semaphore`] directly. pub async fn run_once(config: &Config) -> Result { let llm_slots = Arc::new(Semaphore::new(1)); run_once_with_semaphore(config, llm_slots).await diff --git a/src/openhuman/memory/tree/mod.rs b/src/openhuman/memory/tree/mod.rs index 7a10d546c..4b2383a82 100644 --- a/src/openhuman/memory/tree/mod.rs +++ b/src/openhuman/memory/tree/mod.rs @@ -25,16 +25,16 @@ pub mod canonicalize; pub mod chunker; pub mod content_store; -pub mod global_tree; pub mod ingest; pub mod jobs; pub mod retrieval; pub mod rpc; pub mod schemas; pub mod score; -pub mod source_tree; pub mod store; -pub mod topic_tree; +pub mod tree_global; +pub mod tree_source; +pub mod tree_topic; pub mod types; pub mod util; diff --git a/src/openhuman/memory/tree/retrieval/README.md b/src/openhuman/memory/tree/retrieval/README.md new file mode 100644 index 000000000..a580fbb74 --- /dev/null +++ b/src/openhuman/memory/tree/retrieval/README.md @@ -0,0 +1,30 @@ +# Retrieval + +Phase 4 (#710) — search-time pipeline for the hierarchical memory tree. Exposes six LLM-callable primitives that read across the source / topic / global trees built by Phase 3 and surface results in a uniform [`RetrievalHit`] shape. There is no classifier, gate, or composer in this phase — orchestration (which tool to call, how to combine) is left to the calling LLM. + +## Public surface + +- `pub fn query_source` / `pub struct QuerySourceRequest` — `source.rs`, `rpc.rs` — per-source summary retrieval, optional semantic rerank. +- `pub fn query_global` / `pub struct QueryGlobalRequest` — `global.rs`, `rpc.rs` — cross-source digest for a window in days. +- `pub fn query_topic` / `pub struct QueryTopicRequest` — `topic.rs`, `rpc.rs` — entity-scoped retrieval across every tree. +- `pub fn search_entities` / `pub struct SearchEntitiesRequest` — `search.rs`, `rpc.rs` — fuzzy LIKE lookup over the entity index. +- `pub fn drill_down` / `pub struct DrillDownRequest` — `drill_down.rs`, `rpc.rs` — walk `child_ids` from a summary one (or more) levels down. +- `pub fn fetch_leaves` / `pub struct FetchLeavesRequest` — `fetch.rs`, `rpc.rs` — batch-hydrate raw chunks by id (cap 20). +- `pub struct RetrievalHit` / `pub enum NodeKind` / `pub struct QueryResponse` / `pub struct EntityMatch` — `types.rs` — wire shapes shared by every tool. +- `pub fn all_retrieval_controller_schemas` / `pub fn all_retrieval_registered_controllers` — `schemas.rs` — registry exports wired into `core::all`. + +## Files + +- `mod.rs` — module surface; declares submodules and the `pub use` re-exports. +- `types.rs` — shared wire types and the `hit_from_summary` / `hit_from_chunk` helpers. +- `source.rs` / `global.rs` / `topic.rs` — query the corresponding tree level. +- `search.rs` — free-text LIKE search over `mem_tree_entity_index`. +- `drill_down.rs` — BFS walk of summary children with optional semantic rerank. +- `fetch.rs` — batch hydration of leaf chunks. +- `rpc.rs` — request / response structs and the JSON-RPC handler bodies. +- `schemas.rs` — `ControllerSchema` definitions and dispatch table for the controller registry. +- `integration_test.rs` — end-to-end test that drives the real ingest pipeline through every retrieval tool. + +## Tests + +Per-tool unit tests live in `mod tests` inside each file. The `integration_test.rs` module is private to this crate and exercises ingest → seal → retrieve in one workspace. diff --git a/src/openhuman/memory/tree/retrieval/drill_down.rs b/src/openhuman/memory/tree/retrieval/drill_down.rs index 8b30ad257..88e369d58 100644 --- a/src/openhuman/memory/tree/retrieval/drill_down.rs +++ b/src/openhuman/memory/tree/retrieval/drill_down.rs @@ -28,8 +28,8 @@ use crate::openhuman::memory::tree::retrieval::types::{ hit_from_chunk, hit_from_summary, RetrievalHit, }; use crate::openhuman::memory::tree::score::embed::{build_embedder_from_config, cosine_similarity}; -use crate::openhuman::memory::tree::source_tree::store; use crate::openhuman::memory::tree::store::{get_chunk, get_chunk_embedding}; +use crate::openhuman::memory::tree::tree_source::store; /// Walk the summary hierarchy down one step (or more if `max_depth > 1`) /// and return the hydrated child hits. Children at level 1 are raw chunks; @@ -239,13 +239,13 @@ fn walk_with_embeddings( mod tests { use super::*; use crate::openhuman::memory::tree::content_store; - use crate::openhuman::memory::tree::source_tree::bucket_seal::{ + use crate::openhuman::memory::tree::store::upsert_chunks; + use crate::openhuman::memory::tree::tree_source::bucket_seal::{ append_leaf, LabelStrategy, LeafRef, }; - use crate::openhuman::memory::tree::source_tree::registry::get_or_create_source_tree; - use crate::openhuman::memory::tree::source_tree::summariser::inert::InertSummariser; - use crate::openhuman::memory::tree::source_tree::types::TreeKind; - use crate::openhuman::memory::tree::store::upsert_chunks; + use crate::openhuman::memory::tree::tree_source::registry::get_or_create_source_tree; + use crate::openhuman::memory::tree::tree_source::summariser::inert::InertSummariser; + use crate::openhuman::memory::tree::tree_source::types::TreeKind; use crate::openhuman::memory::tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; use chrono::Utc; use tempfile::TempDir; @@ -282,7 +282,7 @@ mod tests { tags: vec![], source_ref: Some(SourceRef::new("slack://x")), }, - token_count: crate::openhuman::memory::tree::source_tree::types::TOKEN_BUDGET * 6 + token_count: crate::openhuman::memory::tree::tree_source::types::TOKEN_BUDGET * 6 / 10, seq_in_source: seq, created_at: ts, @@ -305,7 +305,7 @@ mod tests { &tree, &LeafRef { chunk_id: c.id.clone(), - token_count: crate::openhuman::memory::tree::source_tree::types::TOKEN_BUDGET + token_count: crate::openhuman::memory::tree::tree_source::types::TOKEN_BUDGET * 6 / 10, timestamp: ts, @@ -410,9 +410,9 @@ mod tests { // (or similar — the key invariant is that BFS returns all siblings at // one depth before any descendant at a deeper depth). - use crate::openhuman::memory::tree::source_tree::store as tree_store; - use crate::openhuman::memory::tree::source_tree::types::{SummaryNode, Tree, TreeStatus}; use crate::openhuman::memory::tree::store::with_connection; + use crate::openhuman::memory::tree::tree_source::store as tree_store; + use crate::openhuman::memory::tree::tree_source::types::{SummaryNode, Tree, TreeStatus}; /// Build a tiny 2-level tree directly via store inserts so we can /// assert BFS ordering without needing ~100 leaves to cascade L1→L2 diff --git a/src/openhuman/memory/tree/retrieval/global.rs b/src/openhuman/memory/tree/retrieval/global.rs index 9d6e22bf4..dce81ae75 100644 --- a/src/openhuman/memory/tree/retrieval/global.rs +++ b/src/openhuman/memory/tree/retrieval/global.rs @@ -1,7 +1,7 @@ //! `memory_tree_query_global` — window-scoped recap from the global digest //! (Phase 4 / #710). //! -//! Thin wrapper on [`global_tree::recap::recap`]. The recap function does +//! Thin wrapper on [`tree_global::recap::recap`]. The recap function does //! the heavy lifting (level selection + time-range filter); we convert its //! output into the uniform [`RetrievalHit`] shape. //! @@ -13,10 +13,10 @@ use anyhow::Result; use chrono::Duration; use crate::openhuman::config::Config; -use crate::openhuman::memory::tree::global_tree::recap::{recap, RecapOutput}; -use crate::openhuman::memory::tree::global_tree::registry::get_or_create_global_tree; use crate::openhuman::memory::tree::retrieval::types::{NodeKind, QueryResponse, RetrievalHit}; -use crate::openhuman::memory::tree::source_tree::types::TreeKind; +use crate::openhuman::memory::tree::tree_global::recap::{recap, RecapOutput}; +use crate::openhuman::memory::tree::tree_global::registry::get_or_create_global_tree; +use crate::openhuman::memory::tree::tree_source::types::TreeKind; /// Return the global digest for the given window in days. Always returns a /// [`QueryResponse`]; the response is empty if the global tree has no @@ -88,13 +88,13 @@ fn recap_to_hits(recap: RecapOutput, tree_id: &str, tree_scope: &str) -> Vec, } +/// JSON-RPC handler body for `memory_tree_query_source`. Parses the +/// request, delegates to [`super::source::query_source`], and wraps the +/// outcome with a PII-redacted log line. pub async fn query_source_rpc( config: &Config, req: QuerySourceRequest, @@ -76,11 +81,13 @@ pub async fn query_source_rpc( // ── query_global ────────────────────────────────────────────────────── +/// Request body for `memory_tree_query_global`. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct QueryGlobalRequest { pub window_days: u32, } +/// JSON-RPC handler body for `memory_tree_query_global`. pub async fn query_global_rpc( config: &Config, req: QueryGlobalRequest, @@ -97,6 +104,7 @@ pub async fn query_global_rpc( // ── query_topic ─────────────────────────────────────────────────────── +/// Request body for `memory_tree_query_topic`. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct QueryTopicRequest { pub entity_id: String, @@ -110,6 +118,7 @@ pub struct QueryTopicRequest { pub limit: Option, } +/// JSON-RPC handler body for `memory_tree_query_topic`. pub async fn query_topic_rpc( config: &Config, req: QueryTopicRequest, @@ -145,6 +154,7 @@ pub async fn query_topic_rpc( // ── search_entities ─────────────────────────────────────────────────── +/// Request body for `memory_tree_search_entities`. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct SearchEntitiesRequest { pub query: String, @@ -154,11 +164,14 @@ pub struct SearchEntitiesRequest { pub limit: Option, } +/// Response envelope for `memory_tree_search_entities`. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct SearchEntitiesResponse { pub matches: Vec, } +/// JSON-RPC handler body for `memory_tree_search_entities`. Validates the +/// optional `kinds` filter against [`EntityKind`]. pub async fn search_entities_rpc( config: &Config, req: SearchEntitiesRequest, @@ -191,6 +204,7 @@ pub async fn search_entities_rpc( // ── drill_down ──────────────────────────────────────────────────────── +/// Request body for `memory_tree_drill_down`. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct DrillDownRequest { pub node_id: String, @@ -207,11 +221,13 @@ pub struct DrillDownRequest { pub limit: Option, } +/// Response envelope for `memory_tree_drill_down`. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct DrillDownResponse { pub hits: Vec, } +/// JSON-RPC handler body for `memory_tree_drill_down`. pub async fn drill_down_rpc( config: &Config, req: DrillDownRequest, @@ -243,16 +259,19 @@ pub async fn drill_down_rpc( // ── fetch_leaves ────────────────────────────────────────────────────── +/// Request body for `memory_tree_fetch_leaves`. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct FetchLeavesRequest { pub chunk_ids: Vec, } +/// Response envelope for `memory_tree_fetch_leaves`. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct FetchLeavesResponse { pub hits: Vec, } +/// JSON-RPC handler body for `memory_tree_fetch_leaves`. pub async fn fetch_leaves_rpc( config: &Config, req: FetchLeavesRequest, diff --git a/src/openhuman/memory/tree/retrieval/schemas.rs b/src/openhuman/memory/tree/retrieval/schemas.rs index b0e457ce6..10b34d942 100644 --- a/src/openhuman/memory/tree/retrieval/schemas.rs +++ b/src/openhuman/memory/tree/retrieval/schemas.rs @@ -23,6 +23,8 @@ use crate::rpc::RpcOutcome; const NAMESPACE: &str = "memory_tree"; +/// Return one [`ControllerSchema`] per Phase 4 retrieval tool. Used by +/// the controller registry to publish the `memory_tree.*` schemas. pub fn all_controller_schemas() -> Vec { vec![ schemas("query_source"), @@ -34,6 +36,8 @@ pub fn all_controller_schemas() -> Vec { ] } +/// Return one [`RegisteredController`] per Phase 4 retrieval tool — schema +/// paired with its dispatch handler. Wired into `core::all` at startup. pub fn all_registered_controllers() -> Vec { vec![ RegisteredController { @@ -90,6 +94,8 @@ fn query_response_outputs() -> Vec { ] } +/// Look up the [`ControllerSchema`] for a single retrieval `function` +/// name. Unknown names return a placeholder schema with an `error` field. pub fn schemas(function: &str) -> ControllerSchema { match function { "query_source" => ControllerSchema { @@ -143,7 +149,7 @@ pub fn schemas(function: &str) -> ControllerSchema { namespace: NAMESPACE, function: "query_global", description: "Return the global digest for the last N days. Wraps \ - `global_tree::recap`; the returned hit carries `child_ids` pointing \ + `tree_global::recap`; the returned hit carries `child_ids` pointing \ at the folded per-day summary ids for drill-down.", inputs: vec![FieldSchema { name: "window_days", diff --git a/src/openhuman/memory/tree/retrieval/source.rs b/src/openhuman/memory/tree/retrieval/source.rs index 8580bd9e1..a282f85af 100644 --- a/src/openhuman/memory/tree/retrieval/source.rs +++ b/src/openhuman/memory/tree/retrieval/source.rs @@ -26,8 +26,8 @@ use crate::openhuman::memory::tree::retrieval::types::{ hit_from_summary, QueryResponse, RetrievalHit, }; use crate::openhuman::memory::tree::score::embed::{build_embedder_from_config, cosine_similarity}; -use crate::openhuman::memory::tree::source_tree::store; -use crate::openhuman::memory::tree::source_tree::types::{SummaryNode, Tree, TreeKind}; +use crate::openhuman::memory::tree::tree_source::store; +use crate::openhuman::memory::tree::tree_source::types::{SummaryNode, Tree, TreeKind}; use crate::openhuman::memory::tree::types::SourceKind; const DEFAULT_LIMIT: usize = 10; @@ -307,12 +307,12 @@ fn filter_by_window(hits: Vec, window_days: u32) -> Vec, ) -> Result> { use crate::openhuman::memory::tree::retrieval::types::NodeKind; - use crate::openhuman::memory::tree::source_tree::store as src_store; use crate::openhuman::memory::tree::store::get_chunk_embedding; + use crate::openhuman::memory::tree::tree_source::store as src_store; let embedder = build_embedder_from_config(config)?; let query_vec = embedder.embed(query).await?; @@ -517,11 +517,11 @@ mod tests { use crate::openhuman::memory::tree::score::extract::EntityKind; use crate::openhuman::memory::tree::score::resolver::CanonicalEntity; use crate::openhuman::memory::tree::score::store as score_store; - use crate::openhuman::memory::tree::source_tree::store as tree_store; - use crate::openhuman::memory::tree::source_tree::types::{ + use crate::openhuman::memory::tree::store::with_connection; + use crate::openhuman::memory::tree::tree_source::store as tree_store; + use crate::openhuman::memory::tree::tree_source::types::{ SummaryNode, Tree, TreeKind, TreeStatus, }; - use crate::openhuman::memory::tree::store::with_connection; let (_tmp, cfg) = test_config(); let ts = Utc::now(); diff --git a/src/openhuman/memory/tree/retrieval/types.rs b/src/openhuman/memory/tree/retrieval/types.rs index b47ca90ac..bd3266ca2 100644 --- a/src/openhuman/memory/tree/retrieval/types.rs +++ b/src/openhuman/memory/tree/retrieval/types.rs @@ -19,7 +19,7 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use crate::openhuman::memory::tree::score::extract::EntityKind; -use crate::openhuman::memory::tree::source_tree::types::{SummaryNode, Tree, TreeKind}; +use crate::openhuman::memory::tree::tree_source::types::{SummaryNode, Tree, TreeKind}; use crate::openhuman::memory::tree::types::{Chunk, SourceKind}; /// Whether a hit represents a leaf (raw chunk) or a summary node. @@ -34,6 +34,8 @@ pub enum NodeKind { } impl NodeKind { + /// Stable lowercase string form (`"leaf"` / `"summary"`) — matches the + /// serde representation and is suitable for SQL discriminator columns. pub fn as_str(self) -> &'static str { match self { Self::Leaf => "leaf", diff --git a/src/openhuman/memory/tree/rpc.rs b/src/openhuman/memory/tree/rpc.rs index 2d370f5cf..5b2beff25 100644 --- a/src/openhuman/memory/tree/rpc.rs +++ b/src/openhuman/memory/tree/rpc.rs @@ -116,11 +116,14 @@ pub struct ListChunksRequest { pub limit: Option, } +/// Response shape for the `list_chunks` RPC. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct ListChunksResponse { pub chunks: Vec, } +/// `list_chunks` RPC handler. Filters and returns persisted chunks ordered by +/// timestamp DESC. pub async fn list_chunks_rpc( config: &Config, req: ListChunksRequest, @@ -151,16 +154,19 @@ pub async fn list_chunks_rpc( )) } +/// Request shape for the `get_chunk` RPC. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct GetChunkRequest { pub id: String, } +/// Response shape for the `get_chunk` RPC. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct GetChunkResponse { pub chunk: Option, } +/// `get_chunk` RPC handler. Returns the chunk identified by `id`, or `None`. pub async fn get_chunk_rpc( config: &Config, req: GetChunkRequest, @@ -192,6 +198,7 @@ pub struct TriggerDigestRequest { pub date_iso: Option, } +/// Response from the `trigger_digest` RPC. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct TriggerDigestResponse { /// True when the job was newly enqueued; false when an active job for @@ -205,6 +212,9 @@ pub struct TriggerDigestResponse { pub date_iso: String, } +/// `trigger_digest` RPC handler. Manually enqueues the global tree's daily +/// digest job for `date_iso` (defaults to yesterday in UTC); idempotent via the +/// jobs-queue dedupe index. pub async fn trigger_digest_rpc( config: &Config, req: TriggerDigestRequest, diff --git a/src/openhuman/memory/tree/schemas.rs b/src/openhuman/memory/tree/schemas.rs index c558a85f6..8894f6839 100644 --- a/src/openhuman/memory/tree/schemas.rs +++ b/src/openhuman/memory/tree/schemas.rs @@ -18,6 +18,8 @@ use crate::rpc::RpcOutcome; const NAMESPACE: &str = "memory_tree"; +/// All `memory_tree` controller schemas, used by the registry to advertise +/// inputs/outputs to CLI + JSON-RPC consumers. pub fn all_controller_schemas() -> Vec { vec![ schemas("ingest"), @@ -27,6 +29,8 @@ pub fn all_controller_schemas() -> Vec { ] } +/// Registered `memory_tree` controllers (schema + handler pairs) wired into +/// `core::all`. pub fn all_registered_controllers() -> Vec { vec![ RegisteredController { @@ -48,6 +52,7 @@ pub fn all_registered_controllers() -> Vec { ] } +/// Lookup the [`ControllerSchema`] for a single `memory_tree` function name. pub fn schemas(function: &str) -> ControllerSchema { match function { "ingest" => ControllerSchema { diff --git a/src/openhuman/memory/tree/score/README.md b/src/openhuman/memory/tree/score/README.md new file mode 100644 index 000000000..8c8785b87 --- /dev/null +++ b/src/openhuman/memory/tree/score/README.md @@ -0,0 +1,23 @@ +# Memory tree — score (Phase 2 / #708) + +Per-chunk admission, enrichment, and entity indexing for the bucket-seal-ready memory tree. Sits between leaf chunking and L0 buffer append: every chunk passes through `score_chunk` which decides whether to keep it, runs entity extraction, and persists score rationale + an inverted entity index used by retrieval. + +## Public surface + +- `pub fn score_chunk` / `pub fn score_chunks` / `pub fn score_chunks_fast` — `mod.rs` — scoring pipeline entry points (full / batch / cheap-only batch). +- `pub struct ScoreResult` / `pub struct ScoringConfig` — `mod.rs` — outcome and configuration of one scoring pass. +- `pub fn persist_score` / `persist_score_tx` — `mod.rs` — write the score row + entity-index rows for one kept chunk. +- `pub const DEFAULT_DROP_THRESHOLD` / `DEFAULT_DEFINITE_KEEP` / `DEFAULT_DEFINITE_DROP` — `mod.rs` — admission band defaults. + +## Subdirectories + +- `signals/` — per-signal feature computation (token count, unique words, metadata weight, source weight, interaction tags, entity density, LLM importance) plus the weighted combine that produces the final `[0.0, 1.0]` total. +- `extract/` — entity extraction: `EntityExtractor` trait, `RegexEntityExtractor` for mechanical identifiers (email, URL, handle, hashtag), `LlmEntityExtractor` for semantic NER + importance rating, `CompositeExtractor` for chaining them. +- `embed/` — Phase 4 vector embedder: `Embedder` trait, `OllamaEmbedder` (default), `InertEmbedder` (tests), pack/unpack helpers for the SQLite BLOB storage layout. + +## Files + +- `mod.rs` — orchestration: `score_chunk` runs extraction → cheap signals → optional borderline LLM call → admission gate → canonicalisation. +- `store.rs` — SQLite CRUD for `mem_tree_score` (per-chunk rationale) and `mem_tree_entity_index` (inverted index `entity_id → node_id`). +- `resolver.rs` — entity canonicalisation: normalises surface forms (lowercase emails, strip leading `@`/`#`) and assigns stable `canonical_id` strings; promotes extracted topics into the canonical entity stream. +- `mod_tests.rs` / `store_tests.rs` — unit tests. diff --git a/src/openhuman/memory/tree/score/embed/README.md b/src/openhuman/memory/tree/score/embed/README.md new file mode 100644 index 000000000..f4d881715 --- /dev/null +++ b/src/openhuman/memory/tree/score/embed/README.md @@ -0,0 +1,18 @@ +# Memory tree — score embed (Phase 4 / #710) + +Vector embedder for chunks and summaries. Produces a fixed-dimension (`EMBEDDING_DIM = 768`) `Vec` per text so retrieval can rerank candidates by semantic similarity. Default backend is local Ollama running `nomic-embed-text`; tests use the deterministic `InertEmbedder` so no network is required. + +## Public surface + +- `pub trait Embedder` — `mod.rs` — `embed(text) -> Vec` contract; impls must return exactly `EMBEDDING_DIM` floats. +- `pub fn build_embedder_from_config` — `factory.rs` — returns `OllamaEmbedder` when configured, otherwise `InertEmbedder` (or bails when `embedding_strict = true`). +- `pub struct OllamaEmbedder` — `ollama.rs` — HTTP client posting to `{endpoint}/api/embeddings`. +- `pub struct InertEmbedder` — `inert.rs` — zero-vector embedder for tests. +- `pub fn cosine_similarity` / `pack_embedding` / `unpack_embedding` / `pack_checked` / `decode_optional_blob` — `mod.rs` — math + SQLite BLOB packing helpers. + +## Files + +- `mod.rs` — trait, `EMBEDDING_DIM`, math + pack/unpack helpers, write-time / read-time semantics. +- `factory.rs` — `Config::memory_tree`-driven embedder selection with `embedding_strict` opt-in. +- `ollama.rs` — Ollama `/api/embeddings` client; defaults at `http://localhost:11434` / `nomic-embed-text` / 10s timeout. +- `inert.rs` — zero-vector embedder; cosine similarity between any two inert vectors is 0.0 (zero-magnitude short-circuit), so retrieval tests that need real reranking should hand-stitch embeddings instead of relying on this path. diff --git a/src/openhuman/memory/tree/score/embed/inert.rs b/src/openhuman/memory/tree/score/embed/inert.rs index 96f34671f..a122043fa 100644 --- a/src/openhuman/memory/tree/score/embed/inert.rs +++ b/src/openhuman/memory/tree/score/embed/inert.rs @@ -22,6 +22,7 @@ use super::{Embedder, EMBEDDING_DIM}; pub struct InertEmbedder; impl InertEmbedder { + /// Construct an inert embedder. Free — `InertEmbedder` is a ZST. pub fn new() -> Self { Self } diff --git a/src/openhuman/memory/tree/score/embed/ollama.rs b/src/openhuman/memory/tree/score/embed/ollama.rs index ba98bd810..c86deaf3c 100644 --- a/src/openhuman/memory/tree/score/embed/ollama.rs +++ b/src/openhuman/memory/tree/score/embed/ollama.rs @@ -88,7 +88,8 @@ impl OllamaEmbedder { } } - /// Convenience constructor using all defaults. + /// Convenience constructor using all defaults ([`DEFAULT_ENDPOINT`], + /// [`DEFAULT_MODEL`], [`DEFAULT_TIMEOUT_MS`]). pub fn default_new() -> Self { Self::new(String::new(), String::new(), 0) } diff --git a/src/openhuman/memory/tree/score/extract/README.md b/src/openhuman/memory/tree/score/extract/README.md new file mode 100644 index 000000000..55b215ac3 --- /dev/null +++ b/src/openhuman/memory/tree/score/extract/README.md @@ -0,0 +1,20 @@ +# Memory tree — score extract + +Entity extraction for the scoring pipeline. Pluggable via the `EntityExtractor` trait so the scorer can run a deterministic regex pass plus an optional LLM pass and merge their outputs. Also surfaces the LLM-derived importance rating consumed by the `llm_importance` signal. + +## Public surface + +- `pub trait EntityExtractor` — `extractor.rs` — async `extract(text) -> ExtractedEntities` contract. +- `pub struct RegexEntityExtractor` / `pub struct CompositeExtractor` — `extractor.rs` — built-in implementations. +- `pub struct LlmEntityExtractor` / `pub struct LlmExtractorConfig` — `llm.rs` — Ollama-backed semantic NER + importance rater. +- `pub fn build_summary_extractor` — `mod.rs` — composes regex + LLM (with `emit_topics: true`) for seal-time summary labelling. +- `pub enum EntityKind` / `pub struct ExtractedEntity` / `pub struct ExtractedTopic` / `pub struct ExtractedEntities` — `types.rs`. + +## Files + +- `mod.rs` — module surface and `build_summary_extractor` for the seal path. +- `types.rs` — output types and the `EntityKind` enum (mechanical kinds `Email/Url/Handle/Hashtag` + semantic kinds `Person/Organization/Location/...` + `Topic`). `ExtractedEntities::merge` deduplicates entities and combines LLM importance by max. +- `extractor.rs` — `EntityExtractor` trait, `RegexEntityExtractor` adapter, `CompositeExtractor` (runs a sequence of extractors and tolerates per-extractor failures). +- `regex.rs` — once-compiled regex patterns for email, URL, handle (`@alice` and Discord-style `alice#1234`), and hashtag. UTF-8 safe — spans are char offsets, not bytes. +- `llm.rs` — Ollama `/api/chat` client that asks the model for NER + an importance rating in one structured-JSON call, with span recovery via `text.find(...)` and a soft fallback (warn + empty) on transport failure. +- `llm_tests.rs` — unit tests for the LLM extractor. diff --git a/src/openhuman/memory/tree/score/extract/extractor.rs b/src/openhuman/memory/tree/score/extract/extractor.rs index a915856fc..6bcacc06d 100644 --- a/src/openhuman/memory/tree/score/extract/extractor.rs +++ b/src/openhuman/memory/tree/score/extract/extractor.rs @@ -1,3 +1,6 @@ +//! [`EntityExtractor`] trait plus the regex and composite implementations +//! used as Phase 2's default extraction stack. + use async_trait::async_trait; use super::regex; @@ -36,6 +39,8 @@ pub struct CompositeExtractor { } impl CompositeExtractor { + /// Build a composite from an explicit list of extractors. Order matters + /// only to logs — outputs are merged and deduplicated. pub fn new(inner: Vec>) -> Self { Self { inner } } diff --git a/src/openhuman/memory/tree/score/extract/llm.rs b/src/openhuman/memory/tree/score/extract/llm.rs index 20072aac6..e7e7c0fac 100644 --- a/src/openhuman/memory/tree/score/extract/llm.rs +++ b/src/openhuman/memory/tree/score/extract/llm.rs @@ -102,6 +102,8 @@ pub struct LlmEntityExtractor { } impl LlmEntityExtractor { + /// Build the extractor and its inner HTTP client. Fails only when + /// `reqwest` rejects the timeout configuration. pub fn new(cfg: LlmExtractorConfig) -> anyhow::Result { let http = Client::builder() .timeout(cfg.timeout) diff --git a/src/openhuman/memory/tree/score/extract/types.rs b/src/openhuman/memory/tree/score/extract/types.rs index 89b4fd672..5458238ec 100644 --- a/src/openhuman/memory/tree/score/extract/types.rs +++ b/src/openhuman/memory/tree/score/extract/types.rs @@ -53,6 +53,7 @@ pub enum EntityKind { } impl EntityKind { + /// Snake-case wire string for serialisation and SQL storage. pub fn as_str(self) -> &'static str { match self { Self::Email => "email", @@ -73,6 +74,7 @@ impl EntityKind { } } + /// Inverse of [`Self::as_str`]; returns `Err` for unknown wire strings. pub fn parse(s: &str) -> Result { match s { "email" => Ok(Self::Email), @@ -144,6 +146,7 @@ pub struct ExtractedEntities { } impl ExtractedEntities { + /// True when neither entities nor topics were extracted. pub fn is_empty(&self) -> bool { self.entities.is_empty() && self.topics.is_empty() } diff --git a/src/openhuman/memory/tree/score/signals/README.md b/src/openhuman/memory/tree/score/signals/README.md new file mode 100644 index 000000000..119fbaf9d --- /dev/null +++ b/src/openhuman/memory/tree/score/signals/README.md @@ -0,0 +1,14 @@ +# Memory tree — score signals + +Per-chunk scoring features. Each submodule computes one signal in `[0.0, 1.0]`; `ops::combine` aggregates them via `SignalWeights` into the final admission total. Signals are stored alongside the total in `mem_tree_score` so admit/drop decisions remain auditable. + +## Files + +- `mod.rs` — module surface: re-exports `compute`, `combine`, `combine_cheap_only`, `entity_density_score`, `ScoreSignals`, `SignalWeights`. +- `types.rs` — `ScoreSignals` (per-signal breakdown) and `SignalWeights` (per-signal multipliers, with `with_llm_enabled()` builder). +- `ops.rs` — `compute(meta, content, token_count, extracted)` populates a `ScoreSignals`; `combine` and `combine_cheap_only` produce the weighted total (the latter excludes the LLM-importance term used by the borderline-band short-circuit). +- `token_count.rs` — plateau-shaped score over chunk token count; scores 0 below `TOKEN_MIN`, ramps to 1 by `TOKEN_RAMP_LOW`, ramps back to 0.5 between `TOKEN_RAMP_HIGH` and `TOKEN_MAX`. +- `unique_words.rs` — type-token-ratio noise detector: low diversity scores low; messages under `MIN_TOTAL_WORDS` return a neutral 0.5. +- `metadata_weight.rs` — base weight per `SourceKind` (Email > Document > Chat). +- `source_weight.rs` — per-`DataSource` weight inferred from `provider:` tags, with `SourceKind` defaults as fallback. +- `interaction.rs` — engagement-tag bonus (`sent`, `reply`, `dm`, `mention`); absent tags return 0.5 so silent content isn't penalised. diff --git a/src/openhuman/memory/tree/score/signals/interaction.rs b/src/openhuman/memory/tree/score/signals/interaction.rs index 1ba96b85e..58a5ac791 100644 --- a/src/openhuman/memory/tree/score/signals/interaction.rs +++ b/src/openhuman/memory/tree/score/signals/interaction.rs @@ -15,9 +15,13 @@ use crate::openhuman::memory::tree::types::Metadata; +/// Tag set when the user replied to this message/thread. pub const TAG_REPLY: &str = "reply"; +/// Tag set when the user authored this content. pub const TAG_SENT: &str = "sent"; +/// Tag set when the user was @-mentioned. pub const TAG_MENTION: &str = "mention"; +/// Tag set when the message arrived in a direct-message channel. pub const TAG_DM: &str = "dm"; /// Score in `[0.0, 1.0]` based on engagement tags present on the chunk. diff --git a/src/openhuman/memory/tree/score/signals/ops.rs b/src/openhuman/memory/tree/score/signals/ops.rs index 5421f6dcd..368311069 100644 --- a/src/openhuman/memory/tree/score/signals/ops.rs +++ b/src/openhuman/memory/tree/score/signals/ops.rs @@ -1,3 +1,6 @@ +//! Cross-signal helpers: signal computation entry point and the two +//! weighted-combine variants (full and cheap-only) used by `score_chunk`. + use super::{interaction, metadata_weight, source_weight, token_count, unique_words}; use super::{ScoreSignals, SignalWeights}; use crate::openhuman::memory::tree::score::extract::ExtractedEntities; diff --git a/src/openhuman/memory/tree/score/signals/token_count.rs b/src/openhuman/memory/tree/score/signals/token_count.rs index a3b6e0cd7..9515a836d 100644 --- a/src/openhuman/memory/tree/score/signals/token_count.rs +++ b/src/openhuman/memory/tree/score/signals/token_count.rs @@ -8,10 +8,15 @@ //! Output is a score in `[0.0, 1.0]` shaped as a plateau between //! `TOKEN_MIN` and `TOKEN_MAX` with linear ramps on both sides. -pub const TOKEN_MIN: u32 = 10; // below this → score 0 -pub const TOKEN_RAMP_LOW: u32 = 30; // 10..30 → linear 0→1 -pub const TOKEN_RAMP_HIGH: u32 = 3_000; // 3000..8000 → linear 1→0.5 -pub const TOKEN_MAX: u32 = 8_000; // above → score 0.5 (not zero — still has content) +/// Below this token count the chunk scores 0 (treated as noise). +pub const TOKEN_MIN: u32 = 10; +/// Top of the linear ramp from 0 → 1 starting at [`TOKEN_MIN`]. +pub const TOKEN_RAMP_LOW: u32 = 30; +/// Start of the linear ramp from 1 → 0.5 ending at [`TOKEN_MAX`]. +pub const TOKEN_RAMP_HIGH: u32 = 3_000; +/// Above this token count the score is clamped to 0.5 (oversized content +/// still carries information but loses the plateau bonus). +pub const TOKEN_MAX: u32 = 8_000; /// Score for a chunk's token count. See module docs for shape. pub fn score(token_count: u32) -> f32 { diff --git a/src/openhuman/memory/tree/score/signals/types.rs b/src/openhuman/memory/tree/score/signals/types.rs index b20bbc167..529136850 100644 --- a/src/openhuman/memory/tree/score/signals/types.rs +++ b/src/openhuman/memory/tree/score/signals/types.rs @@ -1,3 +1,7 @@ +//! Strongly-typed bag of per-signal scores plus the weights used to combine +//! them. Persisted alongside the total in `mem_tree_score` so a chunk's +//! admit/drop decision is auditable after the fact. + use serde::{Deserialize, Serialize}; /// Per-signal score breakdown for one chunk. Persisted alongside the total diff --git a/src/openhuman/memory/tree/score/signals/unique_words.rs b/src/openhuman/memory/tree/score/signals/unique_words.rs index 846b7cb5e..0ae1b6e3d 100644 --- a/src/openhuman/memory/tree/score/signals/unique_words.rs +++ b/src/openhuman/memory/tree/score/signals/unique_words.rs @@ -8,6 +8,8 @@ //! minimum total count before this signal contributes — otherwise "hi bob" //! would score identically to a real message. +/// Below this total-word count the type-token ratio is unreliable, so the +/// signal returns a neutral 0.5 instead of computing a ratio. pub const MIN_TOTAL_WORDS: usize = 5; /// Score in `[0.0, 1.0]` from the type-token ratio of `text`. diff --git a/src/openhuman/memory/tree/score/store.rs b/src/openhuman/memory/tree/score/store.rs index e8670e57b..3c8604185 100644 --- a/src/openhuman/memory/tree/score/store.rs +++ b/src/openhuman/memory/tree/score/store.rs @@ -371,6 +371,9 @@ pub fn lookup_entity( }) } +/// All distinct canonical entity ids associated with `node_id`, ordered by +/// score (desc) then recency. Used by topic-routing to pick which topic +/// trees a node should fan into. pub fn list_entity_ids_for_node(config: &Config, node_id: &str) -> Result> { with_connection(config, |conn| { let mut stmt = conn.prepare( diff --git a/src/openhuman/memory/tree/store.rs b/src/openhuman/memory/tree/store.rs index 9f50ea891..1c258e4b6 100644 --- a/src/openhuman/memory/tree/store.rs +++ b/src/openhuman/memory/tree/store.rs @@ -22,10 +22,15 @@ const DEFAULT_LIST_LIMIT: usize = 100; const MAX_LIST_LIMIT: usize = 10_000; const SQLITE_BUSY_TIMEOUT: Duration = Duration::from_secs(5); +/// Chunk lifecycle: freshly persisted, awaiting the async extract job. pub const CHUNK_STATUS_PENDING_EXTRACTION: &str = "pending_extraction"; +/// Chunk lifecycle: extract ran and the chunk passed admission. pub const CHUNK_STATUS_ADMITTED: &str = "admitted"; +/// Chunk lifecycle: appended to the L0 buffer of its source tree. pub const CHUNK_STATUS_BUFFERED: &str = "buffered"; +/// Chunk lifecycle: rolled into a sealed L1 summary. pub const CHUNK_STATUS_SEALED: &str = "sealed"; +/// Chunk lifecycle: rejected by the admission gate (too low signal). pub const CHUNK_STATUS_DROPPED: &str = "dropped"; const SCHEMA: &str = " @@ -457,6 +462,7 @@ pub fn count_chunks(config: &Config) -> Result { }) } +/// Set the lifecycle status column for `chunk_id`. See `CHUNK_STATUS_*`. pub fn set_chunk_lifecycle_status(config: &Config, chunk_id: &str, status: &str) -> Result<()> { with_connection(config, |conn| { set_chunk_lifecycle_status_conn(conn, chunk_id, status) @@ -471,6 +477,7 @@ pub(crate) fn set_chunk_lifecycle_status_tx( set_chunk_lifecycle_status_conn(tx, chunk_id, status) } +/// Read the lifecycle status column for `chunk_id`, or `None` if the row is absent. pub fn get_chunk_lifecycle_status(config: &Config, chunk_id: &str) -> Result> { with_connection(config, |conn| { get_chunk_lifecycle_status_conn(conn, chunk_id) @@ -495,6 +502,7 @@ fn get_chunk_lifecycle_status_conn(conn: &Connection, chunk_id: &str) -> Result< Ok(row) } +/// Count chunks currently sitting at a given lifecycle status (test/diagnostic helper). pub fn count_chunks_by_lifecycle_status(config: &Config, status: &str) -> Result { with_connection(config, |conn| { let n: i64 = conn.query_row( diff --git a/src/openhuman/memory/tree/store_tests.rs b/src/openhuman/memory/tree/store_tests.rs index 9c12a1493..c4fbe5051 100644 --- a/src/openhuman/memory/tree/store_tests.rs +++ b/src/openhuman/memory/tree/store_tests.rs @@ -1,3 +1,6 @@ +//! Unit tests for [`super`] — chunk upsert / list / lifecycle / embedding / +//! content-pointer accessors against a tempdir-backed SQLite store. + use super::*; use crate::openhuman::memory::tree::types::chunk_id; use chrono::TimeZone; diff --git a/src/openhuman/memory/tree/tree_global/README.md b/src/openhuman/memory/tree/tree_global/README.md new file mode 100644 index 000000000..7df3f2751 --- /dev/null +++ b/src/openhuman/memory/tree/tree_global/README.md @@ -0,0 +1,20 @@ +# Tree global + +Phase 3b (#709) — Global Activity Digest tree. One singleton tree per workspace whose L0 nodes are end-of-day digests folded across every active source tree, sealing upward into weekly (L1), monthly (L2), and yearly (L3) recaps. Reuses Phase 3a storage (`mem_tree_trees` / `mem_tree_summaries` / `mem_tree_buffers` with `kind='global'`) and the `Summariser` trait, but uses a **count-based** seal trigger aligned to the time axis instead of the source tree's token-budget gate. + +## Public surface + +- `pub fn get_or_create_global_tree` — `registry.rs` — singleton lookup keyed on `(kind=global, scope="global")`. +- `pub fn end_of_day_digest` / `pub enum DigestOutcome` — `digest.rs` — build one L0 daily node from cross-source material and cascade-seal upward. +- `pub fn append_daily_and_cascade` — `seal.rs` — append a daily summary id into the L0 buffer and run the count-based cascade. +- `pub fn recap` / `pub fn pick_level` / `pub struct RecapOutput` — `recap.rs` — pick the right level for a window duration and assemble the recap. +- `pub const WEEKLY_SEAL_THRESHOLD` / `pub const MONTHLY_SEAL_THRESHOLD` / `pub const YEARLY_SEAL_THRESHOLD` / `pub const GLOBAL_SCOPE` / `pub const GLOBAL_TOKEN_BUDGET` — `mod.rs`. + +## Files + +- `mod.rs` — module surface, threshold constants, scope literal. +- `registry.rs` — get-or-create for the singleton global tree. +- `digest.rs` — end-of-day digest builder; idempotent on re-runs for the same calendar day. +- `seal.rs` — count-based cascade seal (7 daily → 1 weekly → 1 monthly → 1 yearly). +- `recap.rs` — read-side level picker plus fallback when higher levels haven't sealed yet. +- `digest_tests.rs` — unit tests for the digest builder, included via `#[path]`. diff --git a/src/openhuman/memory/tree/global_tree/digest.rs b/src/openhuman/memory/tree/tree_global/digest.rs similarity index 92% rename from src/openhuman/memory/tree/global_tree/digest.rs rename to src/openhuman/memory/tree/tree_global/digest.rs index e82ae130e..002d46281 100644 --- a/src/openhuman/memory/tree/global_tree/digest.rs +++ b/src/openhuman/memory/tree/tree_global/digest.rs @@ -30,17 +30,17 @@ use crate::openhuman::memory::tree::content_store::{ atomic::stage_summary, paths::slugify_source_id, read as content_read, SummaryComposeInput, SummaryTreeKind, }; -use crate::openhuman::memory::tree::global_tree::registry::get_or_create_global_tree; -use crate::openhuman::memory::tree::global_tree::seal::append_daily_and_cascade; -use crate::openhuman::memory::tree::global_tree::GLOBAL_TOKEN_BUDGET; use crate::openhuman::memory::tree::score::embed::build_embedder_from_config; -use crate::openhuman::memory::tree::source_tree::registry::new_summary_id; -use crate::openhuman::memory::tree::source_tree::store; -use crate::openhuman::memory::tree::source_tree::summariser::{ +use crate::openhuman::memory::tree::store::with_connection; +use crate::openhuman::memory::tree::tree_global::registry::get_or_create_global_tree; +use crate::openhuman::memory::tree::tree_global::seal::append_daily_and_cascade; +use crate::openhuman::memory::tree::tree_global::GLOBAL_TOKEN_BUDGET; +use crate::openhuman::memory::tree::tree_source::registry::new_summary_id; +use crate::openhuman::memory::tree::tree_source::store; +use crate::openhuman::memory::tree::tree_source::summariser::{ Summariser, SummaryContext, SummaryInput, }; -use crate::openhuman::memory::tree::source_tree::types::{SummaryNode, Tree, TreeKind}; -use crate::openhuman::memory::tree::store::with_connection; +use crate::openhuman::memory::tree::tree_source::types::{SummaryNode, Tree, TreeKind}; /// Outcome of a single `end_of_day_digest` call — lets the caller decide /// whether to log skip details or propagate seal counts to telemetry. @@ -77,7 +77,7 @@ pub async fn end_of_day_digest( ) -> Result { let (day_start, day_end) = day_bounds_utc(day)?; log::info!( - "[global_tree::digest] end_of_day_digest day={} window=[{}, {})", + "[tree_global::digest] end_of_day_digest day={} window=[{}, {})", day, day_start, day_end @@ -89,7 +89,7 @@ pub async fn end_of_day_digest( // matches this day. if let Some(existing) = find_existing_daily(config, &global.id, day_start, day_end)? { log::info!( - "[global_tree::digest] daily already exists for {day} id={} — skipping", + "[tree_global::digest] daily already exists for {day} id={} — skipping", existing.id ); return Ok(DigestOutcome::Skipped { @@ -100,7 +100,7 @@ pub async fn end_of_day_digest( // Gather one contribution per active source tree. let source_trees = store::list_trees_by_kind(config, TreeKind::Source)?; log::debug!( - "[global_tree::digest] scanning {} source trees", + "[tree_global::digest] scanning {} source trees", source_trees.len() ); let mut inputs: Vec = Vec::with_capacity(source_trees.len()); @@ -108,7 +108,7 @@ pub async fn end_of_day_digest( match pick_source_contribution(config, source_tree, day_start, day_end)? { Some(inp) => { log::debug!( - "[global_tree::digest] source={} contributed id={} tokens={}", + "[tree_global::digest] source={} contributed id={} tokens={}", source_tree.scope, inp.id, inp.token_count @@ -117,7 +117,7 @@ pub async fn end_of_day_digest( } None => { log::debug!( - "[global_tree::digest] source={} had no material for {day}", + "[tree_global::digest] source={} had no material for {day}", source_tree.scope ); } @@ -126,7 +126,7 @@ pub async fn end_of_day_digest( if inputs.is_empty() { log::info!( - "[global_tree::digest] empty day — no source trees contributed material for {day}" + "[tree_global::digest] empty day — no source trees contributed material for {day}" ); return Ok(DigestOutcome::EmptyDay); } @@ -165,7 +165,7 @@ pub async fn end_of_day_digest( // seal time, so emergent themes don't need another extractor pass // here — global is a sink; union preserves "days that mentioned X" // retrieval without an extra LLM call. See LabelStrategy in - // source_tree::bucket_seal for the full design. + // tree_source::bucket_seal for the full design. let mut entities_set: BTreeSet = BTreeSet::new(); let mut topics_set: BTreeSet = BTreeSet::new(); for inp in &inputs { @@ -233,7 +233,7 @@ pub async fn end_of_day_digest( ) })?; log::debug!( - "[global_tree::digest] staged daily summary {} → {}", + "[tree_global::digest] staged daily summary {} → {}", daily.id, staged_daily.content_path ); @@ -261,7 +261,7 @@ pub async fn end_of_day_digest( })?; log::info!( - "[global_tree::digest] emitted daily id={} sources={} tokens={}", + "[tree_global::digest] emitted daily id={} sources={} tokens={}", daily.id, inputs.len(), daily.token_count @@ -371,7 +371,7 @@ fn pick_source_contribution( Some(n) => n, None => { log::warn!( - "[global_tree::digest] picked id={id} for tree={} but row missing — skipping", + "[tree_global::digest] picked id={id} for tree={} but row missing — skipping", source_tree.scope ); return Ok(None); @@ -385,7 +385,7 @@ fn pick_source_contribution( Ok(b) => b, Err(e) => { log::warn!( - "[global_tree::digest] read_summary_body failed for {} — using preview: {e:#}", + "[tree_global::digest] read_summary_body failed for {} — using preview: {e:#}", node.id ); // Non-fatal: fall back to preview for pre-MD-migration rows. diff --git a/src/openhuman/memory/tree/global_tree/digest_tests.rs b/src/openhuman/memory/tree/tree_global/digest_tests.rs similarity index 96% rename from src/openhuman/memory/tree/global_tree/digest_tests.rs rename to src/openhuman/memory/tree/tree_global/digest_tests.rs index 39822e8b3..2e528f12d 100644 --- a/src/openhuman/memory/tree/global_tree/digest_tests.rs +++ b/src/openhuman/memory/tree/tree_global/digest_tests.rs @@ -1,12 +1,16 @@ +//! Unit tests for [`super::digest`] — end-of-day digest emission, +//! cross-source contribution selection, idempotency on re-runs, and the +//! cascade-seal trigger for weekly/monthly/yearly levels. + use super::*; use crate::openhuman::memory::tree::content_store; -use crate::openhuman::memory::tree::source_tree::bucket_seal::{ +use crate::openhuman::memory::tree::store::upsert_chunks; +use crate::openhuman::memory::tree::tree_source::bucket_seal::{ append_leaf, LabelStrategy, LeafRef, }; -use crate::openhuman::memory::tree::source_tree::registry::get_or_create_source_tree; -use crate::openhuman::memory::tree::source_tree::summariser::inert::InertSummariser; -use crate::openhuman::memory::tree::source_tree::types::TreeStatus; -use crate::openhuman::memory::tree::store::upsert_chunks; +use crate::openhuman::memory::tree::tree_source::registry::get_or_create_source_tree; +use crate::openhuman::memory::tree::tree_source::summariser::inert::InertSummariser; +use crate::openhuman::memory::tree::tree_source::types::TreeStatus; use crate::openhuman::memory::tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; use tempfile::TempDir; diff --git a/src/openhuman/memory/tree/global_tree/mod.rs b/src/openhuman/memory/tree/tree_global/mod.rs similarity index 100% rename from src/openhuman/memory/tree/global_tree/mod.rs rename to src/openhuman/memory/tree/tree_global/mod.rs diff --git a/src/openhuman/memory/tree/global_tree/recap.rs b/src/openhuman/memory/tree/tree_global/recap.rs similarity index 95% rename from src/openhuman/memory/tree/global_tree/recap.rs rename to src/openhuman/memory/tree/tree_global/recap.rs index 1fbf9e089..d4aefe9ca 100644 --- a/src/openhuman/memory/tree/global_tree/recap.rs +++ b/src/openhuman/memory/tree/tree_global/recap.rs @@ -22,9 +22,9 @@ use anyhow::Result; use chrono::{DateTime, Duration, Utc}; use crate::openhuman::config::Config; -use crate::openhuman::memory::tree::global_tree::registry::get_or_create_global_tree; -use crate::openhuman::memory::tree::source_tree::store; -use crate::openhuman::memory::tree::source_tree::types::SummaryNode; +use crate::openhuman::memory::tree::tree_global::registry::get_or_create_global_tree; +use crate::openhuman::memory::tree::tree_source::store; +use crate::openhuman::memory::tree::tree_source::types::SummaryNode; /// Aggregated recap returned to the caller. #[derive(Debug, Clone)] @@ -49,7 +49,7 @@ pub struct RecapOutput { pub async fn recap(config: &Config, window: Duration) -> Result> { let target_level = pick_level(window); log::info!( - "[global_tree::recap] recap window={:?} target_level={}", + "[tree_global::recap] recap window={:?} target_level={}", window, target_level ); @@ -69,14 +69,14 @@ pub async fn recap(config: &Config, window: Duration) -> Result RecapOutput { mod tests { use super::*; use crate::openhuman::memory::tree::content_store; - use crate::openhuman::memory::tree::global_tree::digest::{end_of_day_digest, DigestOutcome}; - use crate::openhuman::memory::tree::source_tree::bucket_seal::{ + use crate::openhuman::memory::tree::store::upsert_chunks; + use crate::openhuman::memory::tree::tree_global::digest::{end_of_day_digest, DigestOutcome}; + use crate::openhuman::memory::tree::tree_source::bucket_seal::{ append_leaf, LabelStrategy, LeafRef, }; - use crate::openhuman::memory::tree::source_tree::registry::get_or_create_source_tree; - use crate::openhuman::memory::tree::source_tree::summariser::inert::InertSummariser; - use crate::openhuman::memory::tree::store::upsert_chunks; + use crate::openhuman::memory::tree::tree_source::registry::get_or_create_source_tree; + use crate::openhuman::memory::tree::tree_source::summariser::inert::InertSummariser; use crate::openhuman::memory::tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; use tempfile::TempDir; diff --git a/src/openhuman/memory/tree/global_tree/registry.rs b/src/openhuman/memory/tree/tree_global/registry.rs similarity index 88% rename from src/openhuman/memory/tree/global_tree/registry.rs rename to src/openhuman/memory/tree/tree_global/registry.rs index c71452602..fbf3453e6 100644 --- a/src/openhuman/memory/tree/global_tree/registry.rs +++ b/src/openhuman/memory/tree/tree_global/registry.rs @@ -3,16 +3,16 @@ //! Unlike source trees (one per `source_id`) the global tree is a true //! singleton per workspace — scope is the literal string `"global"`. The //! lookup and race-recovery pattern otherwise mirrors -//! `source_tree::registry::get_or_create_source_tree`. +//! `tree_source::registry::get_or_create_source_tree`. use anyhow::Result; use chrono::Utc; use uuid::Uuid; use crate::openhuman::config::Config; -use crate::openhuman::memory::tree::global_tree::GLOBAL_SCOPE; -use crate::openhuman::memory::tree::source_tree::store; -use crate::openhuman::memory::tree::source_tree::types::{Tree, TreeKind, TreeStatus}; +use crate::openhuman::memory::tree::tree_global::GLOBAL_SCOPE; +use crate::openhuman::memory::tree::tree_source::store; +use crate::openhuman::memory::tree::tree_source::types::{Tree, TreeKind, TreeStatus}; /// Return the workspace's singleton global tree, creating it lazily on /// first call. Safe to call on every ingest; subsequent calls short-circuit @@ -20,7 +20,7 @@ use crate::openhuman::memory::tree::source_tree::types::{Tree, TreeKind, TreeSta pub fn get_or_create_global_tree(config: &Config) -> Result { if let Some(existing) = store::get_tree_by_scope(config, TreeKind::Global, GLOBAL_SCOPE)? { log::debug!( - "[global_tree::registry] found global tree id={}", + "[tree_global::registry] found global tree id={}", existing.id ); return Ok(existing); @@ -38,14 +38,14 @@ pub fn get_or_create_global_tree(config: &Config) -> Result { }; match store::insert_tree(config, &tree) { Ok(()) => { - log::info!("[global_tree::registry] created global tree id={}", tree.id); + log::info!("[tree_global::registry] created global tree id={}", tree.id); Ok(tree) } Err(err) if is_unique_violation(&err) => { // Another caller beat us to it between our initial lookup and // the insert. The UNIQUE(kind, scope) index caught it — // re-query and return the winner. - log::debug!("[global_tree::registry] UNIQUE race for global tree — re-querying"); + log::debug!("[tree_global::registry] UNIQUE race for global tree — re-querying"); store::get_tree_by_scope(config, TreeKind::Global, GLOBAL_SCOPE)?.ok_or_else(|| { anyhow::anyhow!( "UNIQUE violation on global-tree insert but no row found on re-query" @@ -57,7 +57,7 @@ pub fn get_or_create_global_tree(config: &Config) -> Result { } /// True when `err` wraps a SQLite UNIQUE constraint violation. Duplicated -/// from `source_tree::registry` to keep this module self-contained; the +/// from `tree_source::registry` to keep this module self-contained; the /// two copies are ~5 lines and have the same shape. fn is_unique_violation(err: &anyhow::Error) -> bool { if let Some(rusqlite::Error::SqliteFailure(sqlite_err, _)) = @@ -106,7 +106,7 @@ mod tests { fn race_recovery_returns_existing_row() { // Pre-seed a global tree so the second `get_or_create` path exercises // the normal lookup branch; the UNIQUE-race branch is covered by the - // shared `is_unique_violation` contract in `source_tree::registry`. + // shared `is_unique_violation` contract in `tree_source::registry`. let (_tmp, cfg) = test_config(); let pre_existing = Tree { id: "global:preexisting".into(), diff --git a/src/openhuman/memory/tree/global_tree/seal.rs b/src/openhuman/memory/tree/tree_global/seal.rs similarity index 95% rename from src/openhuman/memory/tree/global_tree/seal.rs rename to src/openhuman/memory/tree/tree_global/seal.rs index 3af02a58a..cb8f5945c 100644 --- a/src/openhuman/memory/tree/global_tree/seal.rs +++ b/src/openhuman/memory/tree/tree_global/seal.rs @@ -6,7 +6,7 @@ //! tree aligned to the time axis (day / week / month / year) so //! window-scoped recap queries can map a duration to a level deterministically. //! -//! Reuses Phase 3a storage primitives from `source_tree::store` without +//! Reuses Phase 3a storage primitives from `tree_source::store` without //! their token-budget cascade logic — all global seals route through //! `mem_tree_summaries` on both sides (children and output), since even L0 //! is a sealed summary node rather than a raw chunk. @@ -20,17 +20,17 @@ use crate::openhuman::config::Config; use crate::openhuman::memory::tree::content_store::{ atomic::stage_summary, SummaryComposeInput, SummaryTreeKind, }; -use crate::openhuman::memory::tree::global_tree::{ +use crate::openhuman::memory::tree::score::embed::build_embedder_from_config; +use crate::openhuman::memory::tree::store::with_connection; +use crate::openhuman::memory::tree::tree_global::{ GLOBAL_TOKEN_BUDGET, MONTHLY_SEAL_THRESHOLD, WEEKLY_SEAL_THRESHOLD, YEARLY_SEAL_THRESHOLD, }; -use crate::openhuman::memory::tree::score::embed::build_embedder_from_config; -use crate::openhuman::memory::tree::source_tree::registry::new_summary_id; -use crate::openhuman::memory::tree::source_tree::store; -use crate::openhuman::memory::tree::source_tree::summariser::{ +use crate::openhuman::memory::tree::tree_source::registry::new_summary_id; +use crate::openhuman::memory::tree::tree_source::store; +use crate::openhuman::memory::tree::tree_source::summariser::{ Summariser, SummaryContext, SummaryInput, }; -use crate::openhuman::memory::tree::source_tree::types::{Buffer, SummaryNode, Tree, TreeKind}; -use crate::openhuman::memory::tree::store::with_connection; +use crate::openhuman::memory::tree::tree_source::types::{Buffer, SummaryNode, Tree, TreeKind}; /// Hard cap on cascade depth — mirrors the source-tree constant. L0→L1→L2→L3 /// is only 3 hops so we have ample slack. @@ -49,7 +49,7 @@ pub async fn append_daily_and_cascade( summariser: &dyn Summariser, ) -> Result> { log::debug!( - "[global_tree::seal] append_daily tree_id={} daily_id={} tokens={}", + "[tree_global::seal] append_daily tree_id={} daily_id={} tokens={}", tree.id, daily_summary.id, daily_summary.token_count @@ -83,7 +83,7 @@ fn append_to_buffer( let mut buf = store::get_buffer_conn(&tx, tree_id, level)?; if buf.item_ids.iter().any(|existing| existing == item_id) { log::debug!( - "[global_tree::seal] append_to_buffer: {item_id} already in buffer \ + "[tree_global::seal] append_to_buffer: {item_id} already in buffer \ tree_id={tree_id} level={level} — no-op" ); return Ok(()); @@ -116,7 +116,7 @@ async fn cascade_seals( let buf = store::get_buffer(config, &tree.id, level)?; if !should_seal(&buf, level) { log::debug!( - "[global_tree::seal] cascade done tree_id={} stop_level={} count={}", + "[tree_global::seal] cascade done tree_id={} stop_level={} count={}", tree.id, level, buf.item_ids.len() @@ -158,7 +158,7 @@ async fn seal_one_level( let inputs = hydrate_summary_inputs(config, &buf.item_ids)?; if inputs.is_empty() { anyhow::bail!( - "[global_tree::seal] refused to seal empty buffer tree_id={} level={}", + "[tree_global::seal] refused to seal empty buffer tree_id={} level={}", tree.id, level ); @@ -281,7 +281,7 @@ async fn seal_one_level( ) })?; log::debug!( - "[global_tree::seal] staged summary {} → {}", + "[tree_global::seal] staged summary {} → {}", node.id, staged_global.content_path ); @@ -366,7 +366,7 @@ async fn seal_one_level( })?; log::info!( - "[global_tree::seal] sealed tree_id={} level={}→{} summary_id={} children={}", + "[tree_global::seal] sealed tree_id={} level={}→{} summary_id={} children={}", tree.id, level, target_level, @@ -387,7 +387,7 @@ fn hydrate_summary_inputs(config: &Config, summary_ids: &[String]) -> Result n, None => { log::warn!( - "[global_tree::seal] hydrate_summary_inputs: missing summary {id} — skipping" + "[tree_global::seal] hydrate_summary_inputs: missing summary {id} — skipping" ); continue; } @@ -409,8 +409,8 @@ fn hydrate_summary_inputs(config: &Config, summary_ids: &[String]) -> Result Result> { log::debug!( - "[source_tree::bucket_seal] append_leaf tree_id={} leaf_id={} tokens={} strategy={:?}", + "[tree_source::bucket_seal] append_leaf tree_id={} leaf_id={} tokens={} strategy={:?}", tree.id, leaf.chunk_id, leaf.token_count, @@ -218,7 +218,7 @@ fn append_to_buffer( // stays on first-seen. if buf.item_ids.iter().any(|existing| existing == item_id) { log::debug!( - "[source_tree::bucket_seal] append_to_buffer: {item_id} already in buffer \ + "[tree_source::bucket_seal] append_to_buffer: {item_id} already in buffer \ tree_id={tree_id} level={level} — no-op" ); return Ok(()); @@ -270,7 +270,7 @@ pub async fn cascade_all_from( if !forced && !should_seal(&buf) { log::debug!( - "[source_tree::bucket_seal] cascade done tree_id={} stop_level={} token_sum={}", + "[tree_source::bucket_seal] cascade done tree_id={} stop_level={} token_sum={}", tree.id, level, buf.token_sum @@ -279,7 +279,7 @@ pub async fn cascade_all_from( } if buf.is_empty() { log::debug!( - "[source_tree::bucket_seal] cascade hit empty buffer tree_id={} level={} — stopping", + "[tree_source::bucket_seal] cascade hit empty buffer tree_id={} level={} — stopping", tree.id, level ); @@ -350,7 +350,7 @@ pub(crate) async fn seal_one_level( let inputs = hydrate_inputs(config, level, &buf.item_ids)?; if inputs.is_empty() { anyhow::bail!( - "[source_tree::bucket_seal] refused to seal empty buffer tree_id={} level={}", + "[tree_source::bucket_seal] refused to seal empty buffer tree_id={} level={}", tree.id, level ); @@ -411,7 +411,7 @@ pub(crate) async fn seal_one_level( // even at 4× tokenizer ratio. let embed_input = truncate_for_embed(&output.content, 1_000); log::info!( - "[source_tree::bucket_seal] embed input: original_chars={} truncated_chars={}", + "[tree_source::bucket_seal] embed input: original_chars={} truncated_chars={}", output.content.len(), embed_input.len() ); @@ -422,7 +422,7 @@ pub(crate) async fn seal_one_level( ) })?; log::debug!( - "[source_tree::bucket_seal] embedded summary tree_id={} level={}→{} bytes={} provider={}", + "[tree_source::bucket_seal] embedded summary tree_id={} level={}→{} bytes={} provider={}", tree.id, level, target_level, @@ -523,7 +523,7 @@ pub(crate) async fn seal_one_level( ) })?; log::debug!( - "[source_tree::bucket_seal] staged summary {} → {}", + "[tree_source::bucket_seal] staged summary {} → {}", node.id, staged.content_path ); @@ -657,7 +657,7 @@ pub(crate) async fn seal_one_level( })?; log::info!( - "[source_tree::bucket_seal] sealed tree_id={} level={}→{} summary_id={} children={}", + "[tree_source::bucket_seal] sealed tree_id={} level={}→{} summary_id={} children={}", tree.id, level, target_level, @@ -718,7 +718,7 @@ fn hydrate_leaf_inputs(config: &Config, chunk_ids: &[String]) -> Result c, None => { log::warn!( - "[source_tree::bucket_seal] hydrate_leaf_inputs: missing chunk {id} — skipping" + "[tree_source::bucket_seal] hydrate_leaf_inputs: missing chunk {id} — skipping" ); continue; } @@ -739,7 +739,7 @@ fn hydrate_leaf_inputs(config: &Config, chunk_ids: &[String]) -> Result Result n, None => { log::warn!( - "[source_tree::bucket_seal] hydrate_summary_inputs: missing summary {id} — skipping" + "[tree_source::bucket_seal] hydrate_summary_inputs: missing summary {id} — skipping" ); continue; } @@ -773,7 +773,7 @@ fn hydrate_summary_inputs(config: &Config, summary_ids: &[String]) -> Result t, None => { log::warn!( - "[source_tree::flush] orphan buffer tree_id={} level={}", + "[tree_source::flush] orphan buffer tree_id={} level={}", buf.tree_id, buf.level ); @@ -89,10 +89,10 @@ pub async fn force_flush_tree( mod tests { use super::*; use crate::openhuman::memory::tree::content_store; - use crate::openhuman::memory::tree::source_tree::bucket_seal::{append_leaf, LeafRef}; - use crate::openhuman::memory::tree::source_tree::registry::get_or_create_source_tree; - use crate::openhuman::memory::tree::source_tree::summariser::inert::InertSummariser; use crate::openhuman::memory::tree::store::upsert_chunks; + use crate::openhuman::memory::tree::tree_source::bucket_seal::{append_leaf, LeafRef}; + use crate::openhuman::memory::tree::tree_source::registry::get_or_create_source_tree; + use crate::openhuman::memory::tree::tree_source::summariser::inert::InertSummariser; use crate::openhuman::memory::tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; use tempfile::TempDir; diff --git a/src/openhuman/memory/tree/source_tree/mod.rs b/src/openhuman/memory/tree/tree_source/mod.rs similarity index 100% rename from src/openhuman/memory/tree/source_tree/mod.rs rename to src/openhuman/memory/tree/tree_source/mod.rs diff --git a/src/openhuman/memory/tree/source_tree/registry.rs b/src/openhuman/memory/tree/tree_source/registry.rs similarity index 95% rename from src/openhuman/memory/tree/source_tree/registry.rs rename to src/openhuman/memory/tree/tree_source/registry.rs index 430c014cf..7c0c69939 100644 --- a/src/openhuman/memory/tree/source_tree/registry.rs +++ b/src/openhuman/memory/tree/tree_source/registry.rs @@ -10,8 +10,8 @@ use chrono::Utc; use uuid::Uuid; use crate::openhuman::config::Config; -use crate::openhuman::memory::tree::source_tree::store; -use crate::openhuman::memory::tree::source_tree::types::{Tree, TreeKind, TreeStatus}; +use crate::openhuman::memory::tree::tree_source::store; +use crate::openhuman::memory::tree::tree_source::types::{Tree, TreeKind, TreeStatus}; /// Look up the source tree for `scope`, or create a new one. /// @@ -21,7 +21,7 @@ use crate::openhuman::memory::tree::source_tree::types::{Tree, TreeKind, TreeSta pub fn get_or_create_source_tree(config: &Config, scope: &str) -> Result { if let Some(existing) = store::get_tree_by_scope(config, TreeKind::Source, scope)? { log::debug!( - "[source_tree::registry] found tree id={} scope={}", + "[tree_source::registry] found tree id={} scope={}", existing.id, scope ); @@ -41,7 +41,7 @@ pub fn get_or_create_source_tree(config: &Config, scope: &str) -> Result { match store::insert_tree(config, &tree) { Ok(()) => { log::info!( - "[source_tree::registry] created source tree id={} scope={}", + "[tree_source::registry] created source tree id={} scope={}", tree.id, scope ); @@ -52,7 +52,7 @@ pub fn get_or_create_source_tree(config: &Config, scope: &str) -> Result { // between our initial lookup and this insert. UNIQUE(kind, // scope) rejected our row; re-query and return the winner. log::debug!( - "[source_tree::registry] UNIQUE race for scope={} — re-querying", + "[tree_source::registry] UNIQUE race for scope={} — re-querying", scope ); store::get_tree_by_scope(config, TreeKind::Source, scope)?.ok_or_else(|| { diff --git a/src/openhuman/memory/tree/source_tree/store.rs b/src/openhuman/memory/tree/tree_source/store.rs similarity index 99% rename from src/openhuman/memory/tree/source_tree/store.rs rename to src/openhuman/memory/tree/tree_source/store.rs index 36fd9f126..4494dd19d 100644 --- a/src/openhuman/memory/tree/source_tree/store.rs +++ b/src/openhuman/memory/tree/tree_source/store.rs @@ -23,10 +23,10 @@ use rusqlite::{params, Connection, OptionalExtension, Transaction}; use crate::openhuman::config::Config; use crate::openhuman::memory::tree::content_store::StagedSummary; use crate::openhuman::memory::tree::score::embed::{decode_optional_blob, pack_checked}; -use crate::openhuman::memory::tree::source_tree::types::{ +use crate::openhuman::memory::tree::store::with_connection; +use crate::openhuman::memory::tree::tree_source::types::{ Buffer, SummaryNode, Tree, TreeKind, TreeStatus, }; -use crate::openhuman::memory::tree::store::with_connection; fn ms_to_utc(ms: i64) -> rusqlite::Result> { Utc.timestamp_millis_opt(ms).single().ok_or_else(|| { @@ -264,7 +264,7 @@ pub fn set_summary_embedding( )?; if changed == 0 { log::warn!( - "[source_tree::store] set_summary_embedding: no row for summary_id={summary_id}" + "[tree_source::store] set_summary_embedding: no row for summary_id={summary_id}" ); } Ok(changed) diff --git a/src/openhuman/memory/tree/source_tree/store_tests.rs b/src/openhuman/memory/tree/tree_source/store_tests.rs similarity index 97% rename from src/openhuman/memory/tree/source_tree/store_tests.rs rename to src/openhuman/memory/tree/tree_source/store_tests.rs index 6d98ddbd6..71ba0e407 100644 --- a/src/openhuman/memory/tree/source_tree/store_tests.rs +++ b/src/openhuman/memory/tree/tree_source/store_tests.rs @@ -1,3 +1,6 @@ +//! Unit tests for [`super::store`] — round-trip tree / summary / buffer +//! persistence including embedding blob handling and stale-buffer queries. + use super::*; use tempfile::TempDir; diff --git a/src/openhuman/memory/tree/tree_source/summariser/README.md b/src/openhuman/memory/tree/tree_source/summariser/README.md new file mode 100644 index 000000000..e29cc3472 --- /dev/null +++ b/src/openhuman/memory/tree/tree_source/summariser/README.md @@ -0,0 +1,16 @@ +# Summariser + +Summariser trait and implementations used by the bucket-seal cascade. A summariser folds N buffered items into one sealed [`SummaryOutput`]; the seal machinery (bucket budgeting, persistence, label resolution) lives in [`super::bucket_seal`] and is unaffected by the choice of implementation. + +## Public surface + +- `pub trait Summariser` / `pub struct SummaryInput` / `pub struct SummaryContext` / `pub struct SummaryOutput` — `mod.rs` — async trait + IO types. +- `pub fn build_summariser` — `mod.rs` — picks the implementation based on `Config::memory_tree.llm_summariser_*`. Returns the LLM summariser when both endpoint and model are set, otherwise the inert fallback. +- `pub struct InertSummariser` — `inert.rs` — deterministic concat-and-truncate fallback. `entities` and `topics` are intentionally empty (an honest stub — derived labels are an LLM concern). +- `pub struct LlmSummariser` / `pub struct LlmSummariserConfig` — `llm.rs` — Ollama `/api/chat` peer of `score::extract::llm`. Soft-falls-back to inert on every error so seal cascades never abort. + +## Files + +- `mod.rs` — trait, IO types, and the `build_summariser` factory. +- `inert.rs` — deterministic fallback, used in tests and when no LLM is configured. +- `llm.rs` — Ollama-backed implementation with prompt construction, per-input clamping for `num_ctx` safety, and post-generation budget enforcement. diff --git a/src/openhuman/memory/tree/source_tree/summariser/inert.rs b/src/openhuman/memory/tree/tree_source/summariser/inert.rs similarity index 92% rename from src/openhuman/memory/tree/source_tree/summariser/inert.rs rename to src/openhuman/memory/tree/tree_source/summariser/inert.rs index f6f897113..5d1f05608 100644 --- a/src/openhuman/memory/tree/source_tree/summariser/inert.rs +++ b/src/openhuman/memory/tree/tree_source/summariser/inert.rs @@ -13,7 +13,7 @@ use anyhow::Result; use async_trait::async_trait; -use crate::openhuman::memory::tree::source_tree::summariser::{ +use crate::openhuman::memory::tree::tree_source::summariser::{ Summariser, SummaryContext, SummaryInput, SummaryOutput, }; use crate::openhuman::memory::tree::types::approx_token_count; @@ -22,9 +22,14 @@ use crate::openhuman::memory::tree::types::approx_token_count; /// provenance visible to a human reading the raw summary. const PROVENANCE_PREFIX: &str = "— "; +/// Deterministic, dependency-free [`Summariser`] implementation that +/// concatenates inputs and truncates to budget. See module docs for why +/// `entities` and `topics` are intentionally empty. pub struct InertSummariser; impl InertSummariser { + /// Construct a fresh summariser. Stateless — multiple instances behave + /// identically. pub fn new() -> Self { Self } @@ -56,7 +61,7 @@ impl Summariser for InertSummariser { let (content, token_count) = truncate_to_budget(&joined, ctx.token_budget); log::debug!( - "[source_tree::summariser::inert] sealed tree_id={} level={} inputs={} tokens={} \ + "[tree_source::summariser::inert] sealed tree_id={} level={} inputs={} tokens={} \ entities=0 topics=0 (honest stub — LLM summariser derives these)", ctx.tree_id, ctx.target_level, @@ -92,7 +97,7 @@ fn truncate_to_budget(text: &str, budget: u32) -> (String, u32) { #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory::tree::source_tree::types::TreeKind; + use crate::openhuman::memory::tree::tree_source::types::TreeKind; use chrono::Utc; fn sample_input(id: &str, content: &str, entities: &[&str]) -> SummaryInput { diff --git a/src/openhuman/memory/tree/source_tree/summariser/llm.rs b/src/openhuman/memory/tree/tree_source/summariser/llm.rs similarity index 96% rename from src/openhuman/memory/tree/source_tree/summariser/llm.rs rename to src/openhuman/memory/tree/tree_source/summariser/llm.rs index 8f9f31135..0580d8a33 100644 --- a/src/openhuman/memory/tree/source_tree/summariser/llm.rs +++ b/src/openhuman/memory/tree/tree_source/summariser/llm.rs @@ -45,7 +45,7 @@ use crate::openhuman::memory::tree::types::approx_token_count; /// Two constraints set this: /// /// 1. The downstream embedder (`nomic-embed-text-v1.5`) accepts up to -/// 8192 tokens, and Phase 4 (`source_tree::bucket_seal`) embeds the +/// 8192 tokens, and Phase 4 (`tree_source::bucket_seal`) embeds the /// summary right after we produce it. An overshoot returns HTTP 500 /// and rolls back the whole seal transaction. /// 2. Empirically, small instruction-tuned models running locally @@ -112,6 +112,8 @@ pub struct LlmSummariser { } impl LlmSummariser { + /// Build a summariser around `cfg`. Returns an error only if the HTTP + /// client fails to construct (timeout / TLS init). pub fn new(cfg: LlmSummariserConfig) -> Result { let http = Client::builder() .timeout(cfg.timeout) @@ -191,7 +193,7 @@ impl Summariser for LlmSummariser { let body = build_user_prompt(inputs, per_input_cap); if body.trim().is_empty() { log::debug!( - "[source_tree::summariser::llm] empty prompt body (no non-blank inputs) \ + "[tree_source::summariser::llm] empty prompt body (no non-blank inputs) \ tree_id={} level={} — returning empty summary", ctx.tree_id, ctx.target_level @@ -208,7 +210,7 @@ impl Summariser for LlmSummariser { let req = self.build_request(&body, effective_budget); log::debug!( - "[source_tree::summariser::llm] POST {url} model={} tree_id={} level={} \ + "[tree_source::summariser::llm] POST {url} model={} tree_id={} level={} \ inputs={} budget={}", self.cfg.model, ctx.tree_id, @@ -221,7 +223,7 @@ impl Summariser for LlmSummariser { Ok(r) => r, Err(e) => { log::warn!( - "[source_tree::summariser::llm] transport failure to {url}: {e} — \ + "[tree_source::summariser::llm] transport failure to {url}: {e} — \ falling back to inert summariser for tree_id={} level={}", ctx.tree_id, ctx.target_level @@ -234,7 +236,7 @@ impl Summariser for LlmSummariser { let status = resp.status(); let body = resp.text().await.unwrap_or_default(); log::warn!( - "[source_tree::summariser::llm] ollama non-success status {status} \ + "[tree_source::summariser::llm] ollama non-success status {status} \ tree_id={} level={}: {} — falling back to inert", ctx.tree_id, ctx.target_level, @@ -247,7 +249,7 @@ impl Summariser for LlmSummariser { Ok(v) => v, Err(e) => { log::warn!( - "[source_tree::summariser::llm] response not Ollama-shaped JSON: {e} — \ + "[tree_source::summariser::llm] response not Ollama-shaped JSON: {e} — \ falling back to inert for tree_id={} level={}", ctx.tree_id, ctx.target_level @@ -260,7 +262,7 @@ impl Summariser for LlmSummariser { Ok(v) => v, Err(e) => { log::warn!( - "[source_tree::summariser::llm] model returned non-JSON or wrong-shape \ + "[tree_source::summariser::llm] model returned non-JSON or wrong-shape \ body: {e}; content was: {} — falling back to inert", truncate_for_log(&envelope.message.content, 400) ); @@ -270,7 +272,7 @@ impl Summariser for LlmSummariser { let (content, token_count) = clamp_to_budget(&parsed.summary, effective_budget); log::debug!( - "[source_tree::summariser::llm] sealed tree_id={} level={} inputs={} tokens={}", + "[tree_source::summariser::llm] sealed tree_id={} level={} inputs={} tokens={}", ctx.tree_id, ctx.target_level, inputs.len(), @@ -397,7 +399,7 @@ struct LlmSummaryOutput { #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory::tree::source_tree::types::TreeKind; + use crate::openhuman::memory::tree::tree_source::types::TreeKind; use chrono::Utc; fn sample_input(id: &str, content: &str) -> SummaryInput { diff --git a/src/openhuman/memory/tree/source_tree/summariser/mod.rs b/src/openhuman/memory/tree/tree_source/summariser/mod.rs similarity index 94% rename from src/openhuman/memory/tree/source_tree/summariser/mod.rs rename to src/openhuman/memory/tree/tree_source/summariser/mod.rs index 8dd510083..372998dea 100644 --- a/src/openhuman/memory/tree/source_tree/summariser/mod.rs +++ b/src/openhuman/memory/tree/tree_source/summariser/mod.rs @@ -13,7 +13,7 @@ use chrono::{DateTime, Utc}; use std::sync::Arc; use crate::openhuman::config::Config; -use crate::openhuman::memory::tree::source_tree::types::TreeKind; +use crate::openhuman::memory::tree::tree_source::types::TreeKind; pub mod inert; pub mod llm; @@ -90,7 +90,7 @@ pub fn build_summariser(config: &Config) -> Arc { let (Some(endpoint), Some(model)) = (endpoint, model) else { log::debug!( - "[source_tree::summariser] llm_summariser not configured — using InertSummariser" + "[tree_source::summariser] llm_summariser not configured — using InertSummariser" ); return Arc::new(inert::InertSummariser::new()); }; @@ -111,7 +111,7 @@ pub fn build_summariser(config: &Config) -> Arc { match llm::LlmSummariser::new(cfg) { Ok(s) => { log::info!( - "[source_tree::summariser] using LlmSummariser endpoint={} model={} timeout_ms={}", + "[tree_source::summariser] using LlmSummariser endpoint={} model={} timeout_ms={}", endpoint, model, timeout_ms @@ -120,7 +120,7 @@ pub fn build_summariser(config: &Config) -> Arc { } Err(err) => { log::warn!( - "[source_tree::summariser] LlmSummariser construction failed: {err:#} — \ + "[tree_source::summariser] LlmSummariser construction failed: {err:#} — \ falling back to InertSummariser" ); Arc::new(inert::InertSummariser::new()) diff --git a/src/openhuman/memory/tree/source_tree/types.rs b/src/openhuman/memory/tree/tree_source/types.rs similarity index 95% rename from src/openhuman/memory/tree/source_tree/types.rs rename to src/openhuman/memory/tree/tree_source/types.rs index 2a4d23082..2b3b72790 100644 --- a/src/openhuman/memory/tree/source_tree/types.rs +++ b/src/openhuman/memory/tree/tree_source/types.rs @@ -26,6 +26,7 @@ pub enum TreeKind { } impl TreeKind { + /// Stable lowercase form used in SQL discriminator columns and ids. pub fn as_str(self) -> &'static str { match self { Self::Source => "source", @@ -34,6 +35,8 @@ impl TreeKind { } } + /// Inverse of [`Self::as_str`] — parse back from a discriminator + /// string. Errors on unknown variants. pub fn parse(s: &str) -> Result { match s { "source" => Ok(Self::Source), @@ -54,6 +57,7 @@ pub enum TreeStatus { } impl TreeStatus { + /// Stable lowercase form used as the SQL discriminator value. pub fn as_str(self) -> &'static str { match self { Self::Active => "active", @@ -61,6 +65,7 @@ impl TreeStatus { } } + /// Inverse of [`Self::as_str`] — parse from the SQL discriminator. pub fn parse(s: &str) -> Result { match s { "active" => Ok(Self::Active), @@ -158,6 +163,7 @@ impl Buffer { } } + /// True when the buffer holds no pending items. pub fn is_empty(&self) -> bool { self.item_ids.is_empty() } diff --git a/src/openhuman/memory/tree/tree_topic/README.md b/src/openhuman/memory/tree/tree_topic/README.md new file mode 100644 index 000000000..88cfd673f --- /dev/null +++ b/src/openhuman/memory/tree/tree_topic/README.md @@ -0,0 +1,24 @@ +# Tree topic + +Phase 3c (#709) — per-entity topic trees with lazy materialisation. A topic tree groups every chunk mentioning one canonical entity, regardless of source. Trees are spawned only when an entity's hotness crosses `TOPIC_CREATION_THRESHOLD`; once spawned they receive new leaves via the ingest path alongside the per-source tree. Tree mechanics (buffer / seal / cascade) reuse `super::tree_source::bucket_seal` end-to-end — only the hotness layer and the per-entity fan-out live here. + +## Public surface + +- `pub fn maybe_spawn_topic_tree` / `pub fn force_recompute` / `pub enum SpawnOutcome` — `curator.rs` — bumps hotness counters per ingest and spawns + backfills on cadence. +- `pub fn route_leaf_to_topic_trees` — `routing.rs` — fan-out hook called by ingest after the source-tree append. +- `pub fn backfill_topic_tree` / `pub fn backfill_topic_tree_at` / `pub const BACKFILL_WINDOW_DAYS` — `backfill.rs` — hydrate a freshly spawned tree from the entity index. +- `pub fn get_or_create_topic_tree` / `pub fn force_create_topic_tree` / `pub fn list_topic_trees` / `pub fn archive_topic_tree` — `registry.rs`. +- `pub fn hotness` / `pub fn hotness_at` / `pub fn recency_decay` — `hotness.rs` — pure arithmetic over the entity stats. +- `pub struct EntityIndexStats` / `pub struct HotnessCounters` / `pub const TOPIC_CREATION_THRESHOLD` / `pub const TOPIC_ARCHIVE_THRESHOLD` / `pub const TOPIC_RECHECK_EVERY` — `types.rs`. +- `pub fn get` / `pub fn get_or_fresh` / `pub fn upsert` / `pub fn distinct_sources_for` / `pub fn count` — `store.rs` — `mem_tree_entity_hotness` persistence. + +## Files + +- `mod.rs` — module surface and re-exports. +- `types.rs` — `EntityIndexStats`, `HotnessCounters`, threshold / cadence constants. +- `hotness.rs` — pure hotness arithmetic; deterministic, unit-testable in isolation. +- `store.rs` — persistence for the per-entity counter row and `distinct_sources` aggregation. +- `curator.rs` — counter bumps, hotness recompute on cadence, spawn-and-backfill on first threshold crossing. +- `routing.rs` — per-leaf fan-out into matching active topic trees plus a curator tick. +- `registry.rs` — get-or-create / archive primitives for topic trees in `mem_tree_trees` (`kind='topic'`). +- `backfill.rs` — windowed (30 d) backfill from `mem_tree_entity_index` after spawn. diff --git a/src/openhuman/memory/tree/topic_tree/backfill.rs b/src/openhuman/memory/tree/tree_topic/backfill.rs similarity index 94% rename from src/openhuman/memory/tree/topic_tree/backfill.rs rename to src/openhuman/memory/tree/tree_topic/backfill.rs index 5e241347e..ac1e9a43d 100644 --- a/src/openhuman/memory/tree/topic_tree/backfill.rs +++ b/src/openhuman/memory/tree/tree_topic/backfill.rs @@ -9,7 +9,7 @@ //! //! ## Why bounded by hotness window //! -//! Hotness uses a 30-day recency decay (see `topic_tree::hotness`). Leaves +//! Hotness uses a 30-day recency decay (see `tree_topic::hotness`). Leaves //! older than 30 days contribute zero to current hotness, so by definition //! they cannot be the reason a tree is spawning *now*. Including them //! bloats the spawn latency, wastes summariser LLM calls, and amplifies @@ -28,19 +28,19 @@ use chrono::Utc; use crate::openhuman::config::Config; use crate::openhuman::memory::tree::score::store::lookup_entity; -use crate::openhuman::memory::tree::source_tree::bucket_seal::{ +use crate::openhuman::memory::tree::store::get_chunk; +use crate::openhuman::memory::tree::tree_source::bucket_seal::{ append_leaf, LabelStrategy, LeafRef, }; -use crate::openhuman::memory::tree::source_tree::summariser::Summariser; -use crate::openhuman::memory::tree::source_tree::types::Tree; -use crate::openhuman::memory::tree::store::get_chunk; +use crate::openhuman::memory::tree::tree_source::summariser::Summariser; +use crate::openhuman::memory::tree::tree_source::types::Tree; use crate::openhuman::memory::tree::util::redact::redact; /// Max leaves to pull from the entity index during backfill. A hard cap /// keeps initial spawn latency bounded even for very active entities. const BACKFILL_LIMIT: usize = 500; -/// Backfill window in days — matches `topic_tree::hotness::recency_decay`'s +/// Backfill window in days — matches `tree_topic::hotness::recency_decay`'s /// hard cliff. Leaves older than this contribute zero to current hotness /// so they cannot have driven the spawn decision. pub const BACKFILL_WINDOW_DAYS: i64 = 30; @@ -79,7 +79,7 @@ pub async fn backfill_topic_tree_at( ) -> Result { let cutoff_ms = now_ms.saturating_sub(BACKFILL_WINDOW_DAYS.saturating_mul(DAY_MS)); log::info!( - "[topic_tree::backfill] start entity_id_hash={} tree_id={} window_days={} cutoff_ms={}", + "[tree_topic::backfill] start entity_id_hash={} tree_id={} window_days={} cutoff_ms={}", redact(entity_id), tree.id, BACKFILL_WINDOW_DAYS, @@ -91,7 +91,7 @@ pub async fn backfill_topic_tree_at( if hits.is_empty() { log::debug!( - "[topic_tree::backfill] no entity-index hits for entity_id_hash={} — empty backfill", + "[tree_topic::backfill] no entity-index hits for entity_id_hash={} — empty backfill", redact(entity_id) ); return Ok(0); @@ -106,14 +106,14 @@ pub async fn backfill_topic_tree_at( let dropped = total_hits - hits.len(); if dropped > 0 { log::debug!( - "[topic_tree::backfill] dropped {dropped} hits older than {BACKFILL_WINDOW_DAYS}d \ + "[tree_topic::backfill] dropped {dropped} hits older than {BACKFILL_WINDOW_DAYS}d \ for entity_id_hash={}", redact(entity_id) ); } if hits.is_empty() { log::debug!( - "[topic_tree::backfill] all entity-index hits fell outside the {BACKFILL_WINDOW_DAYS}d \ + "[tree_topic::backfill] all entity-index hits fell outside the {BACKFILL_WINDOW_DAYS}d \ window for entity_id_hash={} — empty backfill", redact(entity_id) ); @@ -133,7 +133,7 @@ pub async fn backfill_topic_tree_at( // the point. if hit.node_kind != "leaf" { log::debug!( - "[topic_tree::backfill] skipping non-leaf hit node_id={} kind={}", + "[tree_topic::backfill] skipping non-leaf hit node_id={} kind={}", hit.node_id, hit.node_kind ); @@ -144,7 +144,7 @@ pub async fn backfill_topic_tree_at( Some(c) => c, None => { log::warn!( - "[topic_tree::backfill] missing chunk {} for entity_id_hash={} — skipping", + "[tree_topic::backfill] missing chunk {} for entity_id_hash={} — skipping", hit.node_id, redact(entity_id) ); @@ -179,7 +179,7 @@ pub async fn backfill_topic_tree_at( } log::info!( - "[topic_tree::backfill] done entity_id_hash={} tree_id={} appended={}", + "[tree_topic::backfill] done entity_id_hash={} tree_id={} appended={}", redact(entity_id), tree.id, appended @@ -194,10 +194,10 @@ mod tests { use crate::openhuman::memory::tree::score::extract::EntityKind; use crate::openhuman::memory::tree::score::resolver::CanonicalEntity; use crate::openhuman::memory::tree::score::store::index_entity; - use crate::openhuman::memory::tree::source_tree::store as src_store; - use crate::openhuman::memory::tree::source_tree::summariser::inert::InertSummariser; use crate::openhuman::memory::tree::store::upsert_chunks; - use crate::openhuman::memory::tree::topic_tree::registry::get_or_create_topic_tree; + use crate::openhuman::memory::tree::tree_source::store as src_store; + use crate::openhuman::memory::tree::tree_source::summariser::inert::InertSummariser; + use crate::openhuman::memory::tree::tree_topic::registry::get_or_create_topic_tree; use crate::openhuman::memory::tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; use chrono::{TimeZone, Utc}; use tempfile::TempDir; diff --git a/src/openhuman/memory/tree/topic_tree/curator.rs b/src/openhuman/memory/tree/tree_topic/curator.rs similarity index 92% rename from src/openhuman/memory/tree/topic_tree/curator.rs rename to src/openhuman/memory/tree/tree_topic/curator.rs index 85cba1845..255a1c400 100644 --- a/src/openhuman/memory/tree/topic_tree/curator.rs +++ b/src/openhuman/memory/tree/tree_topic/curator.rs @@ -19,16 +19,16 @@ use anyhow::Result; use chrono::Utc; use crate::openhuman::config::Config; -use crate::openhuman::memory::tree::source_tree::store as src_store; -use crate::openhuman::memory::tree::source_tree::summariser::Summariser; -use crate::openhuman::memory::tree::source_tree::types::{Tree, TreeKind}; -use crate::openhuman::memory::tree::topic_tree::backfill::backfill_topic_tree; -use crate::openhuman::memory::tree::topic_tree::hotness::hotness_at; -use crate::openhuman::memory::tree::topic_tree::registry::get_or_create_topic_tree; -use crate::openhuman::memory::tree::topic_tree::store::{ +use crate::openhuman::memory::tree::tree_source::store as src_store; +use crate::openhuman::memory::tree::tree_source::summariser::Summariser; +use crate::openhuman::memory::tree::tree_source::types::{Tree, TreeKind}; +use crate::openhuman::memory::tree::tree_topic::backfill::backfill_topic_tree; +use crate::openhuman::memory::tree::tree_topic::hotness::hotness_at; +use crate::openhuman::memory::tree::tree_topic::registry::get_or_create_topic_tree; +use crate::openhuman::memory::tree::tree_topic::store::{ distinct_sources_for, get_or_fresh, upsert, }; -use crate::openhuman::memory::tree::topic_tree::types::{ +use crate::openhuman::memory::tree::tree_topic::types::{ HotnessCounters, TOPIC_CREATION_THRESHOLD, TOPIC_RECHECK_EVERY, }; @@ -54,7 +54,7 @@ pub enum SpawnOutcome { /// fires, consider spawning a topic tree. /// /// `summariser` is used only when a spawn + backfill happens; passing an -/// [`InertSummariser`](crate::openhuman::memory::tree::source_tree::summariser::inert::InertSummariser) +/// [`InertSummariser`](crate::openhuman::memory::tree::tree_source::summariser::inert::InertSummariser) /// is fine for Phase 3c. pub async fn maybe_spawn_topic_tree( config: &Config, @@ -76,7 +76,7 @@ pub async fn maybe_spawn_topic_tree( if counters.ingests_since_check < TOPIC_RECHECK_EVERY { upsert(config, &counters)?; log::debug!( - "[topic_tree::curator] bumped counters entity={} mentions={} ingests_since_check={}", + "[tree_topic::curator] bumped counters entity={} mentions={} ingests_since_check={}", entity_id, counters.mention_count_30d, counters.ingests_since_check @@ -123,7 +123,7 @@ async fn run_full_recompute( let outcome = if h < TOPIC_CREATION_THRESHOLD { log::debug!( - "[topic_tree::curator] below threshold entity={} hotness={:.3} threshold={}", + "[tree_topic::curator] below threshold entity={} hotness={:.3} threshold={}", entity_id, h, TOPIC_CREATION_THRESHOLD @@ -131,7 +131,7 @@ async fn run_full_recompute( SpawnOutcome::BelowThreshold { hotness: h } } else if let Some(existing) = existing_topic_tree(config, entity_id)? { log::debug!( - "[topic_tree::curator] tree already exists entity={} tree_id={} hotness={:.3}", + "[tree_topic::curator] tree already exists entity={} tree_id={} hotness={:.3}", entity_id, existing.id, h @@ -143,7 +143,7 @@ async fn run_full_recompute( } else { // Crossed threshold for the first time — materialise. log::info!( - "[topic_tree::curator] spawning topic tree entity={} hotness={:.3}", + "[tree_topic::curator] spawning topic tree entity={} hotness={:.3}", entity_id, h ); @@ -171,9 +171,9 @@ mod tests { use crate::openhuman::memory::tree::score::extract::EntityKind; use crate::openhuman::memory::tree::score::resolver::CanonicalEntity; use crate::openhuman::memory::tree::score::store::index_entity; - use crate::openhuman::memory::tree::source_tree::summariser::inert::InertSummariser; use crate::openhuman::memory::tree::store::upsert_chunks; - use crate::openhuman::memory::tree::topic_tree::store::get; + use crate::openhuman::memory::tree::tree_source::summariser::inert::InertSummariser; + use crate::openhuman::memory::tree::tree_topic::store::get; use crate::openhuman::memory::tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; use chrono::{TimeZone, Utc}; use tempfile::TempDir; @@ -187,7 +187,7 @@ mod tests { fn seed_leaf_for_entity(cfg: &Config, entity_id: &str, source_tree: &str, seq: u32) { // Use a "now-anchored" timestamp so backfill's 30-day window - // (see topic_tree::backfill::BACKFILL_WINDOW_DAYS) always + // (see tree_topic::backfill::BACKFILL_WINDOW_DAYS) always // includes these seeded leaves. Spread by seq to keep ordering // deterministic. let ts_ms = Utc::now().timestamp_millis() - (seq as i64) * 1_000; diff --git a/src/openhuman/memory/tree/topic_tree/hotness.rs b/src/openhuman/memory/tree/tree_topic/hotness.rs similarity index 97% rename from src/openhuman/memory/tree/topic_tree/hotness.rs rename to src/openhuman/memory/tree/tree_topic/hotness.rs index e757dcccc..c10a7fbb2 100644 --- a/src/openhuman/memory/tree/topic_tree/hotness.rs +++ b/src/openhuman/memory/tree/tree_topic/hotness.rs @@ -23,7 +23,7 @@ use chrono::Utc; -use crate::openhuman::memory::tree::topic_tree::types::EntityIndexStats; +use crate::openhuman::memory::tree::tree_topic::types::EntityIndexStats; /// Pure hotness function — no I/O, no clocks unless the caller passes one. /// @@ -45,7 +45,7 @@ pub fn hotness_at(entity_id: &str, idx: &EntityIndexStats, now_ms: i64) -> f32 { let total = mention_weight + source_weight + recency_weight + centrality + query_weight; log::debug!( - "[topic_tree::hotness] id={} mentions={} sources={} recency={:.3} centrality={:.3} \ + "[tree_topic::hotness] id={} mentions={} sources={} recency={:.3} centrality={:.3} \ queries={} total={:.3}", entity_id, idx.mention_count_30d, @@ -110,7 +110,7 @@ mod tests { #[test] fn spike_of_mentions_pushes_over_creation_threshold() { - use crate::openhuman::memory::tree::topic_tree::types::TOPIC_CREATION_THRESHOLD; + use crate::openhuman::memory::tree::tree_topic::types::TOPIC_CREATION_THRESHOLD; let now_ms = 1_700_000_000_000; // 100 mentions across 5 sources, 3 recent query hits, seen today. let s = EntityIndexStats { diff --git a/src/openhuman/memory/tree/topic_tree/mod.rs b/src/openhuman/memory/tree/tree_topic/mod.rs similarity index 96% rename from src/openhuman/memory/tree/topic_tree/mod.rs rename to src/openhuman/memory/tree/tree_topic/mod.rs index 5f216005b..7e28a3c77 100644 --- a/src/openhuman/memory/tree/topic_tree/mod.rs +++ b/src/openhuman/memory/tree/tree_topic/mod.rs @@ -22,7 +22,7 @@ //! easy to unit-test. //! //! Tree mechanics (buffer, seal, cascade) are **not reimplemented** here -//! — `append_leaf` from [`super::source_tree::bucket_seal`] takes a +//! — `append_leaf` from [`super::tree_source::bucket_seal`] takes a //! `&Tree` so it works for any `TreeKind`. The Phase 3c code only adds //! the hotness layer and the per-entity fan-out. diff --git a/src/openhuman/memory/tree/topic_tree/registry.rs b/src/openhuman/memory/tree/tree_topic/registry.rs similarity index 88% rename from src/openhuman/memory/tree/topic_tree/registry.rs rename to src/openhuman/memory/tree/tree_topic/registry.rs index 9002b8cdd..21a59dc9f 100644 --- a/src/openhuman/memory/tree/topic_tree/registry.rs +++ b/src/openhuman/memory/tree/tree_topic/registry.rs @@ -12,8 +12,8 @@ use chrono::Utc; use uuid::Uuid; use crate::openhuman::config::Config; -use crate::openhuman::memory::tree::source_tree::store; -use crate::openhuman::memory::tree::source_tree::types::{Tree, TreeKind, TreeStatus}; +use crate::openhuman::memory::tree::tree_source::store; +use crate::openhuman::memory::tree::tree_source::types::{Tree, TreeKind, TreeStatus}; /// Look up the topic tree for `entity_id`, or create a new one. /// @@ -21,11 +21,15 @@ use crate::openhuman::memory::tree::source_tree::types::{Tree, TreeKind, TreeSta /// `"email:alice@example.com"` or `"hashtag:launch"`). Scope uses the /// canonical id verbatim so re-lookups are stable. pub fn get_or_create_topic_tree(config: &Config, entity_id: &str) -> Result { + let entity_kind = entity_id + .split_once(':') + .map(|(k, _)| k) + .unwrap_or("unknown"); if let Some(existing) = store::get_tree_by_scope(config, TreeKind::Topic, entity_id)? { log::debug!( - "[topic_tree::registry] found tree id={} entity={}", + "[tree_topic::registry] found tree id={} entity_kind={}", existing.id, - entity_id + entity_kind ); return Ok(existing); } @@ -79,10 +83,10 @@ pub fn archive_topic_tree(config: &Config, tree_id: &str) -> Result<()> { .with_context(|| format!("failed to archive topic tree {tree_id}"))?; if n == 0 { log::warn!( - "[topic_tree::registry] archive_topic_tree: no topic tree with id={tree_id}" + "[tree_topic::registry] archive_topic_tree: no topic tree with id={tree_id}" ); } else { - log::info!("[topic_tree::registry] archived topic tree id={tree_id}"); + log::info!("[tree_topic::registry] archived topic tree id={tree_id}"); } Ok(()) }) @@ -99,23 +103,30 @@ fn create_new(config: &Config, entity_id: &str) -> Result { created_at: Utc::now(), last_sealed_at: None, }; + let entity_kind = entity_id + .split_once(':') + .map(|(k, _)| k) + .unwrap_or("unknown"); match store::insert_tree(config, &tree) { Ok(()) => { log::info!( - "[topic_tree::registry] created topic tree id={} entity={}", + "[tree_topic::registry] created topic tree id={} entity_kind={}", tree.id, - entity_id + entity_kind ); Ok(tree) } Err(err) if is_unique_violation(&err) => { log::debug!( - "[topic_tree::registry] UNIQUE race for entity={} — re-querying", - entity_id + "[tree_topic::registry] UNIQUE race for entity_kind={} — re-querying", + entity_kind ); + // Re-query is keyed on the full entity_id; only the *log* line + // has been redacted. This still surfaces enough context to + // diagnose without leaking the recoverable id. store::get_tree_by_scope(config, TreeKind::Topic, entity_id)?.ok_or_else(|| { anyhow::anyhow!( - "UNIQUE violation on insert but no row found on re-query for entity {entity_id}" + "UNIQUE violation on insert but no row found on re-query (entity_kind={entity_kind})" ) }) } @@ -137,7 +148,7 @@ fn new_topic_tree_id() -> String { format!("{}:{}", TreeKind::Topic.as_str(), Uuid::new_v4()) } -/// Row mapper — duplicated from `source_tree::store::row_to_tree` because +/// Row mapper — duplicated from `tree_source::store::row_to_tree` because /// that one is private. Kept intentionally loose: topic-tree listing is /// not a hot path so the string parsing cost is immaterial. fn row_to_tree_loose(row: &rusqlite::Row<'_>) -> rusqlite::Result { @@ -230,7 +241,7 @@ mod tests { // the UNIQUE constraint is on (kind, scope), not scope alone. let (_tmp, cfg) = test_config(); let source = - crate::openhuman::memory::tree::source_tree::registry::get_or_create_source_tree( + crate::openhuman::memory::tree::tree_source::registry::get_or_create_source_tree( &cfg, "shared:slack:#eng", ) @@ -272,7 +283,7 @@ mod tests { fn list_topic_trees_returns_only_topics() { let (_tmp, cfg) = test_config(); // Mix of source + topic trees. - crate::openhuman::memory::tree::source_tree::registry::get_or_create_source_tree( + crate::openhuman::memory::tree::tree_source::registry::get_or_create_source_tree( &cfg, "slack:#eng", ) diff --git a/src/openhuman/memory/tree/topic_tree/routing.rs b/src/openhuman/memory/tree/tree_topic/routing.rs similarity index 90% rename from src/openhuman/memory/tree/topic_tree/routing.rs rename to src/openhuman/memory/tree/tree_topic/routing.rs index 1bd77092a..6f5e5b129 100644 --- a/src/openhuman/memory/tree/topic_tree/routing.rs +++ b/src/openhuman/memory/tree/tree_topic/routing.rs @@ -21,13 +21,13 @@ use anyhow::Result; use crate::openhuman::config::Config; -use crate::openhuman::memory::tree::source_tree::bucket_seal::{ +use crate::openhuman::memory::tree::tree_source::bucket_seal::{ append_leaf, LabelStrategy, LeafRef, }; -use crate::openhuman::memory::tree::source_tree::store as src_store; -use crate::openhuman::memory::tree::source_tree::summariser::Summariser; -use crate::openhuman::memory::tree::source_tree::types::{TreeKind, TreeStatus}; -use crate::openhuman::memory::tree::topic_tree::curator::maybe_spawn_topic_tree; +use crate::openhuman::memory::tree::tree_source::store as src_store; +use crate::openhuman::memory::tree::tree_source::summariser::Summariser; +use crate::openhuman::memory::tree::tree_source::types::{TreeKind, TreeStatus}; +use crate::openhuman::memory::tree::tree_topic::curator::maybe_spawn_topic_tree; /// Route `leaf` into every active topic tree matching one of /// `canonical_entities`. Also ticks the curator for each entity so the @@ -47,17 +47,21 @@ pub async fn route_leaf_to_topic_trees( } log::debug!( - "[topic_tree::routing] leaf={} entities={}", + "[tree_topic::routing] leaf={} entities={}", leaf.chunk_id, canonical_entities.len() ); for entity_id in canonical_entities { if let Err(e) = route_one_entity(config, leaf, entity_id, summariser).await { + let entity_kind = entity_id + .split_once(':') + .map(|(k, _)| k) + .unwrap_or("unknown"); log::warn!( - "[topic_tree::routing] failed routing leaf={} entity={} err={:#}", + "[tree_topic::routing] failed routing leaf={} entity_kind={} err={:#}", leaf.chunk_id, - entity_id, + entity_kind, e ); } @@ -80,7 +84,7 @@ async fn route_one_entity( if let Some(tree) = src_store::get_tree_by_scope(config, TreeKind::Topic, entity_id)? { if tree.status == TreeStatus::Active { log::debug!( - "[topic_tree::routing] appending leaf={} → topic_tree={}", + "[tree_topic::routing] appending leaf={} → topic_tree={}", leaf.chunk_id, tree.id ); @@ -103,10 +107,14 @@ async fn route_one_entity( ) .await?; } else { + let entity_kind = entity_id + .split_once(':') + .map(|(k, _)| k) + .unwrap_or("unknown"); log::debug!( - "[topic_tree::routing] skip archived topic tree id={} entity={}", + "[tree_topic::routing] skip archived topic tree id={} entity_kind={}", tree.id, - entity_id + entity_kind ); } } @@ -123,12 +131,12 @@ mod tests { use crate::openhuman::memory::tree::score::extract::EntityKind; use crate::openhuman::memory::tree::score::resolver::CanonicalEntity; use crate::openhuman::memory::tree::score::store::index_entity; - use crate::openhuman::memory::tree::source_tree::summariser::inert::InertSummariser; use crate::openhuman::memory::tree::store::upsert_chunks; - use crate::openhuman::memory::tree::topic_tree::registry::{ + use crate::openhuman::memory::tree::tree_source::summariser::inert::InertSummariser; + use crate::openhuman::memory::tree::tree_topic::registry::{ archive_topic_tree, get_or_create_topic_tree, }; - use crate::openhuman::memory::tree::topic_tree::store::get as get_hotness; + use crate::openhuman::memory::tree::tree_topic::store::get as get_hotness; use crate::openhuman::memory::tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; use chrono::{TimeZone, Utc}; use tempfile::TempDir; @@ -190,7 +198,7 @@ mod tests { .unwrap(); // No hotness rows were created. assert_eq!( - crate::openhuman::memory::tree::topic_tree::store::count(&cfg).unwrap(), + crate::openhuman::memory::tree::tree_topic::store::count(&cfg).unwrap(), 0 ); } @@ -298,18 +306,18 @@ mod tests { // to keep hotness above `TOPIC_CREATION_THRESHOLD` once the index // is queried (two indexed sources below → distinct_sources → 2). let mut counters = - crate::openhuman::memory::tree::topic_tree::types::HotnessCounters::fresh(entity_id, 0); + crate::openhuman::memory::tree::tree_topic::types::HotnessCounters::fresh(entity_id, 0); counters.mention_count_30d = 1_000; counters.distinct_sources = 2; counters.last_seen_ms = Some(Utc::now().timestamp_millis()); counters.query_hits_30d = 5; counters.ingests_since_check = - crate::openhuman::memory::tree::topic_tree::types::TOPIC_RECHECK_EVERY - 1; - crate::openhuman::memory::tree::topic_tree::store::upsert(&cfg, &counters).unwrap(); + crate::openhuman::memory::tree::tree_topic::types::TOPIC_RECHECK_EVERY - 1; + crate::openhuman::memory::tree::tree_topic::store::upsert(&cfg, &counters).unwrap(); // Seed leaves in slack and gmail referencing Alice. Anchor the // timestamps to "now" so the 30-day backfill window - // (topic_tree::backfill::BACKFILL_WINDOW_DAYS) covers them. + // (tree_topic::backfill::BACKFILL_WINDOW_DAYS) covers them. let now_ms = Utc::now().timestamp_millis(); let ts_c1 = now_ms - 20_000; let ts_c2 = now_ms - 10_000; diff --git a/src/openhuman/memory/tree/topic_tree/store.rs b/src/openhuman/memory/tree/tree_topic/store.rs similarity index 98% rename from src/openhuman/memory/tree/topic_tree/store.rs rename to src/openhuman/memory/tree/tree_topic/store.rs index e7c54a7f0..64d518af3 100644 --- a/src/openhuman/memory/tree/topic_tree/store.rs +++ b/src/openhuman/memory/tree/tree_topic/store.rs @@ -2,7 +2,7 @@ //! //! The only new table owned here is `mem_tree_entity_hotness` — the //! per-entity counter block driving lazy materialisation. Tree rows and -//! summary nodes are reused from [`super::super::source_tree::store`] via +//! summary nodes are reused from [`super::super::tree_source::store`] via //! the shared `mem_tree_trees` / `mem_tree_summaries` / `mem_tree_buffers` //! tables, which already carry a `kind` column that discriminates //! `source` from `topic`. No schema additions for those tables in Phase @@ -18,7 +18,7 @@ use rusqlite::{params, OptionalExtension}; use crate::openhuman::config::Config; use crate::openhuman::memory::tree::store::with_connection; -use crate::openhuman::memory::tree::topic_tree::types::HotnessCounters; +use crate::openhuman::memory::tree::tree_topic::types::HotnessCounters; /// Fetch the hotness row for `entity_id`, or `None` if the entity has /// never been seen. Callers usually want [`get_or_fresh`] instead. diff --git a/src/openhuman/memory/tree/topic_tree/types.rs b/src/openhuman/memory/tree/tree_topic/types.rs similarity index 100% rename from src/openhuman/memory/tree/topic_tree/types.rs rename to src/openhuman/memory/tree/tree_topic/types.rs diff --git a/src/openhuman/memory/tree/types.rs b/src/openhuman/memory/tree/types.rs index a7e05a045..7300778c3 100644 --- a/src/openhuman/memory/tree/types.rs +++ b/src/openhuman/memory/tree/types.rs @@ -151,6 +151,7 @@ pub struct SourceRef { } impl SourceRef { + /// Wrap an opaque provider-specific identifier as a [`SourceRef`]. pub fn new(value: impl Into) -> Self { Self { value: value.into(), diff --git a/src/openhuman/memory/tree/util/README.md b/src/openhuman/memory/tree/util/README.md new file mode 100644 index 000000000..eac048c14 --- /dev/null +++ b/src/openhuman/memory/tree/util/README.md @@ -0,0 +1,12 @@ +# util/ + +Shared utility helpers used across the memory-tree subsystem. Kept pure-function and dependency-light so any module in `tree/` can pull them in without cycle risk. + +## Files + +- [`mod.rs`](mod.rs) — module banner; re-exports `redact`. +- [`redact.rs`](redact.rs) — log-time PII redaction. `redact(s)` hashes a string to 8 stable hex chars (safe to grep when the raw value is available externally). `redact_endpoint(url)` strips scheme, path, query, fragment, and credentials, keeping only `host[:port]`. + +## When to use + +Per CLAUDE.md: never log secrets or full PII. After the participant-bucketing change, source_ids and content_paths can embed full email addresses, so any log line that prints them must redact first. diff --git a/src/openhuman/screen_intelligence/tests.rs b/src/openhuman/screen_intelligence/tests.rs index fc95c8bab..6687b8ccb 100644 --- a/src/openhuman/screen_intelligence/tests.rs +++ b/src/openhuman/screen_intelligence/tests.rs @@ -15,7 +15,7 @@ use super::state::{AccessibilityEngine, EngineState}; use super::types::{CaptureFrame, InputActionParams, StartSessionParams}; use crate::openhuman::accessibility::{parse_foreground_output, AppContext}; use crate::openhuman::config::{Config, ScreenIntelligenceConfig}; -use crate::openhuman::memory::embeddings::NoopEmbedding; +use crate::openhuman::embeddings::NoopEmbedding; use crate::openhuman::memory::store::UnifiedMemory; struct EnvVarGuard { diff --git a/src/openhuman/tools/impl/memory/forget.rs b/src/openhuman/tools/impl/memory/forget.rs index 6aabcc717..91c425a14 100644 --- a/src/openhuman/tools/impl/memory/forget.rs +++ b/src/openhuman/tools/impl/memory/forget.rs @@ -91,7 +91,8 @@ impl Tool for MemoryForgetTool { #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory::{embeddings::NoopEmbedding, MemoryCategory, UnifiedMemory}; + use crate::openhuman::embeddings::NoopEmbedding; + use crate::openhuman::memory::{MemoryCategory, UnifiedMemory}; use crate::openhuman::security::{AutonomyLevel, SecurityPolicy}; use tempfile::TempDir; diff --git a/src/openhuman/tools/impl/memory/recall.rs b/src/openhuman/tools/impl/memory/recall.rs index 2fa038d9a..c826e05dc 100644 --- a/src/openhuman/tools/impl/memory/recall.rs +++ b/src/openhuman/tools/impl/memory/recall.rs @@ -105,7 +105,8 @@ impl Tool for MemoryRecallTool { #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory::{embeddings::NoopEmbedding, MemoryCategory, UnifiedMemory}; + use crate::openhuman::embeddings::NoopEmbedding; + use crate::openhuman::memory::{MemoryCategory, UnifiedMemory}; use tempfile::TempDir; fn seeded_mem() -> (TempDir, Arc) { diff --git a/src/openhuman/tools/impl/memory/store.rs b/src/openhuman/tools/impl/memory/store.rs index b70341ed9..a83028e44 100644 --- a/src/openhuman/tools/impl/memory/store.rs +++ b/src/openhuman/tools/impl/memory/store.rs @@ -105,7 +105,8 @@ impl Tool for MemoryStoreTool { #[cfg(test)] mod tests { use super::*; - use crate::openhuman::memory::{embeddings::NoopEmbedding, UnifiedMemory}; + use crate::openhuman::embeddings::NoopEmbedding; + use crate::openhuman::memory::UnifiedMemory; use crate::openhuman::security::{AutonomyLevel, SecurityPolicy}; use tempfile::TempDir; diff --git a/tests/json_rpc_e2e.rs b/tests/json_rpc_e2e.rs index c02063263..d8e5d476e 100644 --- a/tests/json_rpc_e2e.rs +++ b/tests/json_rpc_e2e.rs @@ -981,6 +981,10 @@ async fn json_rpc_memory_sync_and_learn() { "missing channel_id must return an error, got: {sync_bad}" ); + // ── memory.init: explicit one-shot bootstrap (no auto-init fallback) ──── + let init_resp = post_json_rpc(&rpc_base, 7003, "openhuman.memory_init", json!({})).await; + assert_no_jsonrpc_error(&init_resp, "memory_init"); + // ── memory_learn_all: no namespaces → zero processed (empty store) ────── let learn_all = post_json_rpc(&rpc_base, 7004, "openhuman.memory_learn_all", json!({})).await; let learn_result = assert_no_jsonrpc_error(&learn_all, "memory_learn_all"); diff --git a/tests/memory_graph_sync_e2e.rs b/tests/memory_graph_sync_e2e.rs index f572a82b5..e76fc6dd3 100644 --- a/tests/memory_graph_sync_e2e.rs +++ b/tests/memory_graph_sync_e2e.rs @@ -14,9 +14,10 @@ use std::time::Duration; use serde_json::json; use tempfile::tempdir; +use openhuman_core::openhuman::embeddings::NoopEmbedding; use openhuman_core::openhuman::memory::{ - embeddings::NoopEmbedding, MemoryClient, MemoryIngestionConfig, MemoryIngestionRequest, - NamespaceDocumentInput, UnifiedMemory, + MemoryClient, MemoryIngestionConfig, MemoryIngestionRequest, NamespaceDocumentInput, + UnifiedMemory, }; /// Test config for the heuristic-only pipeline. diff --git a/tests/screen_intelligence_vision_e2e.rs b/tests/screen_intelligence_vision_e2e.rs index 6a5d05efc..bd4b6d3d7 100644 --- a/tests/screen_intelligence_vision_e2e.rs +++ b/tests/screen_intelligence_vision_e2e.rs @@ -35,7 +35,7 @@ use image::imageops::FilterType; use image::{ImageBuffer, Rgb, RgbImage}; use tempfile::tempdir; -use openhuman_core::openhuman::memory::embeddings::NoopEmbedding; +use openhuman_core::openhuman::embeddings::NoopEmbedding; use openhuman_core::openhuman::memory::store::types::NamespaceDocumentInput; use openhuman_core::openhuman::memory::store::UnifiedMemory; use openhuman_core::openhuman::screen_intelligence::CaptureFrame; @@ -118,7 +118,7 @@ fn make_capture_frame(image_ref: Option) -> CaptureFrame { /// Open a UnifiedMemory backed by NoopEmbedding in a temp dir. fn open_test_memory(dir: &Path) -> UnifiedMemory { - let embedder: Arc = + let embedder: Arc = Arc::new(NoopEmbedding); UnifiedMemory::new(dir, embedder, Some(5)).expect("UnifiedMemory::new") } diff --git a/tests/subconscious_e2e.rs b/tests/subconscious_e2e.rs index 5d68f2a7e..5a24c5258 100644 --- a/tests/subconscious_e2e.rs +++ b/tests/subconscious_e2e.rs @@ -59,7 +59,7 @@ async fn ingest_doc( #[tokio::test] #[ignore] // requires running Ollama async fn two_tick_e2e_with_real_ollama() { - use openhuman_core::openhuman::memory::embeddings::NoopEmbedding; + use openhuman_core::openhuman::embeddings::NoopEmbedding; use openhuman_core::openhuman::memory::{MemoryClient, UnifiedMemory}; use openhuman_core::openhuman::subconscious::store;