feat(memory): W1 — TinyCortex adapter seam (facade, config + embedding adapters) (#4526)

Cargo.lock

@@ -4393,6 +4393,7 @@ dependencies = [
  "tempfile",
  "thiserror 2.0.18",
  "tinyagents",
+ "tinycortex",
  "tinyflows",
  "tinyplace",
  "tokio",
@@ -4400,7 +4401,7 @@ dependencies = [
  "tokio-stream",
  "tokio-tungstenite 0.24.0",
  "tokio-util",
- "toml",
+ "toml 1.1.2+spec-1.1.0",
  "tower",
  "tracing",
  "tracing-appender",
@@ -5015,7 +5016,7 @@ version = "3.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f"
 dependencies = [
- "toml_edit",
+ "toml_edit 0.25.11+spec-1.1.0",
 ]
 
 [[package]]
@@ -6126,6 +6127,15 @@ dependencies = [
  "syn 2.0.117",
 ]
 
+[[package]]
+name = "serde_spanned"
+version = "0.6.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "serde_spanned"
 version = "1.1.1"
@@ -6712,6 +6722,27 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tinycortex"
+version = "0.1.1"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "chrono",
+ "git2",
+ "parking_lot",
+ "rand 0.8.6",
+ "regex",
+ "rusqlite",
+ "serde",
+ "serde_json",
+ "sha2 0.10.9",
+ "thiserror 2.0.18",
+ "toml 0.8.23",
+ "uuid 1.23.1",
+ "walkdir",
+]
+
 [[package]]
 name = "tinyflows"
 version = "0.3.0"
@@ -6942,6 +6973,18 @@ dependencies = [
  "tokio-util",
 ]
 
+[[package]]
+name = "toml"
+version = "0.8.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"
+dependencies = [
+ "serde",
+ "serde_spanned 0.6.9",
+ "toml_datetime 0.6.11",
+ "toml_edit 0.22.27",
+]
+
 [[package]]
 name = "toml"
 version = "1.1.2+spec-1.1.0"
@@ -6950,11 +6993,20 @@ checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee"
 dependencies = [
  "indexmap",
  "serde_core",
- "serde_spanned",
- "toml_datetime",
+ "serde_spanned 1.1.1",
+ "toml_datetime 1.1.1+spec-1.1.0",
  "toml_parser",
  "toml_writer",
- "winnow",
+ "winnow 1.0.2",
+]
+
+[[package]]
+name = "toml_datetime"
+version = "0.6.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
+dependencies = [
+ "serde",
 ]
 
 [[package]]
@@ -6966,6 +7018,20 @@ dependencies = [
  "serde_core",
 ]
 
+[[package]]
+name = "toml_edit"
+version = "0.22.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
+dependencies = [
+ "indexmap",
+ "serde",
+ "serde_spanned 0.6.9",
+ "toml_datetime 0.6.11",
+ "toml_write",
+ "winnow 0.7.15",
+]
+
 [[package]]
 name = "toml_edit"
 version = "0.25.11+spec-1.1.0"
@@ -6973,9 +7039,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b"
 dependencies = [
  "indexmap",
- "toml_datetime",
+ "toml_datetime 1.1.1+spec-1.1.0",
  "toml_parser",
- "winnow",
+ "winnow 1.0.2",
 ]
 
 [[package]]
@@ -6984,9 +7050,15 @@ version = "1.1.2+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"
 dependencies = [
- "winnow",
+ "winnow 1.0.2",
 ]
 
+[[package]]
+name = "toml_write"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
+
 [[package]]
 name = "toml_writer"
 version = "1.1.1+spec-1.1.0"
@@ -8579,6 +8651,15 @@ version = "0.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
 
+[[package]]
+name = "winnow"
+version = "0.7.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "winnow"
 version = "1.0.2"
@@ -9074,7 +9155,3 @@ checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296"
 dependencies = [
  "zune-core",
 ]
-
-[[patch.unused]]
-name = "tinycortex"
-version = "0.1.1"

Cargo.toml

@@ -58,6 +58,15 @@ tinyflows = "0.3"
 # Durable graph checkpoints still use `SqlRunLedgerCheckpointer` until the
 # migration re-points those rows to the crate checkpointer.
 tinyagents = { version = "1.5.0", features = ["sqlite"] }
+# TinyCortex — Rust core for the memory engine (store/chunks/tree/retrieval/
+# queue/ingest/score + long tail), vendored as a git submodule and patched
+# below to `vendor/tinycortex`. OpenHuman's memory subsystem migrates onto this
+# crate through the adapter seam in `src/openhuman/tinycortex/` (mirroring the
+# tinyagents seam): engine logic in the crate; RPC, agent tools, live sync,
+# security gating, and the global singleton stay host-side. rusqlite/git2 are
+# aligned to the host pins (=0.40 / 0.21) so one bundled SQLite + one libgit2
+# link. Keep the version pin in lockstep with the submodule tag.
+tinycortex = "0.1"
 # TokenJuice code compressor — AST-aware signature extraction. Optional (C build)
 # behind the default `tokenjuice-treesitter` feature; disabling it falls back to
 # the language-agnostic brace-depth heuristic. See src/openhuman/tokenjuice/compressors/code.rs.

src/openhuman/mod.rs

@@ -124,6 +124,7 @@ pub mod text_input;
 pub mod thread_goals;
 pub mod threads;
 pub mod tinyagents;
+pub mod tinycortex;
 pub mod tinyflows;
 pub mod tinyplace;
 pub mod tls;

src/openhuman/tinycortex/chat.rs

@@ -0,0 +1,114 @@
+//! LLM chat seam — bridge OpenHuman's memory chat runtime onto the crate's
+//! [`ChatProvider`] (W1).
+//!
+//! TinyCortex extracts entities/topics and summarises via an injected
+//! `ChatProvider` (it never makes a network call). OpenHuman already owns a
+//! memory LLM surface — `memory::chat::{ChatProvider, ChatPrompt}` with
+//! `build_chat_provider(&Config)` routing through `openhuman::inference`
+//! (provider selection, credit metering, usage accounting). This adapter wraps
+//! that host provider and re-exposes it as the crate's `ChatProvider`, so the
+//! engine's LLM entity extractor / summariser drive OpenHuman inference without
+//! duplicating any routing.
+//!
+//! The two contracts are near-identical (`name` + async `chat_for_json`). The
+//! only conversion is the prompt: the host `ChatPrompt.temperature` is `f64`,
+//! the crate's is `f32`; every other field maps 1:1.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use tinycortex::memory::score::extract::{
+    ChatPrompt as CortexChatPrompt, ChatProvider as CortexChatProvider,
+};
+
+use crate::openhuman::config::Config;
+use crate::openhuman::memory::chat::{
+    build_chat_provider as build_host_chat_provider, ChatPrompt as HostChatPrompt,
+    ChatProvider as HostChatProvider,
+};
+
+/// Wraps an OpenHuman [`HostChatProvider`] as the crate's [`CortexChatProvider`].
+pub struct SeamChatProvider {
+    inner: Arc<dyn HostChatProvider>,
+}
+
+impl SeamChatProvider {
+    /// Build the adapter over a host chat provider (already routed through
+    /// `openhuman::inference`).
+    pub fn new(inner: Arc<dyn HostChatProvider>) -> Self {
+        tracing::debug!(
+            provider = inner.name(),
+            "[memory] constructing tinycortex chat seam over memory::chat::ChatProvider"
+        );
+        Self { inner }
+    }
+}
+
+#[async_trait]
+impl CortexChatProvider for SeamChatProvider {
+    fn name(&self) -> &str {
+        self.inner.name()
+    }
+
+    async fn chat_for_json(&self, prompt: &CortexChatPrompt) -> anyhow::Result<String> {
+        let host = HostChatPrompt {
+            system: prompt.system.clone(),
+            user: prompt.user.clone(),
+            // Crate temperature is f32; host takes f64.
+            temperature: f64::from(prompt.temperature),
+            kind: prompt.kind,
+            max_tokens: prompt.max_tokens,
+        };
+        self.inner.chat_for_json(&host).await
+    }
+}
+
+/// Build a crate [`CortexChatProvider`] from the host [`Config`], routed through
+/// `openhuman::inference` — the entry point the seam's LLM entity extractor and
+/// summariser construct from.
+pub fn build_chat_provider(config: &Config) -> anyhow::Result<Arc<dyn CortexChatProvider>> {
+    let host = build_host_chat_provider(config)?;
+    Ok(Arc::new(SeamChatProvider::new(host)))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Echoes the fields it received back as a JSON body so the test can assert
+    /// the crate->host prompt conversion (incl. the f32->f64 temperature).
+    struct EchoHostProvider;
+
+    #[async_trait]
+    impl HostChatProvider for EchoHostProvider {
+        fn name(&self) -> &str {
+            "echo"
+        }
+        async fn chat_for_json(&self, prompt: &HostChatPrompt) -> anyhow::Result<String> {
+            Ok(format!(
+                "system={};user={};temp={};kind={};max={:?}",
+                prompt.system, prompt.user, prompt.temperature, prompt.kind, prompt.max_tokens
+            ))
+        }
+    }
+
+    #[tokio::test]
+    async fn converts_prompt_and_delegates_to_host_provider() {
+        let seam = SeamChatProvider::new(Arc::new(EchoHostProvider));
+        assert_eq!(CortexChatProvider::name(&seam), "echo");
+
+        let prompt = CortexChatPrompt {
+            system: "sys".to_string(),
+            user: "usr".to_string(),
+            temperature: 0.5,
+            kind: "extract",
+            max_tokens: Some(64),
+        };
+        let out = seam.chat_for_json(&prompt).await.unwrap();
+        // Every field maps 1:1; temperature widens f32 0.5 -> f64 0.5.
+        assert_eq!(
+            out,
+            "system=sys;user=usr;temp=0.5;kind=extract;max=Some(64)"
+        );
+    }
+}

src/openhuman/tinycortex/config.rs

@@ -0,0 +1,78 @@
+//! `Config` → [`tinycortex::memory::MemoryConfig`] mapping (W1).
+//!
+//! The crate's [`MemoryConfig`] is the single input every engine primitive
+//! takes (`workspace`, embedding dims/model/strict, tree budgets, retrieval
+//! weight profile, sync budget). This adapter derives it from OpenHuman's
+//! host [`Config`] plus the resolved memory workspace root, so the rest of the
+//! seam constructs engine calls from real product configuration.
+//!
+//! Field provenance:
+//! - `workspace` ← the memory workspace root (same root `MemoryClient` opens).
+//! - `embedding.dim` ← `config.memory.embedding_dimensions`.
+//! - `embedding.model` ← `config.memory.embedding_model`.
+//! - `embedding.strict` ← `config.memory_tree.embedding_strict` (when false the
+//!   engine tolerates an inert embedder and falls back to scope+recency rerank).
+//! - `tree` / `retrieval` / `sync_budget` ← crate defaults, which already match
+//!   the host engine's constants (`INPUT_TOKEN_BUDGET = 50_000`,
+//!   `OUTPUT_TOKEN_BUDGET = 5_000`, `SUMMARY_FANOUT = 10`,
+//!   `DEFAULT_FLUSH_AGE_SECS = 604_800`). The `tree_policy.rs` flavour overlays
+//!   and per-source `WeightProfile` selection are layered on at call sites in
+//!   later workstreams; this base mapping is the W1 foundation.
+
+use std::path::PathBuf;
+
+use tinycortex::memory::config::EmbeddingConfig;
+use tinycortex::memory::MemoryConfig;
+
+use crate::openhuman::config::Config;
+
+/// Build a [`MemoryConfig`] from the host [`Config`] and the resolved memory
+/// workspace root.
+///
+/// `workspace` is the directory under which the engine stores `chunks.db`, the
+/// content vault, and the tree DBs — it must be the same root the host
+/// `MemoryClient` opens so an existing user workspace is read in place (parity
+/// is gated by the W3 golden-workspace harness).
+pub fn memory_config_from(config: &Config, workspace: PathBuf) -> MemoryConfig {
+    let mut mc = MemoryConfig::new(workspace);
+    mc.embedding = EmbeddingConfig {
+        dim: config.memory.embedding_dimensions,
+        model: config.memory.embedding_model.clone(),
+        strict: config.memory_tree.embedding_strict,
+    };
+    mc
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn maps_workspace_and_embedding_from_host_config() {
+        let mut config = Config::default();
+        config.memory.embedding_dimensions = 1024;
+        config.memory.embedding_model = "embedding-v1".to_string();
+        config.memory_tree.embedding_strict = true;
+
+        let workspace = PathBuf::from("/tmp/openhuman/ws");
+        let mc = memory_config_from(&config, workspace.clone());
+
+        assert_eq!(mc.workspace, workspace);
+        assert_eq!(mc.embedding.dim, 1024);
+        assert_eq!(mc.embedding.model, "embedding-v1");
+        assert!(mc.embedding.strict);
+    }
+
+    #[test]
+    fn tree_defaults_match_engine_constants() {
+        // The base mapping leaves tree budgets at the crate defaults, which are
+        // the host engine's own constants — asserted here so a crate-side change
+        // to those defaults surfaces as a failing parity test rather than a
+        // silent behaviour drift.
+        let mc = memory_config_from(&Config::default(), PathBuf::from("/tmp/ws"));
+        assert_eq!(mc.tree.input_token_budget, 50_000);
+        assert_eq!(mc.tree.output_token_budget, 5_000);
+        assert_eq!(mc.tree.summary_fanout, 10);
+        assert_eq!(mc.tree.flush_age_secs, 604_800);
+    }
+}

src/openhuman/tinycortex/embeddings.rs

@@ -0,0 +1,139 @@
+//! Embedding seam — bridge OpenHuman's [`EmbeddingProvider`] onto the crate's
+//! two embedding traits (W1).
+//!
+//! OpenHuman owns the concrete providers (voyage / openai / cohere / ollama /
+//! cloud / noop) and the `embeddings/factory.rs` construction policy (rate-limit
+//! + retry). TinyCortex "never makes a network call" — it takes compute through
+//! [`EmbeddingBackend`] (the vector store) and [`Embedder`] (retrieval / seal
+//! scoring). This adapter wraps one `Arc<dyn EmbeddingProvider>` and re-exposes
+//! it as both, so the engine drives OpenHuman embeddings without cloning any
+//! provider logic.
+//!
+//! The two host and crate contracts are shape-identical (`name` / `model_id` /
+//! `dimensions` / `signature` / async `embed`), and both use `anyhow::Result`,
+//! so this is a near-pure pass-through. Critically, `signature()` delegates to
+//! the provider so the persisted embedding-space signature
+//! (`provider=…;model=…;dims=…`) stays byte-identical whether the store keys off
+//! the crate backend or the raw provider (#1574 fidelity).
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use tinycortex::memory::score::embed::Embedder;
+use tinycortex::memory::store::vectors::EmbeddingBackend;
+
+use crate::openhuman::embeddings::EmbeddingProvider;
+
+/// Wraps an OpenHuman [`EmbeddingProvider`] as the crate's [`EmbeddingBackend`]
+/// (vector store) and [`Embedder`] (retrieval / seal scoring).
+pub struct SeamEmbedder {
+    provider: Arc<dyn EmbeddingProvider>,
+}
+
+impl SeamEmbedder {
+    /// Build the adapter over an OpenHuman embedding provider, preserving its
+    /// factory-configured rate-limit + retry policy.
+    pub fn new(provider: Arc<dyn EmbeddingProvider>) -> Self {
+        tracing::debug!(
+            provider = provider.name(),
+            model_id = provider.model_id(),
+            dimensions = provider.dimensions(),
+            signature = %provider.signature(),
+            "[memory] constructing tinycortex embedding seam over EmbeddingProvider"
+        );
+        Self { provider }
+    }
+}
+
+#[async_trait]
+impl EmbeddingBackend for SeamEmbedder {
+    fn name(&self) -> &str {
+        self.provider.name()
+    }
+
+    fn model_id(&self) -> &str {
+        self.provider.model_id()
+    }
+
+    fn dimensions(&self) -> usize {
+        self.provider.dimensions()
+    }
+
+    /// Delegate to the provider so the persisted signature is byte-identical to
+    /// the config-derived `active_embedding_signature` — a mismatch would split
+    /// one embedding space into two (#1574).
+    fn signature(&self) -> String {
+        self.provider.signature()
+    }
+
+    async fn embed(&self, texts: &[&str]) -> anyhow::Result<Vec<Vec<f32>>> {
+        self.provider.embed(texts).await
+    }
+}
+
+#[async_trait]
+impl Embedder for SeamEmbedder {
+    fn name(&self) -> &'static str {
+        // The crate's `Embedder` requires a `'static` name (debug/diagnostics
+        // only); the provider's own `name()` is borrowed, so report a stable
+        // seam label rather than leaking a lifetime.
+        "openhuman-seam"
+    }
+
+    async fn embed(&self, text: &str) -> anyhow::Result<Vec<f32>> {
+        self.provider.embed_one(text).await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    struct FakeProvider;
+
+    #[async_trait]
+    impl EmbeddingProvider for FakeProvider {
+        fn name(&self) -> &str {
+            "fake"
+        }
+        fn model_id(&self) -> &str {
+            "fake-model"
+        }
+        fn dimensions(&self) -> usize {
+            3
+        }
+        async fn embed(&self, texts: &[&str]) -> anyhow::Result<Vec<Vec<f32>>> {
+            Ok(texts
+                .iter()
+                .map(|t| vec![t.len() as f32, 0.0, 0.0])
+                .collect())
+        }
+    }
+
+    #[tokio::test]
+    async fn backend_passes_through_metadata_and_signature() {
+        let seam = SeamEmbedder::new(Arc::new(FakeProvider));
+        assert_eq!(EmbeddingBackend::name(&seam), "fake");
+        assert_eq!(seam.model_id(), "fake-model");
+        assert_eq!(seam.dimensions(), 3);
+        // Byte-identical to format_embedding_signature(name, model_id, dims).
+        assert_eq!(
+            EmbeddingBackend::signature(&seam),
+            "provider=fake;model=fake-model;dims=3"
+        );
+    }
+
+    #[tokio::test]
+    async fn backend_and_embedder_both_delegate_to_provider() {
+        let seam = SeamEmbedder::new(Arc::new(FakeProvider));
+
+        let batch = EmbeddingBackend::embed(&seam, &["ab", "cde"])
+            .await
+            .unwrap();
+        assert_eq!(batch, vec![vec![2.0, 0.0, 0.0], vec![3.0, 0.0, 0.0]]);
+
+        let one = Embedder::embed(&seam, "abcd").await.unwrap();
+        assert_eq!(one, vec![4.0, 0.0, 0.0]);
+        assert_eq!(Embedder::name(&seam), "openhuman-seam");
+    }
+}

src/openhuman/tinycortex/mod.rs

@@ -0,0 +1,52 @@
+//! `tinycortex` integration — run OpenHuman's memory engine on the published
+//! [`tinycortex`](https://crates.io/crates/tinycortex) crate.
+//!
+//! OpenHuman's memory subsystem migrates onto the `tinycortex` crate (store /
+//! chunks / tree / retrieval / queue / ingest / score + the long tail). This
+//! module is the **adapter seam**, mirroring `src/openhuman/tinyagents/`: it
+//! implements the crate's engine traits over OpenHuman services and derives the
+//! engine's [`MemoryConfig`] from the host [`Config`]. Nothing here contains
+//! engine logic — that lives in the crate.
+//!
+//! ## Ownership boundary (the seam contract)
+//!
+//! **Engine (crate):** content store + YAML vault, SQLite vectors/kv/entity
+//! index, chunk lifecycle, summary trees, hybrid retrieval, scoring, the async
+//! job model, ingest canonicalize/extract, and the diff/entities/graph/goals/
+//! archivist/tool-memory/conversations long tail.
+//!
+//! **Product (host, stays in OpenHuman):** JSON-RPC schemas/ops/read_rpc, agent
+//! tools + `SecurityPolicy` gating, live sync (`memory_sync`), the event bus,
+//! preferences, `source_scope` per-turn allowlist, redaction, the global
+//! singleton + background queue worker, embedding/LLM **compute**, and the
+//! host-retained `UnifiedMemory` namespace-document tier (episodic/event/
+//! segment/doc/graph/profile tables) plus the `wiki_git`/`obsidian` content
+//! surfaces the crate deliberately excludes.
+//!
+//! The crate never makes a network call or owns a worker pool: LLM/embedding
+//! compute is injected through `EmbeddingBackend`, `ChatProvider`, `Summariser`,
+//! and `EntityExtractor`; the job queue is driven by the host worker loop via
+//! `queue::run_once` / `drain_until_idle`. Those adapters live beside this file
+//! (`embeddings.rs`, `chat.rs`, `queue_driver.rs`, `sinks.rs`, `bus.rs`) and are
+//! added workstream by workstream.
+//!
+//! See `docs/tinycortex-migration-spec.md` for the full ownership split,
+//! drift/gap/parity ledgers, and the workstream order.
+
+mod chat;
+mod config;
+mod embeddings;
+
+pub use chat::{build_chat_provider, SeamChatProvider};
+pub use config::memory_config_from;
+pub use embeddings::SeamEmbedder;
+
+// Facade re-exports — the rest of the host imports memory-engine types through
+// this one seam so consumer import paths stay stable as the internals flip to
+// the crate (the type-unification decision, spec §0.5). `MemoryTaint` is the
+// security-critical provenance type; it is proven byte-identical to the host's
+// (fail-closed to `ExternalSync`) before re-exporting.
+pub use tinycortex::memory::{
+    MemoryCategory, MemoryConfig, MemoryEntry, MemoryTaint, NamespaceSummary, RecallOpts,
+    WeightProfile,
+};