refactor(memory): finish TinyCortex consolidation (#5140)

This commit is contained in:
Steven Enamakel
2026-07-23 05:42:14 +03:00
committed by GitHub
parent c480e6d817
commit 2bcd34496c
105 changed files with 875 additions and 2711 deletions
+1 -1
View File
@@ -856,7 +856,7 @@ jobs:
key: tinycortex
- name: Run TinyCortex engine and Composio sync tests
run: bash scripts/ci-cancel-aware.sh cargo test --manifest-path vendor/tinycortex/Cargo.toml --features git-diff,sync
run: bash scripts/ci-cancel-aware.sh cargo test --manifest-path vendor/tinycortex/Cargo.toml --features git-diff,sync,persona
pr-ci-gate:
name: PR CI Gate
Generated
+5 -22
View File
@@ -3079,7 +3079,7 @@ dependencies = [
"js-sys",
"log",
"wasm-bindgen",
"windows-core 0.62.2",
"windows-core 0.57.0",
]
[[package]]
@@ -4746,7 +4746,7 @@ dependencies = [
"tar",
"tempfile",
"thiserror 2.0.18",
"tinyagents 2.1.0",
"tinyagents",
"tinychannels",
"tinycortex",
"tinyflows",
@@ -7332,23 +7332,6 @@ dependencies = [
"crunchy",
]
[[package]]
name = "tinyagents"
version = "2.0.0"
dependencies = [
"async-trait",
"bytes",
"chrono",
"futures",
"reqwest 0.12.28",
"serde",
"serde_json",
"sha2 0.11.0",
"thiserror 2.0.18",
"tokio",
"tracing",
]
[[package]]
name = "tinyagents"
version = "2.1.0"
@@ -7433,7 +7416,7 @@ dependencies = [
"serde_json",
"sha2 0.10.9",
"thiserror 2.0.18",
"tinyagents 2.0.0",
"tinyagents",
"tokio",
"toml 0.8.23",
"tracing",
@@ -7453,7 +7436,7 @@ dependencies = [
"serde",
"serde_json",
"thiserror 2.0.18",
"tinyagents 2.1.0",
"tinyagents",
"tracing",
]
@@ -8864,7 +8847,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.48.0",
]
[[package]]
+5 -9
View File
@@ -109,10 +109,12 @@ tinyagents = { version = "2.1", features = ["sqlite"] }
# queue/ingest/score + long tail), vendored as a git submodule and patched
# below to `vendor/tinycortex`. OpenHuman's memory subsystem migrates onto this
# crate through the adapter seam in `src/openhuman/tinycortex/` (mirroring the
# tinyagents seam): engine logic in the crate; RPC, agent tools, live sync,
# security gating, and the global singleton stay host-side. rusqlite/git2 are
# tinyagents seam): engine logic (including provider sync pipelines) in the
# crate; RPC, agent tools, sync scheduling/credentials/events, security gating,
# and the global singleton stay host-side. rusqlite/git2 are
# aligned to the host pins (=0.40 / 0.21) so one bundled SQLite + one libgit2
# link. Keep the version pin in lockstep with the submodule tag.
# link. The submodule intentionally tracks reviewed upstream main commits;
# keep this semver requirement compatible with the vendored crate version.
tinycortex = { version = "0.1", features = ["git-diff", "persona", "sync"] }
tinychannels = { version = "0.1", features = ["relay-websocket"] }
serde = { version = "1", features = ["derive"] }
@@ -704,12 +706,6 @@ tinyjuice = { path = "vendor/tinyjuice" }
tinychannels = { path = "vendor/tinychannels" }
tinyplace = { path = "vendor/tinyplace/sdk/rust" }
# TinyCortex temporarily pins the embedding API port while tinyagents #58 is
# pending. Resolve that git source to the same vendored crate so host adapters
# and TinyCortex share one `EmbeddingModel` trait identity.
[patch."https://github.com/senamakel/tinyagents"]
tinyagents = { path = "vendor/tinyagents" }
# Emit just enough DWARF in release builds for Sentry to symbolicate Rust
# panics + render surrounding source lines. `line-tables-only` keeps the
# binary small (only file+line tables, no full type info) while still
+16 -33
View File
@@ -205,7 +205,7 @@ version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -216,7 +216,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
dependencies = [
"anstyle",
"once_cell_polyfill",
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -2061,7 +2061,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users 0.5.2",
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -2479,7 +2479,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -3706,7 +3706,7 @@ dependencies = [
"js-sys",
"log",
"wasm-bindgen",
"windows-core 0.62.2",
"windows-core 0.57.0",
]
[[package]]
@@ -4107,7 +4107,7 @@ dependencies = [
"portable-atomic",
"portable-atomic-util",
"serde_core",
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -4991,7 +4991,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -5603,7 +5603,7 @@ dependencies = [
"tar",
"tempfile",
"thiserror 2.0.18",
"tinyagents 2.1.0",
"tinyagents",
"tinychannels",
"tinycortex",
"tinyflows",
@@ -7133,7 +7133,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -7192,7 +7192,7 @@ dependencies = [
"security-framework 3.7.0",
"security-framework-sys",
"webpki-root-certs",
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -7949,7 +7949,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
dependencies = [
"libc",
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -8809,7 +8809,7 @@ dependencies = [
"getrandom 0.4.2",
"once_cell",
"rustix",
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -8968,23 +8968,6 @@ dependencies = [
"strict-num",
]
[[package]]
name = "tinyagents"
version = "2.0.0"
dependencies = [
"async-trait",
"bytes",
"chrono",
"futures",
"reqwest 0.12.28",
"serde",
"serde_json",
"sha2 0.11.0",
"thiserror 2.0.18",
"tokio",
"tracing",
]
[[package]]
name = "tinyagents"
version = "2.1.0"
@@ -9064,7 +9047,7 @@ dependencies = [
"serde_json",
"sha2 0.10.9",
"thiserror 2.0.18",
"tinyagents 2.0.0",
"tinyagents",
"tokio",
"toml 0.8.2",
"tracing",
@@ -9084,7 +9067,7 @@ dependencies = [
"serde",
"serde_json",
"thiserror 2.0.18",
"tinyagents 2.1.0",
"tinyagents",
"tracing",
]
@@ -9670,7 +9653,7 @@ checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e"
dependencies = [
"memoffset",
"tempfile",
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -10466,7 +10449,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.48.0",
]
[[package]]
+2 -2
View File
@@ -366,10 +366,10 @@ End-to-end coverage of the agent harness via the web-chat RPC surface against an
| ----- | ------------------------------------------ | ----- | -------------------------------------------------------------------------------------------------------------- | ------ | ----------------------------------------------------------------------- |
| 8.4.1 | Save Preference (general / situational) | RU | `src/openhuman/agent/tools/save_preference_tests.rs` | ✅ | `save_preference` tool → `user_pref_{general,situational}`, topic-keyed |
| 8.4.2 | Lane A — Standing Prefs in System Prompt | RU | `src/openhuman/learning/prompt_sections.rs`, `src/openhuman/agent/harness/session/turn_tests.rs` | ✅ | General prefs rendered into the system prompt at thread start |
| 8.4.3 | Lane B — Situational Recall (vector-gated) | RU | `src/openhuman/memory/store/unified/query_tests.rs::recall_relevant_by_vector_gates_on_similarity` | ✅ | Per-turn; relevant query injects, unrelated suppresses |
| 8.4.3 | Lane B — Situational Recall (vector-gated) | RU | `src/openhuman/memory_store/namespace_store/query_tests.rs::recall_relevant_by_vector_gates_on_similarity` | ✅ | Per-turn; relevant query injects, unrelated suppresses |
| 8.4.4 | Same-Topic Contradiction (replace) | RU | `src/openhuman/agent/tools/save_preference_tests.rs::recategorising_moves_pref_between_namespaces` | ✅ | `ON CONFLICT REPLACE`; a topic lives in exactly one scope |
| 8.4.5 | Cross-Topic Contradiction Surfacing | RU | `src/openhuman/agent/tools/save_preference_tests.rs::save_surfaces_related_preference_for_contradiction_check` | ✅ | Related prefs surfaced in the tool result for the chat agent to resolve |
| 8.4.6 | vector_chunks Model-Signature Recall Guard | RU | `src/openhuman/memory/store/unified/query_tests.rs::vector_recall_excludes_other_model_signature` | ✅ | Excludes cross-model vectors; dim-guards legacy rows |
| 8.4.6 | vector_chunks Model-Signature Recall Guard | RU | `src/openhuman/memory_store/namespace_store/query_tests.rs::vector_recall_excludes_other_model_signature` | ✅ | Excludes cross-model vectors; dim-guards legacy rows |
### 8.5 Long-term Goals
+11 -6
View File
@@ -1,5 +1,10 @@
# TinyCortex API Gap Audit (Phase 0.2)
**Status (2026-07-22):** Historical gap inventory for the completed engine
cutover. Counts and source anchors below describe the audit base; unresolved
host consolidation work, including G1 re-homing and sync dedupe, is tracked in
`tinycortex-migration-plan-2026-07-22.md`.
**Purpose.** Map every host call site into the engine-mapping modules
(`memory_store`, `memory_tree`, `memory_queue`, and the long tail) onto a TinyCortex
public API. Where no crate API covers a host need, record a **gap** — each gap becomes a
@@ -48,16 +53,16 @@ The crate exposes every seam the plan expects, plus more. Host implements these:
| `Summariser` (time-tree, string) | `tree/runtime/engine.rs:29` | **divergent — see G6** |
| `EntityExtractor` (regex + LLM) | `score/extract/composite.rs:16` | `tinycortex/chat.rs` (LLM variant) |
| `QueueDelegates` | `queue/handlers.rs:96` | `tinycortex/queue_driver.rs` |
| `TreeJobSink` | `ingest/types.rs:15` | `tinycortex/sinks.rs` |
| `TreeLeafSink` | `archivist/sink.rs:44` | `tinycortex/sinks.rs` |
| `SnapshotItemSource` | `diff/source.rs:28` | `tinycortex/sinks.rs` |
| `EntityOccurrenceIndex` | `graph/types.rs:44` | `tinycortex/sinks.rs` |
| `TreeJobSink` | `ingest/types.rs:15` | `tinycortex/ingest.rs` |
| `TreeLeafSink` | `archivist/sink.rs:44` | `tinycortex/ingest.rs` |
| `SnapshotItemSource` | `diff/source.rs:28` | host diff adapter |
| `EntityOccurrenceIndex` | `graph/types.rs:44` | host graph adapter |
| `SelfIdentity` | `store/entity_index/store.rs:25` | host identity registry |
| `GoalsGenerator` | `goals/reflect.rs:57` | `tinycortex/chat.rs` |
| `SourceReader` | `sources/readers/mod.rs:1732` | local readers only *(amended: live-sync readers join the crate in W-SYNC, plan §8)* |
| `SyncEventSink` *(new, W-SYNC)* | `sync/traits.rs` (planned) | `tinycortex/sync_sink.rs``MemorySyncStage` bus events |
| `SkillDocSink` *(new, W-SYNC)* | `sync/traits.rs` (planned) | forwards to `MemoryClient::store_skill_sync` (host-retained unified tier) |
| `ConversationEventBus` / `ChannelEventHandler` | `conversations/bus.rs:106/95` | `tinycortex/bus.rs` (translate to `DomainEvent`) |
| `ConversationEventBus` / `ChannelEventHandler` | `conversations/bus.rs:106/95` | host `memory_conversations/bus.rs` (translate to `DomainEvent`) |
Config seam: `MemoryConfig{workspace, embedding: EmbeddingConfig{dim,model,strict},
tree: TreeConfig{input_token_budget,output_token_budget,summary_fanout,flush_age_secs},
@@ -86,7 +91,7 @@ retrieval: RetrievalConfig{default_profile}, sync_budget: SyncBudgetConfig}` +
### G2 — Graph relation-edge persistence (E2GraphRAG accumulation) — **HARD**
- **Host:** `memory_store/unified/graph.rs`, `unified/query.rs` persist and query co-occurrence
- **Host:** `memory_store/namespace_store/{graph,query}.rs` persist and query co-occurrence
/ LLM-triple relations; `NamespaceMemoryHit.supporting_relations: Vec<GraphRelationRecord>`
is populated at retrieval.
- **Crate:** persists the **entity occurrence index** at persist time
+21 -17
View File
@@ -25,7 +25,9 @@ touched an engine-mapping memory module after the port line, and classifies each
| Host audit SHA | `7850cf363559bcbb7ba688cbc4fccdb6bd9ce754` (`main`, 2026-07-04) |
| TinyCortex submodule | `vendor/tinycortex``tinyhumansai/tinycortex` |
| TinyCortex audit SHA | `d1a8c7be2babc8fff7a72ed93861f459f3d6fa58` |
| **TinyCortex pinned SHA (current)** | `a8e10f7dd8ebdb9b0905e1380fefcc6bf5a65207` — audit SHA **+ #59** (native-dep alignment, §0.4) **+ #63/#64** (D2/D1 drift ports) merged; this is the live gitlink |
| TinyCortex historical cutover SHA | `a8e10f7dd8ebdb9b0905e1380fefcc6bf5a65207` — audit SHA **+ #59** (native-dep alignment, §0.4) **+ #63/#64** (D2/D1 drift ports) merged |
| **TinyCortex reviewed gitlink (2026-07-22)** | `daaaf6ba5f02635c08deae2b2b2ed7fcc8c06b6a` — includes OpenHuman #4794/#4820/#4863-era crate work; upstream currently has no tags |
| **TinyCortex migration branch (current)** | `7b4b115` — TinyAgents 2.1 dependency alignment, standalone patch configuration, and corrected sync ownership docs |
| TinyCortex crate version | `0.1.1` |
| **Port line (derived)** | **after 2026-06-25, before 2026-06-28** (see below) |
@@ -38,6 +40,11 @@ touched an engine-mapping memory module after the port line, and classifies each
> (conversations) are now unblocked per the gate rule. D4 (memory_sync corpus,
> W-SYNC) remains its own separate track.
> **Consolidation status (2026-07-22): D1D4 CLOSED.** D1D3 remain closed.
> D4 was re-audited against the crate sync implementation and the live host
> dispatch path. The host schedulers, credentials, RPC, source-scope/redaction
> policy, product task normalizers, and event-bus adapters remain host-owned.
### How the port line was located
The port commits in `vendor/tinycortex` are all dated **2026-06-29**, but that is the date the
@@ -132,26 +139,23 @@ its `tools/` stay host (agent tools), its `vector`/`scoring` are engine (W5) —
## D4 — memory_sync corpus (2026-07-09 reclassification, W-SYNC)
The plan's §8 amendment moves the sync engine into the crate, so the blanket **HOST-OWNED
"live sync"** classification above is superseded for engine-mapping sync code. Unlike D1D3 there
is **no existing crate port to diff against** — the entire `memory_sync/` engine (plus the
`memory_sources` dispatcher/reconcile parts) is port scope. The port baseline for W-SYNC is taken
**fresh from host `main` at W-SYNC.1 branch time**, so ordinary drift cannot accumulate; this entry
exists to (a) record the reclassification and (b) pin the post-port-line commits that already
touched the corpus, so the W-SYNC.1 port provably includes them:
The plan's §8 amendment moved the generic sync engine into the crate, so the
blanket **HOST-OWNED "live sync"** classification above is superseded for
engine-mapping sync code. The crate port landed in `0333d10` and is the path
called by `memory_sync::composio::run_connection_sync` and the default
`ComposioProvider::sync` implementation.
| # | Host commit | Files in corpus | Note |
| --- | --- | --- | --- |
| D4.1 | `c43f79641` (07-03) | `composio/providers/{sync_state,traits}.rs` | TinyAgents-cutover import churn |
| D4.2 | `27b00b539` (07-05) | `sources/rebuild.rs` | test-parity cleanup (1 line) |
| D4.3 | `653e6e143` (07-06) | `memory_sources/sync.rs` (+312) | self-heal drifted content-sha tokens + prune vanished folder items |
| D4.4 | `e456b7799` (07-07) | `memory_sources/{rpc,sync}.rs`, `canonicalize/email.rs`, `composio/providers/{gmail/post_process,notion/source,orchestrator}.rs`, `sources/github.rs` | orchestration-fixes wave |
| D4.1 | `c43f79641` (07-03) | `composio/providers/{sync_state,traits}.rs` | ✅ **CLOSED.** Import churn only; persistence and provider dispatch are implemented by the `SyncStateStore` seam and crate dispatcher. |
| D4.2 | `27b00b539` (07-05) | `sources/rebuild.rs` | ✅ **CLOSED.** Test-only parity cleanup; no engine semantic delta. |
| D4.3 | `653e6e143` (07-06) | `memory_sources/sync.rs` (+312) | ✅ **CLOSED.** Crate `sync/workspace.rs` updates changed items and prunes vanished items through `LocalDocumentSink::delete`; covered by workspace sync tests. |
| D4.4 | `e456b7799` (07-07) | `memory_sources/{rpc,sync}.rs`, `canonicalize/email.rs`, `composio/providers/{gmail/post_process,notion/source,orchestrator}.rs`, `sources/github.rs` | ✅ **CLOSED.** Generic provider fetch/pagination/canonical memory records are in the crate port; orchestration, product task normalization, action-result presentation, and RPC fixes are host policy by design. |
**Gate:** W-SYNC.1 (crate scaffolding PR) requires this enumeration current as of its branch point
(re-run the scan: `git log --since=2026-06-25 --oneline -- src/openhuman/memory_sync
src/openhuman/memory_sources`); the W-SYNC.3 host flip requires D4 **CLOSED** — every listed commit
(and any accrued since) verifiably contained in the crate port or explicitly waived. Host-retained
parts (schedulers, bus subscribers, RPC wrappers, keychain/OAuth) remain HOST-OWNED as before.
**Gate result:** D4 is **CLOSED**. Gmail's Composio 413 mitigation is additionally
present in crate commit `ba9e12e` (25-message fetch pages). Host-retained parts
(schedulers, bus subscribers, RPC wrappers, keychain/OAuth, action tools and
product task/profile projections) remain HOST-OWNED.
## Closing the ledger (procedure)
+8 -5
View File
@@ -1,6 +1,9 @@
# TinyCortex Memory Migration — Plan & Audit
**Status:** W1W2 landed; W3 partial (chunks sub-store flipped). Amended 2026-07-09 (see §8) to a **sync-inclusive** scope.
**Status:** Historical execution plan. W1W8 and the engine test port are landed
(OpenHuman #4794/#4820); persona/coding-session ingest followed in #4863. The
remaining consolidation work is tracked in
[`tinycortex-migration-plan-2026-07-22.md`](tinycortex-migration-plan-2026-07-22.md).
**Anchor precedent:** the TinyAgents harness migration (#4249 / #4399 / #4473 and follow-ups).
**Target:** migrate large portions of the OpenHuman memory subsystem onto the `tinycortex` crate, vendored as a git submodule at **`vendor/tinycortex`** (`https://github.com/tinyhumansai/tinycortex`).
@@ -83,8 +86,8 @@ New sibling module mirroring `src/openhuman/tinyagents/`, holding every impl of
- `chat.rs``impl ChatProvider` + `impl Summariser` over `memory::chat::build_chat_provider` / `inference::provider`.
- `queue_driver.rs` — tokio worker loop calling `queue::run_once`/`drain_until_idle`, owned by `memory/global.rs`, with event-bus progress emission and Sentry hooks (host-side, since TinyCortex dropped its scheduler).
- `config.rs``Config``MemoryConfig` (workspace roots, `EmbeddingConfig`, `TreeConfig`, `WeightProfile`, `SyncBudgetConfig`) with `tree_policy.rs` flavour overlays.
- `sinks.rs``TreeJobSink`, `SnapshotItemSource` impls bridging to host state.
- `bus.rs`translate engine outcomes into `core::event_bus` `DomainEvent`s (host-side only; the crate stays bus-free).
- `ingest.rs` / `seal.rs``TreeJobSink`, seal observer, and host state bridges.
- `sync.rs`sync persistence/policy plus translation into host `DomainEvent`s (the crate stays bus-free).
- `mod.rs` — facade re-exports (`pub use tinycortex::memory::{…}`) so the rest of the host imports through one seam, plus module-doc explaining the boundary (the tinyagents seam's `mod.rs` header is the template).
---
@@ -103,7 +106,7 @@ This phase produces documents and upstream issues only; it is the gate for every
**0.5 Type-unification decision.** Host `memory::traits` types and crate types are wire-compatible twins. Decide: host re-exports crate types (preferred — one source of truth, 30+ consumer sites unchanged via `pub use`), vs. keeping host types + `From` conversions (fallback if serde/API divergence is found in 0.3). Special care: `MemoryTaint` is **security-critical provenance** (fails closed to `ExternalSync`, drives external-effect-tool gating) — its semantics, serde representation, and fail-closed defaults must be proven identical before re-exporting.
**0.6 Spec doc.** Write `docs/tinycortex-migration-spec.md` version-anchored to exact reviewed SHAs (host + tinycortex), with the ownership lists above, the drift/gap/parity ledgers, and a **deletion ledger** skeleton (every legacy file, with preconditions for deletion) — directly modeled on `docs/tinyagents-migration-spec.md` + `99-deletion-ledger.md`.
**0.6 Spec doc.** Write `docs/tinycortex-migration-spec.md` version-anchored to exact reviewed SHAs (host + tinycortex), with the ownership lists above, the drift/gap/parity ledgers, and a **deletion ledger** skeleton (every legacy file, with preconditions for deletion) — coordinated with `docs/tinyagents-migration-plan-2026-07-22.md`; the TinyCortex deletion ledger lives in §2 of the spec.
---
@@ -146,7 +149,7 @@ W3W5 are the risky core (user data on disk). W1W2 can land quickly; W6W
Ordering rule for this migration: **within each workstream, the implementation (adapter + cutover + deletion) lands first; the test work follows as the second slice.** Tests are not skipped — they are sequenced after the implementation is proven to compile and pass the *existing* suites. Concretely:
1. **Slice A (impl):** adapters + cutover + legacy deletion. Gate: `cargo check` both worlds, existing crate-level integration tests (`tests/memory_roundtrip_e2e.rs`, `memory_tree_sync_deep_raw_coverage_e2e.rs`, etc.) still green — these exercise the public `openhuman::memory::` surface and act as the built-in parity harness for every flip.
1. **Slice A (impl):** adapters + cutover + legacy deletion. Gate: `cargo check` both worlds, existing crate-level integration tests (`tests/memory_roundtrip_e2e.rs`, `tests/raw_coverage/memory_tree_sync_deep_raw_coverage_e2e.rs`, etc.) still green — these exercise the public `openhuman::memory::` surface and act as the built-in parity harness for every flip.
2. **Slice B (tests):** port and extend the test estate for the new boundary:
- **Engine-internal tests move upstream.** The host's `#[path]`-included sibling tests (`ops_tests.rs` pattern) that poke engine internals migrate into `vendor/tinycortex` as crate tests (via tinycortex PRs), following the crate's own `*_tests.rs` sibling convention.
- **Host keeps boundary tests:** RPC schema/handler tests, tool gating tests, seam adapter tests (config mapping, embedding signature, taint fail-closed, source-scope enforcement), and the E2E files under `tests/`.
@@ -0,0 +1,417 @@
# TinyCortex Migration — Current-State Audit & Consolidated Plan (2026-07-22)
**Status:** **DONE** on `feat/tinycortex-migration-2026-07-22`.
[CI Full run 29925645209](https://github.com/senamakel/openhuman/actions/runs/29925645209)
is green, including the final gate and every Linux, macOS, and Windows desktop
shard
(companion to `tinyagents-migration-plan-2026-07-22.md`;
supersedes the *status/phasing* in `tinycortex-memory-migration-plan.md`,
`tinycortex-migration-spec.md`, `tinycortex-api-gap-audit.md`, and
`tinycortex-parity-checklist.md` — those docs' ownership splits, gap lists, and
parity dimensions remain the reference detail. `tinycortex-drift-ledger.md`
stays the row-level ledger but its anchors are stale — see §2).
**Scope:** finish moving the remaining generic memory-engine code from
`src/openhuman/` into the vendored `tinycortex` crate (`vendor/tinycortex`),
delete the in-tree duplicates and staging code, migrate/retire the affected
tests, and clean up every dangling doc/code reference.
**Method:** fresh four-way audit — (1) the crate surface, (2) the core memory
domains, (3) the periphery + the `src/openhuman/tinycortex/` seam, (4) the
docs/tests/git history — against the working tree at `main` (`5b8a9f269`,
2026-07-22).
---
## 1. Executive summary
**The engine migration is essentially done and the docs don't know it.** All
five `docs/tinycortex-*` docs self-describe a "W1W2 landed; W3 partial" state,
but git shows the full W-track landed: W1 seam (#4526), W2 type re-export
(#4529), the W3 chunk-store cutover chain (#4532#4559), W4 queue flip
(#4781), W5 vector+scoring (#4790), the W7 long-tail shims (#4785#4789),
**engine-migration completion (#4794)**, **engine test port into the crate
(#4820)**, and post-completion feature work (persona/coding-session ingest
#4863) on top. The drift ledger's D1D3 rows are all CLOSED.
What remains is not "migrate the engine" but five bounded packages (§5):
1. **Housekeeping** — the pin/tag story is broken (crate has *no git tags*
despite the Cargo.toml comment demanding tag-lockstep), the host CI vendor
suite omits the `persona` feature the host enables, and there are ~8
broken/stale references including a seam module doc naming files that were
never created.
2. **Delete `memory_store/unified/`** — the explicitly "staging for removal"
tier (~5k LOC + ~3.7k test LOC), gated by the G1 `sqlite_conn()` escape
hatch.
3. **W-SYNC completion**`memory_sync/` (17.7k LOC, the largest remaining
host mass) still runs the host provider pipelines while the crate now ships
a full `memory::sync` Composio engine; the D4 ledger rows gate the flip.
4. **Embedding-provider dedupe** — host `embeddings/` adapters and
`memory_tree/score/embed/` providers now duplicate what
`tinyagents::harness::embeddings` ships since tinyagents #58 (this package
lands in **tinyagents**, not tinycortex — the crates split the job).
5. **Shim retirement + funnel decision** — the compatibility shims left behind
by W3W7, and a policy call on the fact that domains import
`tinycortex::memory::*` directly while the seam's re-export facade has
**zero** consumers.
---
## 2. Ground truth: pins, versions, and discrepancies (fix first)
| Fact | Value | Where |
| --- | --- | --- |
| Host requirement | `tinycortex = { version = "0.1", features = ["git-diff", "persona", "sync"] }` | root `Cargo.toml:116` |
| Path override | `[patch.crates-io] tinycortex = { path = "vendor/tinycortex" }` | `Cargo.toml:682` |
| Vendored crate | `0.1.1`, edition 2021, **MIT** (not GPL — unlike tinyagents), repo `tinyhumansai/tinycortex` | `vendor/tinycortex/Cargo.toml` |
| Submodule HEAD | `daaaf6ba` (`heads/main`, dependabot-era, ~50+ commits past every SHA the docs cite) | `git submodule status` |
| Tags | **none exist**`git describe --tags` fails; yet host `Cargo.toml:115` says "Keep the version pin in lockstep with the submodule tag" | submodule |
| Docs' claimed pin | gitlink `a8e10f7` (exists in history, long since passed) | drift ledger / spec anchors |
| Nested dep | tinycortex requires `tinyagents = { version = "2", path = "vendor/tinyagents" }`, **its own nested submodule pinned at v2.0.0** | `vendor/tinycortex/Cargo.toml:69` |
| Host tinyagents req | `2.1` | root `Cargo.toml:107` |
| Fork patch | `[patch."https://github.com/senamakel/tinyagents"]` exists *for tinycortex's sake* ("temporarily pins the embedding API port while tinyagents #58 is pending" — #58 is now merged at the tinyagents submodule HEAD) | `Cargo.toml:685-691` |
| Host CI vendor suite | `cargo test --manifest-path vendor/tinycortex/Cargo.toml --features git-diff,sync`**omits `persona`**, which the host enables | `.github/workflows/ci-lite.yml:859` |
Version skew summary: three different tinyagents expectations (host `2.1`,
tinycortex `2`, nested submodule `2.0.0`) resolve only via path patches. The
tinyagents plan's WP-0 (bump `vendor/tinyagents` to v2.1.0) should be executed
together with bumping tinycortex's *nested* `vendor/tinyagents` so both crates
share one trait identity at one version, after which the `senamakel` patch
block and its "temporarily pins" comment can likely be retired.
---
## 3. What the crate already ships
`tinycortex 0.1.1` exposes a single `memory` module — the engine "ported from
OpenHuman" — with features `tokio`, `git-diff` (git2-backed `memory::diff`),
`providers-http` (reqwest embedding/LLM providers, host does **not** enable),
`sync` (live Composio + workspace-scan engine), `persona` (doc-06 persona
distillation). Modules: `store` (content/vectors/kv/entity_index/safety+PII),
`chunks`, `sources` (+readers), `score` (embed/extract/signals), `tree`
(+runtime, bucket/document seal, flavoured trees), `queue` (SQLite job queue +
tokio runtime), `retrieval` (hybrid search, MMR, rerank), `entities`, `graph`,
`goals`, `tool_memory`, `conversations`, `archivist`, `ingest`
(canonicalize→chunk→score→tree), `diff`, `sync` (composio
clickup/github/linear/notion/slack/gmail), `providers`, `persona`
(readers for claude_code/codex/git history), `fsutil`.
Host-facing extension points (~20 public traits) with the seam's current
implementations in parentheses: `Memory`, `MemoryStore`, `EmbeddingBackend` +
`Embedder` (`SeamEmbedder`), `ChatProvider` (`SeamChatProvider`), `Summariser`
(`HostSummariser`), `SealObserver` + seal-time `Embedder` bridge (`seal.rs`),
`TreeJobSink` (`HostTreeJobSink`), `QueueDelegates` (`HostQueueDelegates`),
`SourceReader`, `SelfIdentity`, `EntityOccurrenceIndex`, `SnapshotItemSource`,
`GoalsGenerator`, `ConversationEventBus`/`ChannelEventHandler`,
`PersonaStateStore`, and the sync seams `SyncStateStore` / `SyncEventSink` /
`SkillDocSink` / `LocalDocumentSink` / `ExternalSourceReader` / `SyncPipeline`
/ `ActionExecutor` / `IncrementalSource` (`sync.rs`, `HostSyncAdapter`).
The crate consumes `tinyagents::harness::embeddings::{EmbeddingModel,
format_embedding_signature}` in `store/vectors/embedding.rs` — that is the
whole tinyagents dependency (the W-EMB decision made real by tinyagents #58).
Crate CI: fmt, clippy `-D warnings`, build+test `--all-features`, doc build
with warnings denied, plus a per-feature matrix (`core`, `tokio`, `git-diff`,
`sync`, all). Offline by convention; Composio/live tests are wiremock-mocked or
env-key-gated.
---
## 4. Audit: what remains host-side
### 4.1 Per-domain state (crate-backed file counts via `use tinycortex::`)
| Domain | Files / LOC | Crate-backed | State |
| --- | --- | --- | --- |
| `memory/` | 69 / 18.0k | 3 | Orchestration + policy + RPC layer (by design "no SQLite here"). Predominantly host: `read_rpc/`, `schema*/`, `rpc_models`, agent tools, sync orchestration, tree *policy*. Stays, minus shim cleanup. |
| `memory_store/` | 54 / 17.7k | 15 | Sub-stores flipped to crate (chunks, content, vectors, kv, entities, trees, safety/PII per W3/W7 markers). **`unified/` (~5k + ~3.7k tests) is README-marked "Staging for removal"** — documents/query/segments/events/profile await per-kind flips. `factories.rs` (config/provider selection) stays host. |
| `memory_tree/` | 47 / 10.3k | 16 | Largest crate-backed surface; engine parts delegate. Host keeps RPC (`tree/rpc.rs` 1,387, `retrieval/rpc.rs`), CLI, bus subscriber, `health/` doctor (G5, host-retained by decision), and `score/embed/` **providers** (~1.9k — see WP-3). |
| `memory_queue/` | 7 / 1.4k | 2 | W4 flipped: `worker.rs` delegates claim→dispatch→settle to `tinycortex::memory::queue::run_once` via `HostQueueDelegates`; per-kind handlers deleted. Host keeps worker pool, scheduler tick, `scheduler_gate` throttle, SQLite glue. Done, minus the `memory::jobs` re-export shim. |
| `memory_search/` | 8 / 0.8k | 2 | W5 flipped: `scoring.rs`/`vector/mmr.rs` are near-empty shims over crate `WeightProfile`/`retrieval::mmr`. Host keeps the three agent tools. Done. |
| `memory_tools/` | 9 / 1.4k | 3 | W7 flipped: types/store/prompt are shims over `tool_memory::{types,store,render}`. Host keeps `capture.rs` (`PostTurnHook`) + put/list tools. Done. |
| `memory_archivist/` | 1 / 44 | 1 | Pure W7 shim over `memory::archivist`. Done. |
| `memory_conversations/` | 2 / 0.9k | 2 | W7 shim over `memory::conversations`; host keeps `bus.rs` event-bus glue (+ a `tinychannels` legacy-message conversion). Done. |
| `memory_diff/` | 7 / 2.0k | 11 uses | W7 flipped: engine crate-owned; host keeps RPC/tools + `DomainEvent::MemoryDiff*` publishing. Done. |
| `memory_goals/` | 6 / 0.9k | 4 uses | W7 flipped: types/store crate-owned (byte-identical `MEMORY_GOALS.md`); host keeps agent wiring. Done. |
| `memory_sources/` | 16 / 5.0k | 12 uses | Product registry (config-persisted connectors) with crate readers underneath. Host by design. |
| `memory_sync/` | **73 / 17.7k** | 5 uses | **The largest remaining mass.** Host Composio provider pipelines (gmail/slack/github/notion/linear/clickup) still run host-side while the crate's `memory::sync` ships the same provider engine. W-SYNC seam (`sync.rs`, 679) is landed; the host flip (W-SYNC.3) has not happened. See WP-2. |
| `embeddings/` | 13 / 3.1k | **0** | Bridges **tinyagents** (9 files), not tinycortex. Post-#58, the host per-provider adapters duplicate crate `harness::embeddings` providers. See WP-4. |
| `agent_memory/`, `subconscious/` (memory parts), `learning/` (memory parts) | — | 0 | Host product consumers of the memory stack; no crate duplication. Stay. |
Consumer counts (host memory tiers are deeply embedded and *stay* host, so
these are context, not deletion blockers): `memory::` 166 files,
`memory_store::` 132, `memory_tree::` 67, `memory_queue::` 18,
`memory_tools::` 11, `memory_search::` 3.
### 4.2 The seam — `src/openhuman/tinycortex/` (11 files, ~3.2k LOC)
`config.rs`, `embeddings.rs`, `chat.rs`, `summariser.rs`, `ingest.rs`,
`seal.rs`, `sync.rs` (679), `queue_driver.rs` (1,007), `persona.rs` (446),
`parity.rs` (`#[cfg(test)]` format pins), `mod.rs`. Healthy and intentionally
host-owned. Two findings:
- **The seam is not the funnel.** `mod.rs` re-exports crate types
(`MemoryEntry`, `MemoryTaint`, `RecallOpts`, …) as the intended
type-unification funnel, but **zero** files import `crate::…::tinycortex::`
re-exports — every memory domain imports `tinycortex::memory::*` directly.
Either convention works; pick one and record it (WP-5). Direct crate imports
are simpler and match how `tinyagents` is consumed; if so, shrink the facade
instead of promoting it.
- **Stale internal doc:** `mod.rs:28-30` names sibling adapters `sinks.rs` and
`bus.rs` that were never created (their roles landed as
`ingest.rs`/`seal.rs`/`sync.rs`). `queue_driver.rs:21-26` still says "This
brick is additive: nothing is flipped yet" — the flip landed in #4781.
Coupling is bidirectional by design (seam reaches back into
`memory_store`/`memory_tree`/`memory_sync` to do host work inside crate
callbacks); not a defect, but `queue_driver.rs`'s 11 host imports will thin as
WP-1/WP-3 shims retire.
### 4.3 Broken/dangling references (resolved in WP-0)
| Resolved issue | Resolution |
| --- | --- |
| Missing TinyAgents companion-spec link | Replaced with the existing July 22 migration plan. |
| Missing standalone deletion-ledger link | Replaced with §2 of the TinyCortex migration spec. |
| Wrong agent-harness documentation path | Pointed at the architecture subdirectory. |
| Never-landed golden-workspace generator | Removed after the post-cutover parity decision descoped synthetic pre-cutover fixtures. |
| Wrong raw-coverage test path | Corrected to the `tests/raw_coverage/` location. |
| Stale line anchors `Cargo.toml:82`, `src/openhuman/mod.rs:130` | `tinycortex-migration-spec.md` §0.4 (actual: `Cargo.toml:116`, `mod.rs:140`) |
| Never-created seam sibling names | Replaced with the adapters that actually landed. |
| Developer-specific absolute path in crate docs | Replaced with repository-relative paths. |
Also stale-in-place: all five docs' status headers (pre-#4794), the drift
ledger's pin anchors, and the crate migration doc's claim that "TinyCortex
will not own memory sync" (superseded by the plan's §8 sync-inclusive
amendment and the crate's shipped `sync` module — the *scheduler/credentials/
RPC* stay host, the *engine* is crate; restate the boundary in one place).
### 4.4 Test inventory
- **In-crate (done):** engine coverage was ported by #4820; the crate runs its
own suite offline (wiremock Composio mock; live tests env-gated). W8's CI
wiring exists: `ci-lite.yml:859` runs the vendor suite on submodule-pointer
changes — **but with `--features git-diff,sync` only; `persona` is untested
in host CI while enabled in the host build.**
- **Dies with WP-1:** `memory_store/unified/*_tests.rs` (~3.7k LOC:
`documents_tests` 1,275, `query_tests` 1,000, `profile_tests` 763,
`segments_tests` 393, `events_tests` 287) go with the unified tier; port any
still-unique assertions to the per-kind crate backends first.
- **Retargets with WP-2:** the memory_sync suites (389 inline tests across 35
files + provider `*_tests.rs`: github 655, gmail 301+354, slack 180, linear
172, clickup 155, notion 119) — post-flip, provider *engine* behavior is
crate-tested; host keeps orchestration/bus/config tests. Integration:
`memory_sync_pipeline_e2e.rs` (570), `memory_golden_parity_e2e.rs` (275),
`raw_coverage/memory_sync_*` retarget to the crate-backed path.
- **Golden parity harness:** comparators 1 & 5 green; 2/3/4 TODO;
`frontmatter_parity` Layer-1 asserter pending; fixture generator script
missing. Decide finish-vs-descope in WP-0 (post-cutover, parity pins matter
mainly for `MemoryTaint`/on-disk stability — keep those, consider dropping
the rest).
- **Stays host:** `json_rpc_e2e.rs` memory sections, `memory_sources_e2e`
(837), `memory_tree_summarizer_e2e` (583), `memory_roundtrip_e2e`,
`memory_graph_sync_e2e`, `memory_fast_retrieve_e2e`, `autocomplete_memory_e2e`,
`memory_artifacts_e2e`, the 15 `raw_coverage/memory_*` modules, and all
subconscious/learning suites.
---
## 5. The plan — five work packages
Same conventions as the tinyagents companion plan: WP-0 first (hygiene,
unblocks honest review), then largely independent packages; drift-ledger rows
before deletions; failing-before/passing-after regression tests; small
validated commits on feature branches per submodule.
### WP-0 — Version/tag + CI + docs housekeeping (no engine behavior change)
1. **Establish the tag discipline the Cargo comment promises:** tag
`vendor/tinycortex` (e.g. `v0.1.1` at the current release point, or cut
`v0.2.0` at HEAD), and record the bump workflow the plan prescribed
(`chore(vendor): bump tinycortex …` — zero such commits exist in host
history). Alternatively amend the comment to describe reality
(main-tracking submodule). Either is fine; the current state (comment
demands tags, none exist) is the worst option.
2. **Align the nested tinyagents:** when the tinyagents plan's WP-0 bumps
`vendor/tinyagents` to v2.1.0, bump `vendor/tinycortex/vendor/tinyagents`
in lockstep and update tinycortex's `tinyagents = "2"` if needed; then
evaluate retiring the `[patch."…senamakel/tinyagents"]` block and its
obsolete "while tinyagents #58 is pending" comment (#58 is merged).
3. **Close the persona CI hole:** add `persona` to the vendor-suite features
in `.github/workflows/ci-lite.yml:859` (and confirm the crate's own
feature-matrix covers `persona`, which today it does not list either).
4. **Docs refresh:** update the five docs' status headers to post-#4794/#4820
reality; update drift-ledger anchors (pin `daaaf6ba`+tag, note #4794/#4820/
#4863); document the `persona` feature + seam (currently invisible in all
five docs); restate the sync ownership boundary once (engine=crate,
scheduler/credentials/RPC/bus=host) and fix the crate-side migration doc's
contradictory "will not own sync" line + its absolute local path.
5. **Fix the §4.3 broken references**, including the two code-adjacent ones:
correct the seam module inventory and the queue-driver's stale pre-flip
description.
6. **Golden-parity decision:** finish comparators 2/3/4 + `frontmatter_parity`
+ the fixture script, **or** descope to the security-relevant pins
(`MemoryTaint` byte-identity, chunk/vector on-disk format) with the
decision recorded in the parity checklist.
**Exit:** pins/tags/CI/docs agree with the tree; `grep` for the phantom paths
comes back clean.
### WP-1 — Retire `memory_store/unified/` (the "staging for removal" tier)
1. Inventory the five remaining unified facets (documents, query, segments,
events, profile — ~5k LOC) against their per-kind crate backends; flip
callers facet-by-facet (the W3 sub-store pattern, already proven for
chunks/content/vectors/kv/entities/trees/safety).
2. The blocker is **G1** (`sqlite_conn()` escape hatch, ~312 refs at audit
time): re-audit the current count, then either land the crate-side access
the gap audit sketched or explicitly re-scope G1 to the host-retained
namespace-document tables and record it.
3. Decide the fate of the 10-table host-retained `UnifiedMemory`
namespace-document tier: keep host (spec's standing decision) but move it
out of `unified/` into a clearly-named home so "staging for removal" can
actually be removed.
4. Tests: port unique assertions from the ~3.7k LOC of `unified/*_tests.rs`
to per-kind backends (crate-side where engine behavior, host-side where
glue), then delete with the code.
**Exit:** `memory_store/unified/` gone or reduced to the renamed host tier;
`memory_store/` ≈ 910k LOC of glue + host tier; `memory_roundtrip_e2e` +
`memory_golden_parity_e2e` + full mock-suite green.
### WP-2 — W-SYNC.3: flip `memory_sync/` onto the crate sync engine
The seam (`sync.rs`: `ExternalSourceReader`, `SkillDocSink`,
`LocalDocumentSink`, `SyncStateStore`, `SyncEventSink`) is landed; the crate
ships the Composio engine incl. per-provider modules. Remaining:
1. Close the **D4.1D4.4** drift rows (ledger requires this before the flip).
2. Flip provider-by-provider (gmail first — the crate already carries the
Gmail 413-page fix #73), keeping host-side: scheduler/periodic trigger,
credentials, `config.toml` source registry (`memory_sources/`), event-bus
publishing (`ChannelMessageReceived` et al.), RPC/status surface, and
redaction/`source_scope` gating.
3. Dedupe post-processing: host `providers/*/post_process*` vs crate provider
modules — upstream generic normalization, keep host product policy.
4. Tests per §4.4: engine behavior moves to crate suites (wiremock Composio),
host keeps orchestration/bus/e2e; `memory_sync_pipeline_e2e` retargets.
**Exit:** `memory_sync/` shrinks from 17.7k toward ~68k LOC of
orchestration/product glue; D4 rows CLOSED; no duplicated provider parsing.
### WP-3 — Embedding-provider dedupe (lands in *tinyagents*, coordinated here)
Two host clusters now duplicate `tinyagents::harness::embeddings` (post-#58:
openai/cohere/voyage/ollama/cloud + rate-limit/retry-after):
1. `src/openhuman/embeddings/` adapters (`openai/voyage/cohere/ollama/
cloud_adapter.rs`, `provider_trait.rs`) → construct crate
`EmbeddingModel`s directly; host keeps `rpc.rs` (1,479), `schemas.rs`,
`catalog.rs`, `factory.rs` (config/BYOK selection, #4056 dimension
gating).
2. `memory_tree/score/embed/{openai_compat,ollama,cloud,factory}.rs` (~1.9k)
→ same crate models via the seam's `SeamEmbedder`/`EmbedderBridge`.
Because tinycortex consumes the same `EmbeddingModel` trait, this single
change serves both crates — it is the concrete payoff of the W-EMB decision.
Sequence after WP-0's tinyagents version alignment.
**Exit:** one embedding-provider implementation per provider across the
workspace; `embeddings_rpc_e2e` (822) + `ollama_embeddings_fallback_e2e` (234)
green against the crate-backed path.
### WP-4 — Shim retirement + funnel decision
1. **Funnel policy:** adopt direct `tinycortex::memory::*` imports as the
convention (matches reality and the tinyagents precedent), shrink the seam
`mod.rs` facade to the types host code genuinely re-brands
(`MemoryTaint` stays pinned by parity tests) — or the reverse; either way,
record it in the spec and stop carrying both.
2. Retire compatibility shims whose consumers can move: `memory_tree/tools.rs`
(points at `memory`), `memory_store/trees` legacy paths, the
`memory_queue`→`memory::jobs` re-export, `memory_archivist` (44-line shim —
move its 2 call sites and delete the domain), and the W5/W7 near-empty
shims once import counts hit zero (`memory_search/scoring.rs`,
`vector/mmr.rs`, `memory_tools/{types,store}.rs`).
3. Thin `queue_driver.rs` (1,007) as WP-1 removes its `memory_store`
reach-backs.
**Exit:** shim count measurably down (each deletion = one commit with its
import-migration); seam ≤ ~2.5k LOC; no module whose entire body is a
re-export except deliberate facades recorded in the spec.
### WP-5 — Exit gate
- Full suite: `scripts/test-rust-with-mock.sh` (incl. the 15
`raw_coverage/memory_*` modules), vendor suite
`cargo test --manifest-path vendor/tinycortex/Cargo.toml --features
git-diff,sync,persona`, slim disabled build (`--no-default-features
--features tokenjuice-treesitter` — memory domains are ungated but the
standing repo rule applies), `pnpm rust:check`.
- Drift ledger: D4 CLOSED; new rows for every WP-1/WP-2 deletion; the spec's
§2 deletion-ledger skeleton filled in with actuals.
- Docs: the five tinycortex docs + `src/openhuman/memory*/README.md` files +
`gitbooks/developing/architecture.md` reflect the post-flip reality; this
plan stamped done.
### Execution record (2026-07-22)
| Package | Result |
| --- | --- |
| WP-0 | TinyAgents unified at 2.1 across OpenHuman, TinyCortex, and TinyFlows; obsolete fork patch removed; TinyCortex CI covers `persona`; docs and phantom references repaired. TinyCortex intentionally tracks reviewed upstream commits rather than nonexistent tags. |
| WP-1 | G1 count is zero. The ten-table namespace/document product tier is host-owned, so the complete `unified/` directory was renamed to `namespace_store/` instead of deleted as engine duplication. Its 153 targeted tests pass. |
| WP-2 | The live default path was already TinyCortex-backed. D4 is CLOSED; dead Gmail duplicate removed; remaining provider task/profile projections explicitly classified as product policy. Provider tests pass (301). |
| WP-3 | Concrete provider transports deduplicated into TinyAgents. The memory tree now uses a thin `ProviderEmbedder`; Ollama's 8k context/batch and missing-model guidance moved upstream before the host client was deleted. TinyAgents embedding tests pass (38); host library check passes. |
| WP-4 | Archivist, search scoring/MMR, tool-memory type/store, tree-tool, jobs-alias, and unused seam type facades retired. Direct crate imports are canonical. Seam production code is 2,229 LOC (below the 2.5k exit target). |
| WP-5 | Local focused validation is recorded above; the slim `--no-default-features --features tokenjuice-treesitter` build also passes. [CI Full run 29925645209](https://github.com/senamakel/openhuman/actions/runs/29925645209) is green: core quality and full tests, TinyCortex, Tauri, frontend, mock-backend Rust E2E, Playwright, three desktop builds, every launched desktop shard, and the final gate. Two first-attempt Linux jobs ended without uploaded logs; rerunning only failed jobs passed the Linux build, Rust integration suite, and all eight Linux shards. |
---
## 6. Risks and standing gotchas
- **`MemoryTaint` is security-critical.** The type is proven byte-identical
host↔crate and fails closed to `ExternalSync`; the parity pins in
`tinycortex/parity.rs` are the regression guard — whatever WP-0 decides
about the golden harness, **keep these**.
- **Redaction and `source_scope` gating stay host.** WP-2 must not let crate
sync code become a path around host redaction or the source allowlist —
review the `SyncEventSink`/`LocalDocumentSink` implementations for every
provider flipped.
- **On-disk compat:** chunk/vector formats, `MEMORY_GOALS.md` byte-identity,
and the P5 fresh-DB divergence (3 legacy inline-embedding columns in
`mem_tree_chunks`) — migrations of user data must be forward-only and
tested against a pre-migration workspace fixture.
- **MIT vs GPL:** tinycortex is MIT (tinyagents is GPL-3.0) — moving host code
*down* into tinycortex relicenses it; fine for our own code, but keep the
"no product policy / no keys / nothing openhuman-branded as API" rule, and
don't move anything derived from GPL-only sources into the MIT crate.
- **Two Cargo worlds + nested submodule:** vendor bumps now touch up to three
lockfiles (root, `app/src-tauri`, and tinycortex's own) and two submodule
pointers (`vendor/tinycortex`, `vendor/tinycortex/vendor/tinyagents`). CI
clones with `submodules: recursive` for the vendor suite — verify after any
bump.
- **Event-bus surface:** `memory_sync` publishes 7 distinct `DomainEvent`
variants; the WP-2 flip must keep publishing them from host callbacks (the
crate has no event bus by design).
---
## 7. Quick reference — disposition of every audited module
| Module | Verdict |
| --- | --- |
| `memory_store/unified/` | DELETE after per-kind flips; host tier re-homed (WP-1) |
| `memory_store/` sub-stores, `factories.rs` | DONE (crate-backed) / STAYS (host glue) |
| `memory_sync/` provider engines | FLIP to crate `memory::sync` (WP-2) |
| `memory_sync/` scheduler, bus, RPC, credentials | STAYS |
| `embeddings/` per-provider adapters | REPLACE with `tinyagents::harness::embeddings` models (WP-3) |
| `embeddings/` rpc/schemas/catalog/factory | STAYS |
| `memory_tree/score/embed/` providers | REPLACE via seam + crate models (WP-3) |
| `memory_tree/` engine files | DONE (crate-backed); RPC/CLI/bus/health STAY |
| `memory_queue/`, `memory_search/`, `memory_tools/`, `memory_diff/`, `memory_goals/`, `memory_conversations/` | DONE — retire residual shims when import counts reach zero (WP-4) |
| `memory_archivist/` | DELETE the 44-line shim after moving 2 call sites (WP-4) |
| `memory/` (orchestration/RPC/tools/tree-policy) | STAYS |
| `memory_sources/` | STAYS (product registry over crate readers) |
| `agent_memory/`, `subconscious/`, `learning/` | STAYS (consumers, no duplication) |
| Seam `src/openhuman/tinycortex/` | STAYS, shrinks; fix stale module docs (WP-0), thin `queue_driver.rs` (WP-4) |
| Golden-parity harness | FINISH or DESCOPE by decision (WP-0); `MemoryTaint`/format pins KEEP |
+29 -15
View File
@@ -1,11 +1,10 @@
# TinyCortex Memory Migration — Spec (Phase 0.5 / 0.6)
**Status:** Phase 0 baseline **+ execution underway** — #59 (native-dep alignment) merged and the dep
is active (§0.4); W1 seam + W2 type re-export + W3-chunks partial landed; **drift closure COMPLETE**
(**D3** via #59, **D2** via tinycortex#63, **D1** via tinycortex#64; gitlink `a8e10f7`) — so **W4
(queue) and W7 (conversations) are unblocked**. Anchors the migration to exact reviewed SHAs and ties
together the drift, gap, and parity ledgers. Modeled on `docs/tinyagents-migration-spec.md` + its
deletion ledger.
**Status:** Post-engine-cutover reference. W1W8 and the crate-owned engine test
port landed in OpenHuman #4794/#4820, with persona/coding-session ingest in
#4863. The remaining host consolidation is tracked by
[`tinycortex-migration-plan-2026-07-22.md`](tinycortex-migration-plan-2026-07-22.md).
This document retains the ownership contract and deletion ledger detail.
**Companion plan:** [`tinycortex-memory-migration-plan.md`](tinycortex-memory-migration-plan.md)
**Ledgers:** [`tinycortex-drift-ledger.md`](tinycortex-drift-ledger.md) ·
@@ -19,7 +18,10 @@ deletion ledger.
| `tinyhumansai/openhuman` | `7850cf363559bcbb7ba688cbc4fccdb6bd9ce754` | host audit base (`main`, 2026-07-04) |
| `tinyhumansai/tinycortex` | `d1a8c7be2babc8fff7a72ed93861f459f3d6fa58` | crate audit base (v0.1.1) |
| `tinyhumansai/tinycortex` | `33dda943053e61ef585fc39647cf1854344b6323` | audit base **+ #59** (native-dep alignment, §0.4) merged |
| `tinyhumansai/tinycortex` | `a8e10f7dd8ebdb9b0905e1380fefcc6bf5a65207` | **current pinned gitlink****+ #63/#64** (D2/D1 drift closure) merged; all drift rows CLOSED |
| `tinyhumansai/tinycortex` | `a8e10f7dd8ebdb9b0905e1380fefcc6bf5a65207` | historical cutover gitlink — **+ #63/#64** (D2/D1 drift closure) merged |
| `tinyhumansai/openhuman` | `5b8a9f269` | 2026-07-22 consolidation audit base |
| `tinyhumansai/tinycortex` | `daaaf6ba5f02635c08deae2b2b2ed7fcc8c06b6a` | 2026-07-22 reviewed gitlink; no upstream tags exist, so the host tracks reviewed main commits |
| `tinyhumansai/tinycortex` | `7b4b115` | current migration branch: TinyAgents 2.1 alignment + corrected ownership docs |
Port line (derived by content, §0.1): **after 2026-06-25, before 2026-06-28** for engine features.
@@ -72,14 +74,14 @@ ledger. Re-export covers the data types (`MemoryEntry`, `MemoryCategory`, `Memor
| **rusqlite alignment** | ✅ **resolved & merged (#59).** Crate was pinned `0.32` (bundled), host pins `=0.40.0` (bundled). Two `links = "sqlite3"` = hard Cargo error. Fixed in #59 (bump to `0.40` + `usize``i64`/`try_from` — the same sweep that closed drift **D3**). |
| **git2 alignment** | ✅ **resolved & merged (#59).** Crate was pinned `0.19`, host `0.21` (vendored-libgit2). Two `links = "git2"`. Fixed in #59 (bump to `0.21` + API deltas: `Tag::message`, `StringArray::Iter`, `Buf::as_str`). |
| Crate compiles with aligned deps | ✅ `cargo check --all-targets` clean; 38 diff/checkpoint tests pass. |
| **Host root world compiles with dep active** | ✅ `cargo check --manifest-path Cargo.toml --lib` **exit 0** with `tinycortex = "0.1"` active (`Cargo.toml:82`) + submodule at `33dda94`. **No `multiple packages link to native library` error** — one bundled SQLite + one libgit2 confirmed. **Now landed** (post-#59-merge): the `[dependencies]` line and gitlink are committed on the working branch, not reverted. Re-verified 2026-07-09. |
| **Host root world compiles with dep active** | ✅ `cargo check --manifest-path Cargo.toml --lib` **exit 0** with `tinycortex = "0.1"` active (`Cargo.toml:116`) + submodule at `33dda94`. **No `multiple packages link to native library` error** — one bundled SQLite + one libgit2 confirmed. **Now landed** (post-#59-merge): the `[dependencies]` line and gitlink are committed on the working branch, not reverted. Re-verified 2026-07-09. |
| Host `app/src-tauri` world | to verify in W1 (separate Cargo world / lockfile). |
| `GGML_NATIVE=OFF` macOS ARM | to verify on a macOS runner in W1 (no macOS host here). |
**Activation landed (post-#59).** The native-dep alignment merged upstream as **#59** and the host
gitlink was bumped to `33dda94`, so — per the submodule rule (bump only to a **merged** SHA) — W1's
activation is now in place: `[dependencies] tinycortex = "0.1"` is active in the root world
(`Cargo.toml:82`), the seam (`src/openhuman/tinycortex/`) is wired (`src/openhuman/mod.rs:130`), and
(`Cargo.toml:116`), the seam (`src/openhuman/tinycortex/`) is wired (`src/openhuman/mod.rs:140`), and
`cargo check --lib` is **exit 0**. Remaining §0.4 follow-ups: the `app/src-tauri` world and the
`GGML_NATIVE=OFF` macOS-runner check still verify in W1 (see rows above).
@@ -117,7 +119,7 @@ activation is now in place: `[dependencies] tinycortex = "0.1"` is active in the
coexist in the shared DB but **do not move**: `memory_docs`, `graph_global`, `graph_namespace`,
`episodic_log` (+ `episodic_fts` + triggers), `event_log` (+ `event_fts`, `event_embeddings`,
triggers), `conversation_segments`, `segment_embeddings`, `vector_chunks`, `user_profile`.
These live in `memory_store/unified/{init,fts5,events,segments,profile}.rs` and remain host — the
These live in `memory_store/namespace_store/{init,fts5,events,segments,profile}.rs` and remain host — the
crate is the **primitive substrate**, not a drop-in for the whole DB.
- **Content-store host surfaces the crate explicitly excludes:** `content::wiki_git`,
`content::obsidian`, `content::obsidian_registry`.
@@ -128,8 +130,11 @@ activation is now in place: `[dependencies] tinycortex = "0.1"` is active in the
`embeddings.rs` (`EmbeddingBackend`/`Embedder`), `chat.rs` (`ChatProvider`/`Summariser`×2/
`EntityExtractor`/`GoalsGenerator`), `queue_driver.rs` (`QueueDelegates` + tokio worker loop +
Sentry/bus), `config.rs` (`Config``MemoryConfig`), `sinks.rs` (`TreeJobSink`/`TreeLeafSink`/
`SnapshotItemSource`/`EntityOccurrenceIndex`), `bus.rs` (engine outcomes → `DomainEvent`),
`mod.rs` (facade re-exports + boundary doc). All 17 W1 seam traits confirmed present (§0.2).
`SnapshotItemSource`/`EntityOccurrenceIndex`), `sync.rs` (sync outcomes → `DomainEvent`),
`mod.rs` (adapter namespace + compatibility re-exports + boundary doc). New
engine consumers import `tinycortex::memory::*` directly; the seam owns
implementations rather than serving as a second type funnel. All 17 W1 seam
traits confirmed present (§0.2).
**Later seam additions (amended 2026-07-09):**
@@ -144,7 +149,7 @@ Sentry/bus), `config.rs` (`Config`→`MemoryConfig`), `sinks.rs` (`TreeJobSink`/
---
## 2. Deletion ledger (skeleton)
## 2. Deletion ledger
Every legacy engine file is deleted only when its module's **drift row is closed**, its **gaps are
resolved**, and the **golden-workspace parity harness is green** for its flip. Counts from the host
@@ -152,7 +157,7 @@ audit SHA.
| Legacy module | Files (test files) | Deletes in | Preconditions |
| --- | --- | --- | --- |
| `memory_store/` | 66 (11) | W3 | drift **D3** closed; gap **G1** (escape hatch) migrated to `with_connection`; parity P3/P5/P11/P12 green; `unified/` tier re-homed as host-retained (kept, not deleted) |
| `memory_store/` | 66 (11) | W3/WP-1 | drift **D3** closed; gap **G1** migrated completely (zero `sqlite_conn()` call sites at the 2026-07-22 audit); parity P3/P5/P11/P12 green; namespace tier re-homed to `namespace_store/` as host-retained |
| `memory_tree/` | 65 (7) | W5 | gaps **G3** (seal-embed), **G6** (2× Summariser) resolved; `source_scope` allowlist re-verified; parity P7/P11 green; `health/` + `tree_policy.rs` kept host (G5) |
| `memory_queue/` | 10 (1) | W4 | drift **D2** closed (predicate upstreamed); job payload_json parity (P4/P9); host worker loop + Sentry/degraded wiring kept host |
| `memory_conversations/` | 7 (1) | W7 | drift **D1** closed; `bus.rs` kept host |
@@ -168,10 +173,19 @@ audit SHA.
| `memory_search/` | 8 (0) | W5 | `vector`/`scoring` → crate `retrieval`/`score`; `tools/` kept host |
| `memory/ingest_pipeline.rs` internals | (thin entry points kept) | W6 | `ingest_chat`/`ingest_document_with_scope` signatures unchanged; 11 call sites untouched |
### 2026-07-22 consolidation actuals
| Package | Actual result |
| --- | --- |
| WP-1 namespace tier | `memory_store/unified/` removed and re-homed byte-for-byte as `memory_store/namespace_store/`; G1 re-audit found zero `sqlite_conn()` call sites. The retained ten-table product store explains why total `memory_store` remains 17.7k LOC rather than the plan's speculative 910k target. |
| WP-2 sync | The default provider `sync()` and `run_connection_sync` already call the TinyCortex engine. Deleted the dead host Gmail sync parser; renamed GitHub/Notion/Linear/ClickUp product projections from `sync.rs` to `normalization.rs`. D4.1-D4.4 are CLOSED. The retained 17.2k LOC is schedulers, bus/RPC, action tools/catalogs, credentials, profiles, post-processing, and product task projections; 6.9k LOC is provider catalogs/tools/normalization/profile/RPC alone. |
| WP-3 embeddings | Deleted host OpenAI, Cohere, Voyage, general Ollama, memory-tree cloud, and memory-tree Ollama provider implementations (746 production LOC) plus their obsolete 828-line raw-coverage suite. Provider transport now has one implementation in TinyAgents; OpenHuman retains selection and credential/privacy adapters. |
| WP-4 shims | Deleted `memory_archivist`, `memory_search::{scoring,vector}`, `memory_tools::{types,store}`, `memory_tree::tools`, and the `memory::jobs` alias. Removed the unused TinyCortex type facade; direct `tinycortex::memory::*` imports are the convention. The seam is 2,229 pre-test LOC. |
**Kept host (never deleted):** `memory/{ops,schemas,schema,read_rpc,tools,query,tree_source,
ingestion,util}`, `memory/{global,source_scope,chat,sync,preferences,remember,tree_policy,rpc_models,
traits(→re-exports)}.rs`, `memory_sync/`'s host-retained shell only (schedulers `periodic.rs`,
`bus.rs` subscribers, RPC registration — the engine moves in W-SYNC, plan §8), `memory_store/unified/*` (the namespace-document
`bus.rs` subscribers, RPC registration — the engine moves in W-SYNC, plan §8), `memory_store/namespace_store/*` (the namespace-document
tier), `memory_store/content/{wiki_git,obsidian,obsidian_registry}`, `memory_tree/health/`, and the
new `src/openhuman/tinycortex/` seam.
+25 -17
View File
@@ -1,5 +1,13 @@
# TinyCortex Data-Format Parity Checklist (Phase 0.3)
**Status (2026-07-22):** The engine cutover and crate test port are complete.
The proposed pre-migration golden-fixture generator and differential
comparators 24 are deliberately **descoped**: there is no trustworthy
pre-cutover binary/fixture left to generate them from after cutover. Retained
gates are the security-critical `MemoryTaint` byte/serde pins, deterministic
chunk IDs, vector encoding/signature, content paths, schema composition,
idempotent reopen, and the host public-surface E2E suites.
**Purpose.** Existing user workspaces must open **unchanged** after every cutover flip.
This checklist enumerates every on-disk format shared between the host engine and TinyCortex,
records the audit result, and specifies the **golden-workspace parity harness** that gates W3, W5, W6.
@@ -7,7 +15,8 @@ records the audit result, and specifies the **golden-workspace parity harness**
**Hard rule (plan §0.3/§6):** any mismatch is fixed **upstream in tinycortex**, never papered over
with a host shim.
Anchors: host `7850cf363` · tinycortex `d1a8c7be` (v0.1.1).
Anchors: historical audit host `7850cf363` · crate `d1a8c7be`; consolidation
base host `5b8a9f269` · reviewed crate `daaaf6ba` · migration branch `7b4b115`.
## Ownership tiers in the shared workspace (key finding)
@@ -28,10 +37,10 @@ A user workspace's `chunks.db` (and content vault) holds **two tiers**:
`event_ai/ad/au` triggers), `conversation_segments`, `segment_embeddings`, `vector_chunks`,
`user_profile` **(10 tables + FTS + triggers)**.
These live in `src/openhuman/memory_store/unified/{init,fts5,events,segments,profile}.rs`.
**W3 must keep the host creating/reading these in the same DB the crate now manages**
These live in `src/openhuman/memory_store/namespace_store/{init,fts5,events,segments,profile}.rs`.
**The host continues creating/reading these in the same DB the crate now manages**
the crate's `chunks::with_connection` opens the shared handle; host `UnifiedMemory` schema
init runs alongside the crate's. Parity requirement: crate schema init and host `unified`
init runs alongside the crate's. Parity requirement: crate schema init and host namespace-store
init must **compose without collision** on both fresh and existing DBs.
## Parity results by format dimension
@@ -93,17 +102,15 @@ The core mechanism from plan §0.3: **one on-disk workspace, opened by both engi
> tables) and the host `UnifiedMemory` tier (10 tables) **coexist without collision** (P3/P5/P11/P12),
> and that re-running the flow adds/drops no tables (comparator 5). Still TODO: `vectors`/`store_meta`/
> `kv_*` (created by the chunk/embed pipeline, need a widened ingest flow), the seeded golden fixture +
> `scripts/gen-golden-workspace.sh`, and comparators **2** (recall/retrieval snapshot), **3** (tree
> read), **4** (byte-compare vault) — these require a populated, sealed fixture and the W5 retrieval
> surface, so they land alongside the W3/W5 flips.
> Comparators **2** (recall/retrieval snapshot), **3** (tree read), and **4**
> (byte-compare vault) were never landed before cutover and are now descoped by
> the decision above. Comparator 1 (schema composition) and comparator 5
> (idempotent reopen) remain active.
**Fixture.** Check in a small, deterministic `tests/fixtures/golden-workspace/` produced by the
*pre-migration* build: a real `chunks.db` + content vault + diff `.git`, seeded via a fixed script
(`scripts/gen-golden-workspace.sh`) with: a handful of chat + document + email sources across ≥2
namespaces, ingested + scored + sealed to ≥2 tree levels, some entities/edges, a few queue jobs in
mixed states, and both tiers populated (episodic/event/segment rows present). Store the **generator
script** and a **manifest** (expected chunk-ids, summary paths, recall snapshots) so the fixture is
regenerable and reviewable, not an opaque blob.
**Fixture decision.** Do not synthesize a fixture and label it “pre-migration”
after the fact. Compatibility is guarded by the retained format pins, schema
composition/idempotency test, and real host E2E fixtures. A future format
migration may add a versioned fixture captured before that migration starts.
**Comparators (read-only, both engines open the SAME copied workspace):**
@@ -124,7 +131,8 @@ regenerable and reviewable, not an opaque blob.
**Wiring.** Runs under `pnpm test:rust` (host-side, counts toward coverage) and as a dedicated
`tests/memory_golden_parity_e2e.rs`. The existing crate-level integration tests
(`tests/memory_roundtrip_e2e.rs`, `memory_tree_sync_deep_raw_coverage_e2e.rs`) act as the
(`tests/memory_roundtrip_e2e.rs`,
`tests/raw_coverage/memory_tree_sync_deep_raw_coverage_e2e.rs`) act as the
"public-surface still green" guard (plan §5.1); this harness adds the **differential** guard that a
flip preserves *existing* data, not just that the API still functions.
@@ -154,13 +162,13 @@ Engine tests now run at their ownership boundary rather than through OpenHuman r
surfaces, agent-tool response post-processing, source registry side effects, and the
security-critical `MemoryTaint` seam.
- OpenHuman CI now runs `cargo test --manifest-path vendor/tinycortex/Cargo.toml --features
git-diff,sync` when the submodule pointer changes and in the reusable full Rust suite. This is
git-diff,sync,persona` when the submodule pointer changes and in the reusable full Rust suite. This is
required because Cargo does not run dependency test targets while testing `openhuman`.
The focused local verification commands are:
```bash
cargo test --manifest-path vendor/tinycortex/Cargo.toml --features git-diff,sync
cargo test --manifest-path vendor/tinycortex/Cargo.toml --features git-diff,sync,persona
cargo test --test raw_coverage_all memory_sync -- --test-threads=1
cargo test --test memory_sync_pipeline_e2e --test memory_artifacts_e2e \
--test memory_golden_parity_e2e --test memory_roundtrip_e2e --test memory_sources_e2e
+2 -2
View File
@@ -291,10 +291,10 @@ Every layer is async and non-blocking. The Rust core processes thousands of conc
## Vendored crate family & recent shifts
Core subsystems are being re-platformed onto published `tiny*` crates, vendored as git submodules under `vendor/` (`tinyagents`, `tinyflows`, `tinycortex`, `tinychannels`, `tinyjuice`, `tinyplace`) so crate changes can be tested in-tree before publishing. The major ongoing shifts:
Core subsystems run on published `tiny*` crates, vendored as git submodules under `vendor/` (`tinyagents`, `tinyflows`, `tinycortex`, `tinychannels`, `tinyjuice`, `tinyplace`) so crate changes can be tested in-tree before publishing. The major ownership boundaries are:
- **Agent engine on tinyagents** — every agent turn runs through the `tinyagents` crate harness via the seam in `src/openhuman/tinyagents/`; see [Agent Harness](architecture/agent-harness.md).
- **Memory on tinycortex** — memory modules (`memory_diff`, `memory_conversations`, `memory_queue`, …) are being deleted or shimmed onto the `tinycortex` crate engine (W7 migration; #4785#4788).
- **Memory on tinycortex** — the generic store/tree/queue/retrieval/sync engine is crate-owned. OpenHuman keeps RPC, tools, scheduling, credentials, security/event policy, worker orchestration, and the host namespace-document store; `src/openhuman/tinycortex/` implements those seams. Concrete embedding transports are shared through `tinyagents::harness::embeddings`.
- **Inference on the crate ModelRouter** — host workload-tier model routing and cloud provider slugs now use the crate-native `ModelRouter`/`OpenAiModel` (#4782, #4783).
- **Hosted-only brain** — the client-local orchestration graph engine (`src/openhuman/orchestration/graph/`) was retired (#4738); the client is a thin hosted-brain participant (pushers, effect/tool executors, wire allowlist — #4725) surfaced in the `/orchestration` and `/brain/tinyplace-orchestration` routes.
+5 -6
View File
@@ -674,7 +674,7 @@ pub fn expected_error_kind(message: &str) -> Option<ExpectedErrorKind> {
/// disk-full condition during its own page bookkeeping (journal/WAL extension)
/// before the next syscall surfaces an errno, rusqlite renders the `SQLITE_FULL`
/// result code as `"database or disk is full"` (Sentry TAURI-RUST-B6N, hit at
/// `memory_store::unified::documents::tx.commit()` during
/// `memory_store::namespace_store::documents::tx.commit()` during
/// `openhuman.memory_doc_ingest`). `SQLITE_FULL` has only two causes:
/// genuine ENOSPC/ERROR_DISK_FULL (always the case in practice — the same
/// burst always produces an os-error-28/112 sibling event) or a
@@ -683,7 +683,7 @@ pub fn expected_error_kind(message: &str) -> Option<ExpectedErrorKind> {
/// rusqlite renders `SQLITE_FULL` in one of two shapes. The **bare** shape is
/// the five words `"database or disk is full"` — Our local memory-store write
/// call-sites wrap it with `format!("<verb>: {e}")` (e.g. `"commit tx: ..."` /
/// `"clear_namespace commit tx: ..."` in `memory_store::unified::documents`),
/// `"clear_namespace commit tx: ..."` in `memory_store::namespace_store::documents`),
/// so the phrase lands as the **suffix** of the local emit. The **extended**
/// shape carries the full error-code envelope, `"database or disk is full:
/// Error code 13: Insertion failed because database is full"` (Sentry
@@ -1190,9 +1190,8 @@ fn is_loopback_unavailable(lower: &str) -> bool {
/// the local Ollama daemon — pure user-state errors the UI already surfaces
/// (toast / settings page warning) where Sentry has no remediation path.
///
/// Several canonical wire shapes are covered, all emitted by
/// `openhuman::embeddings::ollama::OllamaEmbedding::embed` and the embed
/// service fallback path:
/// Several canonical wire shapes are covered, all emitted by the TinyAgents
/// Ollama embedder and the host embed service fallback path:
///
/// - **TAURI-RUST-XS** (~376 events on self-hosted Sentry): user pointed the
/// embedder at a chat / vision model id with a temperature suffix (e.g.
@@ -4250,7 +4249,7 @@ mod tests {
// SQLITE_FULL rendering from rusqlite — engine-level disk-full
// detection during page-bookkeeping (journal/WAL extension) that
// beats the next syscall to the errno. Production hit at
// `memory_store::unified::documents::tx.commit()` during
// `memory_store::namespace_store::documents::tx.commit()` during
// `openhuman.memory_doc_ingest`, in the same burst that emits
// os-error-112 siblings (Sentry TAURI-RUST-B6N).
"commit tx: database or disk is full",
@@ -80,15 +80,17 @@ impl PostTurnHook for ArchivistHook {
tracing::debug!("[archivist] episodic rows written: session={session_id}");
// Dual-write into memory_archivist::store (md-backed) so we can
// Dual-write into the crate-owned archivist store (md-backed) so we can
// validate the FTS5 → md migration before flipping the read side.
// Best-effort: a write failure here must not break the turn. The
// user turn's assigned seq is captured into `current_seq` so the
// segment ops can store it alongside the FTS5 episodic id.
let mut current_seq: Option<u32> = None;
if let Some(cfg) = self.config.as_ref() {
let engine_config =
crate::openhuman::tinycortex::memory_config_from(cfg, cfg.workspace_dir.clone());
let ts_ms = (timestamp * 1000.0) as i64;
let user_turn = crate::openhuman::memory_archivist::ArchivedTurn {
let user_turn = tinycortex::memory::archivist::types::ArchivedTurn {
session_id: session_id.to_string(),
seq: 0, // assigned by record_turn
timestamp_ms: ts_ms,
@@ -98,7 +100,7 @@ impl PostTurnHook for ArchivistHook {
tool_calls_json: None,
cost_microdollars: 0,
};
match crate::openhuman::memory_archivist::store::record_turn(cfg, user_turn) {
match tinycortex::memory::archivist::store::record_turn(&engine_config, user_turn) {
Ok(stored) => current_seq = Some(stored.seq),
Err(e) => {
tracing::warn!("[archivist] memory_archivist user dual-write failed: {e}");
@@ -113,7 +115,7 @@ impl PostTurnHook for ArchivistHook {
} else {
Some(serde_json::to_string(&ctx.tool_calls).unwrap_or_default())
};
let assistant_turn = crate::openhuman::memory_archivist::ArchivedTurn {
let assistant_turn = tinycortex::memory::archivist::types::ArchivedTurn {
session_id: session_id.to_string(),
seq: 0,
timestamp_ms: ts_ms + 1,
@@ -124,7 +126,7 @@ impl PostTurnHook for ArchivistHook {
cost_microdollars: 0,
};
if let Err(e) =
crate::openhuman::memory_archivist::store::record_turn(cfg, assistant_turn)
tinycortex::memory::archivist::store::record_turn(&engine_config, assistant_turn)
{
tracing::warn!("[archivist] memory_archivist assistant dual-write failed: {e}");
}
@@ -11,7 +11,7 @@ use std::sync::Arc;
impl ArchivistHook {
/// Read every entry recorded for `session_id`, preferring the
/// md-backed `memory_archivist::store` when `self.config` is set and
/// crate-owned md-backed archivist store when `self.config` is set and
/// falling back to the legacy FTS5 episodic table otherwise.
///
/// Returns `EpisodicEntry` so the existing call sites (segment
@@ -23,7 +23,10 @@ impl ArchivistHook {
session_id: &str,
) -> Vec<EpisodicEntry> {
if let Some(cfg) = self.config.as_ref() {
match crate::openhuman::memory_archivist::store::session_entries(cfg, session_id) {
let engine_config =
crate::openhuman::tinycortex::memory_config_from(cfg, cfg.workspace_dir.clone());
match tinycortex::memory::archivist::store::session_entries(&engine_config, session_id)
{
Ok(turns) => {
return turns
.into_iter()
+24 -99
View File
@@ -1,106 +1,31 @@
# embeddings
Embedding providers for the OpenHuman memory system. Converts text into numerical vectors for semantic search, abstracting over multiple backends behind a single `EmbeddingProvider` trait. Owns provider construction (factory + slug catalog), per-endpoint client-side rate limiting, 429/503 retry-with-backoff logic, and the JSON-RPC surface that the Settings UI uses to pick a provider, store API keys, test connectivity, and embed text. The default provider is the OpenHuman backend ("managed" / "cloud", Voyage-backed); other paths include direct Voyage, OpenAI, Cohere, local Ollama, any OpenAI-compatible `custom:<url>` endpoint, and a no-op (keyword-only) fallback.
Host policy and RPC surface for vector embeddings. Concrete OpenAI-compatible,
Cohere, Voyage, Ollama, cloud-transport, retry, and rate-limit implementations
live in `tinyagents::harness::embeddings`; this domain selects and adapts those
models for OpenHuman.
## Responsibilities
## Host-owned responsibilities
- Define `EmbeddingProvider` (async trait) and the canonical embedding-space signature format (`provider=…;model=…;dims=…`) — the single source of truth so config-derived and live-provider signatures stay byte-identical (#1574).
- Implement each provider: cloud/managed, Voyage, OpenAI(-compatible), Cohere, Ollama, noop.
- Construct providers from a slug + model + dims (with or without credentials) via the factory.
- Maintain the static provider/model catalog (slugs, labels, API-key/endpoint requirements, dimension presets) consumed by the frontend picker.
- Throttle outbound cloud-embedding HTTP requests with a process-global, per-endpoint token bucket (loopback exempt).
- Parse `Retry-After` and apply 429/503 exponential backoff in HTTP-based providers.
- Expose RPC handlers for settings, API-key management, embed, and connection testing; trigger memory wipe / re-embed backfill when the embedding signature changes.
- `factory.rs`: provider slug/model/dimension selection and construction of
TinyAgents models.
- `provider_trait.rs`: compatibility contract used by existing OpenHuman
consumers plus `TinyAgentsEmbeddingProvider`, the one trait adapter.
- `cloud_adapter.rs`: OpenHuman session-token resolution, egress disclosure,
and local-only privacy enforcement around TinyAgents `CloudEmbeddingModel`.
- `catalog.rs`, `rpc.rs`, `schemas.rs`: Settings catalog, credentials, JSON-RPC,
connection tests, and re-embed/wipe policy.
- `noop.rs`: unit-struct compatibility adapter over TinyAgents' no-op model.
## Key files
Provider HTTP behavior and its tests belong in `vendor/tinyagents`. Do not add
new per-provider clients here.
| File | Role |
| --- | --- |
| `src/openhuman/embeddings/mod.rs` | Module docstring + module decls + embedding provider `pub use` re-exports; exposes `all_embeddings_controller_schemas` / `all_embeddings_registered_controllers`. |
| `src/openhuman/embeddings/provider_trait.rs` | `EmbeddingProvider` trait (`name`/`model_id`/`dimensions`/`signature`/`embed`/`embed_one`) and `format_embedding_signature`. |
| `src/openhuman/embeddings/factory.rs` | `create_embedding_provider`, `create_embedding_provider_with_credentials`, `default_embedding_provider` (cloud), `default_local_embedding_provider` (Ollama). Maps provider slugs to concrete impls; unknown slugs error. |
| `src/openhuman/embeddings/catalog.rs` | Static catalog of `EmbeddingProviderEntry` + `EmbeddingModelPreset`; slug constants; `all_providers` / `find_provider` / `find_model` / `default_model_for`. |
| `src/openhuman/embeddings/cloud.rs` | `OpenHumanCloudEmbedding` — default provider; resolves session JWT + API URL per call, delegates HTTP to `OpenAiEmbedding` against `<api>/openai/v1`. `DEFAULT_CLOUD_EMBEDDING_MODEL` = `embedding-v1`, dims 1024. |
| `src/openhuman/embeddings/openai.rs` | `OpenAiEmbedding` — OpenAI-compatible `POST /v1/embeddings`; URL inference, 429/503 retry, rate-limit gating, dimension/count validation. Used directly by cloud, voyage, and custom paths. |
| `src/openhuman/embeddings/voyage.rs` | `VoyageEmbedding` — thin wrapper delegating to `OpenAiEmbedding` against `api.voyageai.com`. |
| `src/openhuman/embeddings/cohere.rs` | `CohereEmbedding` — Cohere-native `POST /v2/embed` wire format (`texts`, `embedding_types`, nested `embeddings.float`) with its own 429/503 retry loop. |
| `src/openhuman/embeddings/ollama.rs` | `OllamaEmbedding` — local Ollama `POST /api/embed`; base-url/model normalization, blank-input zero-vector preservation, NaN-encoding 500 per-text recovery (TAURI-RUST-AZ). Defaults `bge-m3`/1024. |
| `src/openhuman/embeddings/noop.rs` | `NoopEmbedding` — returns empty vectors; keyword-only fallback (`name`/`model_id` = "none", dims 0). |
| `src/openhuman/embeddings/rate_limit.rs` | Process-global, per-endpoint token-bucket request limiter; `set_embedding_rate_limit`, `embedding_rate_limit`, `acquire_embedding_slot`. Default 60/min; loopback exempt; `0` disables. |
| `src/openhuman/embeddings/retry_after.rs` | `parse_retry_after_ms`, `backoff_ms_for_attempt`, and the `MAX_429_RETRIES` / backoff constants. |
| `src/openhuman/embeddings/rpc.rs` | RPC business logic: `get_settings`, `update_settings`, `set_api_key`, `clear_api_key`, `embed`, `test_connection`, plus `provider_from_config` and `resolve_api_key`. |
| `src/openhuman/embeddings/schemas.rs` | Controller schemas + `handle_*` param-deserializing wrappers delegating to `rpc.rs`. |
| `src/openhuman/embeddings/mod_tests.rs` | Module-level tests (via `#[path]`). |
| `src/openhuman/embeddings/openai_tests.rs` / `ollama_tests.rs` | Sibling test suites for the OpenAI and Ollama providers (via `#[path]`). |
The canonical embedding-space signature is
`provider={name};model={model};dims={dims}`. Configuration-derived and live
provider signatures must remain byte-identical or stored vectors split into
incompatible spaces.
## Public surface
From `mod.rs` re-exports:
- Trait + helper: `EmbeddingProvider`, `format_embedding_signature`.
- Providers: `OpenHumanCloudEmbedding`, `OpenAiEmbedding`, `OllamaEmbedding`, `NoopEmbedding` (Voyage/Cohere are public via their submodules through the factory).
- Factory fns: `create_embedding_provider`, `default_embedding_provider`, `default_local_embedding_provider`.
- Config-driven builder: `provider_from_config` (re-exported from `rpc`) — same construction `embed` uses, for callers like `codegraph` that need a provider without a JSON-RPC round-trip.
- Defaults/consts: `DEFAULT_CLOUD_EMBEDDING_MODEL`, `DEFAULT_CLOUD_EMBEDDING_DIMENSIONS`, `DEFAULT_OLLAMA_MODEL`, `DEFAULT_OLLAMA_DIMENSIONS`.
- RPC registry: `all_embeddings_controller_schemas`, `all_embeddings_registered_controllers`.
## RPC / controllers
Namespace `embeddings` (6 controllers, registered through `src/core/all.rs`):
| Method | Description |
| --- | --- |
| `embeddings.get_settings` | Current provider/model/dims/rate-limit + full provider catalog with `has_api_key` flags and `vector_search_enabled`. |
| `embeddings.update_settings` | Update provider/model/dimensions/custom_endpoint/rate_limit. Requires `confirm_wipe=true` when **dimensions** change (returns `EMBEDDINGS_DIMENSION_CHANGE_REQUIRES_WIPE` otherwise); wipes memory on dim change and queues a re-embed backfill on any signature change. Also syncs `config.embeddings_provider` workload routing. |
| `embeddings.set_api_key` | Store an API key under credential provider `embeddings:<slug>`. |
| `embeddings.clear_api_key` | Remove the stored key for `embeddings:<slug>`. |
| `embeddings.embed` | Embed input texts using the configured provider; returns vectors, dimensions, count. |
| `embeddings.test_connection` | Run a single test embed against the configured or specified provider/model/dims. |
All handlers return `RpcOutcome<serde_json::Value>` via `into_cli_compatible_json`.
## Agent tools
None. This module owns no `tools.rs`; agent-facing memory tools live in `memory/tools/` and consume embeddings indirectly through the memory store.
## Events
None. No `bus.rs` / `EventHandler` impls. Cross-module side effects in `update_settings` are direct calls (`memory::read_rpc::wipe_all_rpc`, `memory_queue::ensure_reembed_backfill`), not domain events.
## Persistence
No `store.rs`. Settings live in the global `Config` (`config.memory.embedding_provider` / `embedding_model` / `embedding_dimensions` / `embedding_rate_limit_per_min`, plus `config.embeddings_provider`). API keys are persisted via the credentials domain (`AuthService`) under provider key `embeddings:<slug>`. Vectors themselves are stored by `memory_store::vectors`.
## Dependencies
- `crate::openhuman::config``Config`, `config::ops::load_config_with_timeout`, `build_runtime_proxy_client` (proxy-aware reqwest), config save.
- `crate::openhuman::credentials``AuthService`, `APP_SESSION_PROVIDER` for resolving the session JWT (cloud) and storing/loading provider API keys.
- `crate::openhuman::memory_store::vectors` — canonical vector store owner used by memory storage and retrieval.
- `crate::openhuman::memory::read_rpc``wipe_all_rpc` on a dimension change.
- `crate::openhuman::memory_queue``ensure_reembed_backfill` on a signature change.
- `crate::openhuman::inference::local``ollama_base_url()` when constructing the Ollama provider.
- `crate::api::config``effective_api_url` for the cloud backend base URL.
- `crate::core::all``ControllerFuture`, `RegisteredController` for the RPC registry.
- `crate::core::{ControllerSchema, FieldSchema, TypeSchema}` — schema definitions.
- `crate::core::observability``report_error_or_expected` so transient upstream HTTP failures demote to warning breadcrumbs instead of Sentry errors.
- `crate::rpc::RpcOutcome` — handler return contract.
## Used by
- `memory_store/*` — factories, vectors store, chunks store, unified store, retrieval, client (primary consumer; constructs providers and uses signatures to partition the embedding space).
- `memory/*` — ingestion queue, preferences, tools (recall/store/forget), `read_rpc`.
- `memory_tree/score/embed/cloud.rs` — scoring path.
- `codegraph/*` — index/search/tools obtain a provider via `provider_from_config` for `signature()` + direct embedding.
- `voice/factory.rs`, `screen_intelligence`, `channels`, `agent` tools — indirect consumers.
- `config/schema/load.rs` — wires `memory.embedding_rate_limit_per_min` into `rate_limit::set_embedding_rate_limit`.
- `core/all.rs` — registers the RPC controllers; `core/observability.rs` — classifies the canonical embedding error strings.
## Notes / gotchas
- **Signature drift is the core hazard.** `format_embedding_signature` is the single source of truth; both config-derived and live-provider signatures route through it. A mismatch silently splits one embedding space into two (#1574). `update_settings` keys memory wipe off **dimension** change (vectors stay comparable across provider/model swaps at the same dimensionality) but queues a re-embed backfill on **any** signature change.
- **Rate limiting is account-wide and process-global**, keyed by resolved base URL, with capacity = 1 token (no burst) to stay strictly under a hard per-minute cap. Ephemeral provider instances share one budget per endpoint. Loopback hosts are exempt; `limit==0` disables. The acquire chokepoint sits **inside** the retry loop so retried attempts consume tokens.
- **Cloud provider resolves auth lazily per call** — it can be constructed before login; the first `embed()` errors clearly if unauthenticated. It honors `OPENHUMAN_WORKSPACE` for the auth-profiles directory so non-default workspaces (tests/multi-instance) don't silently lose their session.
- **Ollama NaN recovery (TAURI-RUST-AZ):** a single bad input can poison a whole batch with a 500 `unsupported value: NaN`; the provider re-issues per-text and substitutes empty embeddings for the offending entries. Blank/whitespace inputs are preserved as zero-vectors so result length always matches input length. `local-*` model IDs are rejected (they're virtual routing aliases).
- **Custom endpoints** are encoded in the provider slug as `custom:<url>`; `resolve_api_key` and the RPC handlers normalize this back to the `custom` credential slug.
- **Voyage and cloud reuse `OpenAiEmbedding`** for HTTP plumbing (Voyage's API is an OpenAI superset). **Cohere** speaks a distinct wire format and has its own retry loop.
- Error strings from OpenAI/Cohere providers intentionally preserve the `(429 ` / status substring so `core::observability`'s `TransientUpstreamHttp` classifier downgrades them.
The memory tree uses the same factory through
`memory_tree/score/embed::ProviderEmbedder`; its fixed on-disk dimension remains
1024. Ollama requests use TinyAgents'
`RECOMMENDED_OLLAMA_CONTEXT_TOKENS` for both `num_ctx` and `num_batch`.
@@ -1,51 +0,0 @@
//! Compatibility wrapper for tinyagents' Cohere model.
use async_trait::async_trait;
use tinyagents::harness::embeddings::{CohereEmbeddingModel, EmbeddingModel};
use super::EmbeddingProvider;
pub struct CohereEmbedding {
inner: CohereEmbeddingModel,
}
impl CohereEmbedding {
pub fn new(api_key: &str, model: &str, dimensions: usize) -> Self {
Self {
inner: CohereEmbeddingModel::new(api_key)
.with_model(model)
.with_dimensions(dimensions),
}
}
pub fn with_base_url(mut self, base_url: impl Into<String>) -> Self {
self.inner = self.inner.with_base_url(base_url);
self
}
}
#[async_trait]
impl EmbeddingProvider for CohereEmbedding {
fn name(&self) -> &str {
self.inner.name()
}
fn model_id(&self) -> &str {
self.inner.model_id()
}
fn dimensions(&self) -> usize {
self.inner.dimensions()
}
fn signature(&self) -> String {
self.inner.signature()
}
async fn embed(&self, texts: &[&str]) -> anyhow::Result<Vec<Vec<f32>>> {
let texts = texts
.iter()
.map(|text| (*text).to_owned())
.collect::<Vec<_>>();
self.inner
.embed(&texts)
.await
.map_err(|error| anyhow::anyhow!(error))
}
}
+3 -11
View File
@@ -15,14 +15,8 @@
pub mod catalog;
#[path = "cloud_adapter.rs"]
pub mod cloud;
#[path = "cohere_adapter.rs"]
pub mod cohere;
mod factory;
pub mod noop;
#[path = "ollama_adapter.rs"]
pub mod ollama;
#[path = "openai_adapter.rs"]
pub mod openai;
mod provider_trait;
pub mod rate_limit {
pub use tinyagents::harness::embeddings::{
@@ -42,8 +36,6 @@ pub mod retry_after {
}
mod rpc;
mod schemas;
#[path = "voyage_adapter.rs"]
pub mod voyage;
pub use catalog::non_embedding_model_reason;
pub use cloud::{
@@ -60,8 +52,6 @@ pub(crate) use factory::model_supports_dimensions;
// #002 FR-015: the memory-tree OpenAI-compat embedder reuses the same key
// resolution the embeddings RPC uses, so there is one source of truth.
pub use noop::NoopEmbedding;
pub use ollama::OllamaEmbedding;
pub use openai::OpenAiEmbedding;
pub use provider_trait::{
format_embedding_signature, EmbeddingProvider, TinyAgentsEmbeddingProvider,
};
@@ -71,7 +61,9 @@ pub use schemas::{
all_controller_schemas as all_embeddings_controller_schemas,
all_registered_controllers as all_embeddings_registered_controllers,
};
pub use tinyagents::harness::embeddings::{DEFAULT_OLLAMA_DIMENSIONS, DEFAULT_OLLAMA_MODEL};
pub use tinyagents::harness::embeddings::{
DEFAULT_OLLAMA_DIMENSIONS, DEFAULT_OLLAMA_MODEL, DEFAULT_OLLAMA_URL,
};
#[cfg(test)]
#[path = "mod_tests.rs"]
+3 -5
View File
@@ -21,15 +21,13 @@ fn noop_name_and_dims() {
async fn noop_embed_returns_empty() {
let p = NoopEmbedding;
let result = p.embed(&["hello"]).await.unwrap();
assert!(result.is_empty());
assert_eq!(result, vec![Vec::<f32>::new()]);
}
#[tokio::test]
async fn noop_embed_one_returns_error() {
// embed returns empty vec → pop() returns None → error from default impl
async fn noop_embed_one_returns_empty_vector() {
let p = NoopEmbedding;
let err = p.embed_one("hello").await.unwrap_err();
assert!(err.to_string().contains("Empty embedding result"));
assert!(p.embed_one("hello").await.unwrap().is_empty());
}
#[tokio::test]
+10 -2
View File
@@ -1,6 +1,7 @@
//! No-op embedding provider for keyword-only search fallback.
use async_trait::async_trait;
use tinyagents::harness::embeddings::{EmbeddingModel, NoopEmbeddingModel};
use super::EmbeddingProvider;
@@ -22,7 +23,14 @@ impl EmbeddingProvider for NoopEmbedding {
0
}
async fn embed(&self, _texts: &[&str]) -> anyhow::Result<Vec<Vec<f32>>> {
Ok(Vec::new())
async fn embed(&self, texts: &[&str]) -> anyhow::Result<Vec<Vec<f32>>> {
let texts = texts
.iter()
.map(|text| (*text).to_owned())
.collect::<Vec<_>>();
NoopEmbeddingModel
.embed(&texts)
.await
.map_err(|error| anyhow::anyhow!(error))
}
}
@@ -1,61 +0,0 @@
//! Compatibility wrapper for tinyagents' Ollama model.
use async_trait::async_trait;
use tinyagents::harness::embeddings::{EmbeddingModel, OllamaEmbeddingModel};
use super::EmbeddingProvider;
pub use tinyagents::harness::embeddings::{
DEFAULT_OLLAMA_DIMENSIONS, DEFAULT_OLLAMA_MODEL, DEFAULT_OLLAMA_URL,
};
#[derive(Default)]
pub struct OllamaEmbedding {
inner: OllamaEmbeddingModel,
}
impl OllamaEmbedding {
pub fn try_new(base_url: &str, model: &str, dimensions: usize) -> anyhow::Result<Self> {
Ok(Self {
inner: OllamaEmbeddingModel::try_new(base_url, model, dimensions)?,
})
}
pub fn new(base_url: &str, model: &str, dimensions: usize) -> Self {
Self::try_new(base_url, model, dimensions).expect("invalid Ollama embedding configuration")
}
pub fn base_url(&self) -> &str {
self.inner.base_url()
}
pub fn model(&self) -> &str {
self.inner.model()
}
}
#[async_trait]
impl EmbeddingProvider for OllamaEmbedding {
fn name(&self) -> &str {
self.inner.name()
}
fn model_id(&self) -> &str {
self.inner.model_id()
}
fn dimensions(&self) -> usize {
self.inner.dimensions()
}
fn signature(&self) -> String {
self.inner.signature()
}
async fn embed(&self, texts: &[&str]) -> anyhow::Result<Vec<Vec<f32>>> {
let texts = texts
.iter()
.map(|text| (*text).to_owned())
.collect::<Vec<_>>();
self.inner
.embed(&texts)
.await
.map_err(|error| anyhow::anyhow!(error))
}
}
@@ -1,71 +0,0 @@
//! Compatibility wrapper for tinyagents' OpenAI-compatible model.
use async_trait::async_trait;
use tinyagents::harness::embeddings::{EmbeddingModel, OpenAiEmbeddingModel};
use super::EmbeddingProvider;
pub struct OpenAiEmbedding {
inner: OpenAiEmbeddingModel,
}
impl OpenAiEmbedding {
pub fn new(base_url: &str, api_key: &str, model: &str, dimensions: usize) -> Self {
Self {
inner: OpenAiEmbeddingModel::new(api_key)
.with_base_url(base_url)
.with_model(model)
.with_dimensions(dimensions)
.with_send_dimensions(false)
.with_required_api_key(false),
}
}
pub fn with_send_dimensions(mut self, send: bool) -> Self {
self.inner = self.inner.with_send_dimensions(send);
self
}
pub fn with_required_api_key(mut self, required: bool) -> Self {
self.inner = self.inner.with_required_api_key(required);
self
}
pub fn base_url(&self) -> &str {
self.inner.base_url()
}
pub fn model(&self) -> &str {
self.inner.model()
}
pub fn embeddings_url(&self) -> String {
self.inner.embeddings_url()
}
}
#[async_trait]
impl EmbeddingProvider for OpenAiEmbedding {
fn name(&self) -> &str {
self.inner.name()
}
fn model_id(&self) -> &str {
self.inner.model_id()
}
fn dimensions(&self) -> usize {
self.inner.dimensions()
}
fn signature(&self) -> String {
self.inner.signature()
}
async fn embed(&self, texts: &[&str]) -> anyhow::Result<Vec<Vec<f32>>> {
let texts = texts
.iter()
.map(|text| (*text).to_owned())
.collect::<Vec<_>>();
self.inner
.embed(&texts)
.await
.map_err(|error| anyhow::anyhow!(error))
}
}
@@ -1,69 +0,0 @@
//! Compatibility wrapper for tinyagents' Voyage model.
use async_trait::async_trait;
use tinyagents::harness::embeddings::{
EmbeddingModel, VoyageEmbeddingModel, VOYAGE_API_BASE, VOYAGE_DEFAULT_DIMENSIONS,
VOYAGE_DEFAULT_MODEL,
};
use super::EmbeddingProvider;
pub struct VoyageEmbedding {
inner: VoyageEmbeddingModel,
}
impl VoyageEmbedding {
pub fn new(api_key: &str, model: &str, dimensions: usize) -> Self {
Self::new_with_base_url(api_key, model, dimensions, VOYAGE_API_BASE)
}
pub fn new_with_base_url(
api_key: &str,
model: &str,
dimensions: usize,
base_url: &str,
) -> Self {
Self {
inner: VoyageEmbeddingModel::with_options(
api_key,
if model.is_empty() {
VOYAGE_DEFAULT_MODEL
} else {
model
},
if dimensions == 0 {
VOYAGE_DEFAULT_DIMENSIONS
} else {
dimensions
},
base_url,
),
}
}
}
#[async_trait]
impl EmbeddingProvider for VoyageEmbedding {
fn name(&self) -> &str {
self.inner.name()
}
fn model_id(&self) -> &str {
self.inner.model_id()
}
fn dimensions(&self) -> usize {
self.inner.dimensions()
}
fn signature(&self) -> String {
self.inner.signature()
}
async fn embed(&self, texts: &[&str]) -> anyhow::Result<Vec<Vec<f32>>> {
let texts = texts
.iter()
.map(|text| (*text).to_owned())
.collect::<Vec<_>>();
self.inner
.embed(&texts)
.await
.map_err(|error| anyhow::anyhow!(error))
}
}
@@ -2,7 +2,7 @@
//!
//! The memory tree's embedder (`bge-m3`) is requested with
//! `num_ctx = 8192` (see
//! [`crate::openhuman::memory_tree::score::embed::ollama::EMBED_NUM_CTX`])
//! [`tinyagents::harness::embeddings::RECOMMENDED_OLLAMA_CONTEXT_TOKENS`])
//! and the summariser hard-caps its output to fit that 8192-token embed
//! ceiling. A local model whose native context window is below this floor
//! silently truncates chunks/summaries and corrupts recall, so we refuse
@@ -16,12 +16,12 @@ use serde::Serialize;
/// Minimum native context window (tokens) a local model must advertise to
/// be accepted by the memory layer.
///
/// Re-exported from the embedder's own `EMBED_NUM_CTX` so this gate can
/// Re-exported from TinyAgents' canonical Ollama context setting so this gate can
/// never drift from what the memory pipeline actually requests at embed
/// time. Changing the embedder's context request automatically moves the
/// acceptance floor with it.
pub const MIN_CONTEXT_TOKENS: u64 =
crate::openhuman::memory_tree::score::embed::ollama::EMBED_NUM_CTX as u64;
tinyagents::harness::embeddings::RECOMMENDED_OLLAMA_CONTEXT_TOKENS as u64;
/// Verdict for a single model's context window against
/// [`MIN_CONTEXT_TOKENS`]. Serialized into the diagnostics payload so the
@@ -79,7 +79,7 @@ mod tests {
// requests; this guards against the two drifting apart.
assert_eq!(
MIN_CONTEXT_TOKENS,
crate::openhuman::memory_tree::score::embed::ollama::EMBED_NUM_CTX as u64
tinyagents::harness::embeddings::RECOMMENDED_OLLAMA_CONTEXT_TOKENS as u64
);
assert_eq!(MIN_CONTEXT_TOKENS, 8_192);
}
+1 -1
View File
@@ -111,7 +111,7 @@ These are subscriber registrations rather than a single `bus.rs`; subscriptions
- `src/core/all.rs` — registers the `learning.*` controllers + schemas.
- `src/openhuman/agent/harness/session/{builder,turn}.rs` and `agent_memory/memory_loader.rs` — wire the post-turn hooks, prompt sections, and learned-context loading into the agent loop.
- `src/openhuman/channels/runtime/startup.rs` — likely registers schedulers/subscribers at startup.
- `src/openhuman/memory_store/unified/profile.rs`, `memory_sync/composio/providers/profile.rs`, `memory_tools/{capture,mod}.rs`, `tools/impl/system/tool_stats.rs`, `tools/schemas.rs` — consume facet/learning types.
- `src/openhuman/memory_store/namespace_store/profile.rs`, `memory_sync/composio/providers/profile.rs`, `memory_tools/{capture,mod}.rs`, `tools/impl/system/tool_stats.rs`, `tools/schemas.rs` — consume facet/learning types.
## Notes / gotchas
+1 -1
View File
@@ -14,7 +14,7 @@ use crate::openhuman::memory_store::profile::{self, ProfileFacet, UserState};
/// Thin wrapper around the `user_profile` table.
///
/// All methods delegate to the standalone helpers in
/// `memory::store::unified::profile`. This type exists so callers
/// `memory_store::namespace_store::profile`. This type exists so callers
/// (stability detector, prompt sections, RPCs) share a single typed
/// entry-point that can be constructed from any `Arc<Mutex<Connection>>`.
pub struct FacetCache {
+1 -1
View File
@@ -26,7 +26,7 @@ one job. memory orchestrates and routes between them.
| ------------------------------------------ | --------------------------------------------------------------------------------------------------- |
| [`memory_store`](../memory_store/) | Storage primitives: raw / chunks / entities / trees / vectors / kv / contacts. SQLite + on-disk md. |
| [`memory_tree`](../memory_tree/) | Generic tree mechanics: bucket-seal, flush, summarise, and retrieval/traversal backends. |
| [`memory_archivist`](../memory_archivist/) | Chat conversation → clip tool-calls → push to tree (shim over `tinycortex::memory::archivist`). |
| `tinycortex::memory::archivist` | Chat conversation → clip tool-calls → push to tree; called directly by the host harness. |
| [`memory_tools`](../memory_tools/) | Tool-scoped rules + agent read/write tools. |
| [`memory_sync`](../memory_sync/) | Composio + workspace + MCP sync pipelines. |
-1
View File
@@ -57,6 +57,5 @@ pub use traits::{Memory, MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSumm
// Re-export types that external tests and consumers historically imported
// from `memory::*`. The definitions moved to sibling crates during the
// memory refactor; these aliases keep the public surface stable.
pub use crate::openhuman::memory_queue as jobs;
pub use crate::openhuman::memory_store::types::NamespaceDocumentInput;
pub use crate::openhuman::memory_store::{MemoryClient, UnifiedMemory};
+1 -1
View File
@@ -4,7 +4,7 @@ use crate::openhuman::embeddings::NoopEmbedding;
use crate::openhuman::memory::ingest_pipeline::ingest_chat;
use crate::openhuman::memory_queue::drain_until_idle;
use crate::openhuman::memory_store::content::raw::{write_raw_items, RawItem, RawKind};
use crate::openhuman::memory_store::unified::UnifiedMemory;
use crate::openhuman::memory_store::namespace_store::UnifiedMemory;
use crate::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage};
use chrono::{TimeZone, Utc};
use rusqlite::params;
-44
View File
@@ -1,44 +0,0 @@
//! Memory archivist — thin host shim over `tinycortex::memory::archivist` (W7).
//!
//! The archivist engine (clip / compose / tree-writer / episodic store + its
//! types) lives in the crate now. This module keeps the host's
//! `memory_archivist::…` import paths stable and adapts the host [`Config`] to
//! the crate's `MemoryConfig` for the two episodic-store functions host callers
//! use (`store::record_turn` / `store::session_entries`, from the agent
//! archivist hook + recap).
//!
//! On-disk layout is unchanged — the crate writes to the same path the host
//! did: `<workspace>/memory_tree/content/episodic/<sanitized-session>/<seq>.md`.
//! `Config::workspace_dir` maps to the crate `MemoryConfig.workspace`, so
//! `memory_tree_content_root()` (`<workspace>/memory_tree/content`) resolves
//! identically on both sides.
pub use tinycortex::memory::archivist::types::{ArchivedTurn, Turn};
/// Episodic conversation archive — thin adapters over the crate store that
/// convert the host [`Config`](crate::openhuman::config::Config) into the
/// crate's `MemoryConfig`.
pub mod store {
use anyhow::Result;
use super::ArchivedTurn;
use crate::openhuman::config::Config;
use crate::openhuman::tinycortex::memory_config_from;
fn engine_config(config: &Config) -> tinycortex::memory::MemoryConfig {
memory_config_from(config, config.workspace_dir.clone())
}
/// Append one turn to its session's episodic archive, assigning the next
/// per-session sequence number.
pub fn record_turn(config: &Config, turn: ArchivedTurn) -> Result<ArchivedTurn> {
tinycortex::memory::archivist::store::record_turn(&engine_config(config), turn)
}
/// All turns recorded for a session, ordered by sequence.
pub fn session_entries(config: &Config, session_id: &str) -> Result<Vec<ArchivedTurn>> {
tinycortex::memory::archivist::store::session_entries(&engine_config(config), session_id)
}
}
pub use store::{record_turn, session_entries};
-1
View File
@@ -28,7 +28,6 @@
//! This queue used to live under `openhuman::memory::jobs`; it now has a
//! dedicated top-level home (`openhuman::memory_queue`) because it is an
//! execution/runtime concern rather than a leaf of the memory policy API.
//! `openhuman::memory` re-exports it as `memory::jobs` during the migration.
mod ops;
pub mod scheduler;
-2
View File
@@ -5,9 +5,7 @@
//! `memory_tree`) provide persistence and tree traversal; this module composes
//! them into tools the agent can invoke.
pub mod scoring;
pub mod tools;
pub mod vector;
// ── Public re-exports ───────────────────────────────────────────────────────
-40
View File
@@ -1,40 +0,0 @@
//! Scoring weight profiles for hybrid retrieval — thin host shim over
//! `tinycortex::memory::WeightProfile` (W5).
//!
//! The weight profile (graph/vector/keyword/freshness weights + the
//! `BALANCED`/`SEMANTIC`/`LEXICAL`/`GRAPH_FIRST` presets + `by_name`) is the
//! crate's, a byte-identical port. The host keeps only [`compose_score`] — the
//! trivial weighted combination the crate expresses via
//! `retrieval::scoring::hybrid_score` at its own call sites; exposed here as a
//! free function so `memory_search::tools::hybrid_search` keeps its call shape.
pub use tinycortex::memory::WeightProfile;
/// Weighted composite of the four retrieval signals under `profile`.
///
/// `graph·graph_relevance + vector·vector_similarity + keyword·keyword_relevance
/// + freshness·freshness`.
pub fn compose_score(
profile: &WeightProfile,
graph_relevance: f64,
vector_similarity: f64,
keyword_relevance: f64,
freshness: f64,
) -> f64 {
(profile.graph * graph_relevance)
+ (profile.vector * vector_similarity)
+ (profile.keyword * keyword_relevance)
+ (profile.freshness * freshness)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn compose_score_is_weighted_sum() {
let p = WeightProfile::BALANCED;
let s = compose_score(&p, 1.0, 1.0, 1.0, 1.0);
assert!((s - (p.graph + p.vector + p.keyword + p.freshness)).abs() < 1e-9);
}
}
@@ -12,10 +12,10 @@ use std::sync::Arc;
use crate::openhuman::config::rpc as config_rpc;
use crate::openhuman::embeddings::{provider_from_config, EmbeddingProvider};
use crate::openhuman::memory_search::scoring::WeightProfile;
use crate::openhuman::memory_store::types::MemoryItemKind;
use crate::openhuman::memory_store::UnifiedMemory;
use crate::openhuman::tools::traits::{Tool, ToolResult};
use tinycortex::memory::WeightProfile;
pub struct MemoryHybridSearchTool;
@@ -178,13 +178,14 @@ impl Tool for MemoryHybridSearchTool {
.enumerate()
.map(|(i, hit)| {
let bd = &hit.score_breakdown;
let score = crate::openhuman::memory_search::scoring::compose_score(
let score = tinycortex::memory::retrieval::scoring::hybrid_score(
&profile,
bd.graph_relevance,
bd.vector_similarity,
bd.keyword_relevance,
bd.freshness,
);
)
.final_score;
(i, score)
})
.filter(|(_, score)| *score > 0.0)
@@ -11,13 +11,13 @@ use std::fmt::Write;
use crate::openhuman::config::rpc as config_rpc;
use crate::openhuman::embeddings::provider_from_config;
use crate::openhuman::memory_search::vector::mmr::{mmr_select, MmrCandidate};
use crate::openhuman::memory_store::chunks::store::{
get_chunk_embeddings_for_signature_batch, list_chunks, ListChunksQuery,
};
use crate::openhuman::memory_store::chunks::types::SourceKind;
use crate::openhuman::memory_store::vectors::cosine_similarity;
use crate::openhuman::tools::traits::{Tool, ToolResult};
use tinycortex::memory::retrieval::mmr::{mmr_select, MmrCandidate};
pub struct MemoryVectorSearchTool;
@@ -1,9 +0,0 @@
//! Maximal Marginal Relevance selection — thin host re-export of
//! `tinycortex::memory::retrieval::mmr` (W5).
//!
//! The MMR algorithm (relevancediversity tradeoff over embeddings) is the
//! crate's, a byte-identical port. Host consumers (`memory_search::tools`) keep
//! their `memory_search::vector::mmr::{MmrCandidate, MmrResult, mmr_select}`
//! import paths unchanged.
pub use tinycortex::memory::retrieval::mmr::{mmr_select, MmrCandidate, MmrResult};
@@ -1,3 +0,0 @@
//! Vector diversity algorithms.
pub mod mmr;
+3 -4
View File
@@ -12,9 +12,8 @@ trees/ summary tree persistence (one table, kind-parameterized)
vectors/ local vector DB (cosine, brute-force)
kv/ global + namespace key-value (kv_global, kv_namespace)
contacts/ [removed] facade over people::store (Person/Handle/Interaction)
unified/ [staging for removal] UnifiedMemory's remaining SQLite surface
(documents/query/segments/events/profile) — replaced as the
Memory trait callers migrate to per-kind backends
namespace_store/ host-retained namespace documents, graph, episodic/event/
segment/profile tables, and retrieval policy
```
## Cross-cutting modules
@@ -43,7 +42,7 @@ unified/ [staging for removal] UnifiedMemory's remaining SQLite surface
| [`vectors/`](vectors/) | Standalone vector store. `VectorStore` over SQLite, byte-codec for f32 vectors, cosine similarity. |
| [`kv.rs`](kv.rs) | Global + namespace key-value (`kv_global`, `kv_namespace` tables). |
| `contacts/` | Removed. Contact access now lives outside `memory_store` via `people::store`. |
| [`unified/`](unified/) | **Staging for removal.** Shrinking SQLite surface that still backs the `Memory` trait while callers migrate to per-kind modules. Active pieces today: `documents`, `query`, `segments`, `events`, `profile`. The `fts5` episodic surface is replaced by [`memory_archivist`](../memory_archivist/) and `graph` by the crate's `tinycortex::memory::graph` (the host `memory_graph` placeholder was deleted in the W7 migration). See [`unified/README.md`](unified/README.md). |
| [`namespace_store/`](namespace_store/) | Host-retained namespace/document tier over the shared SQLite database: documents, persisted product graph relations, episodic/events, segments, profile facets, and host retrieval policy. TinyCortex owns the generic chunk/vector/tree/queue substrate; this tier remains the stable `Memory` implementation. See [`namespace_store/README.md`](namespace_store/README.md). |
## Layer rules
+2 -2
View File
@@ -17,10 +17,10 @@ use crate::openhuman::memory::ingestion::{
IngestionJob, IngestionQueue, IngestionState, MemoryIngestionConfig, MemoryIngestionRequest,
MemoryIngestionResult,
};
use crate::openhuman::memory_store::namespace_store::UnifiedMemory;
use crate::openhuman::memory_store::types::{
NamespaceDocumentInput, NamespaceMemoryHit, NamespaceRetrievalContext,
};
use crate::openhuman::memory_store::unified::UnifiedMemory;
/// Reference-counted handle to a `MemoryClient`.
pub type MemoryClientRef = Arc<MemoryClient>;
@@ -56,7 +56,7 @@ pub struct MemoryClient {
impl MemoryClient {
/// Returns a handle to the underlying SQLite connection for direct
/// profile-facet writes via
/// [`crate::openhuman::memory_store::unified::profile::profile_upsert`].
/// [`crate::openhuman::memory_store::namespace_store::profile::profile_upsert`].
///
/// Intentionally `pub(crate)` — external consumers should use the
/// higher-level `MemoryClient` API; this escape hatch exists so
+1 -1
View File
@@ -21,7 +21,7 @@ use crate::openhuman::embeddings::{
DEFAULT_CLOUD_EMBEDDING_MODEL, DEFAULT_OLLAMA_DIMENSIONS, DEFAULT_OLLAMA_MODEL,
};
use crate::openhuman::memory::traits::Memory;
use crate::openhuman::memory_store::unified::UnifiedMemory;
use crate::openhuman::memory_store::namespace_store::UnifiedMemory;
/// One-shot guard so the Ollama health-gate fallback only reports to Sentry
/// once per process lifetime. Memory is constructed many times per session
+1 -1
View File
@@ -2,8 +2,8 @@
use tinycortex::memory::store::kv::KvStore;
use crate::openhuman::memory_store::namespace_store::UnifiedMemory;
use crate::openhuman::memory_store::types::MemoryKvRecord;
use crate::openhuman::memory_store::unified::UnifiedMemory;
impl UnifiedMemory {
fn tinycortex_kv(&self) -> Result<KvStore, String> {
+2 -2
View File
@@ -17,11 +17,11 @@ use serde_json::json;
use crate::openhuman::memory::traits::{
Memory, MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary, RecallOpts,
};
use crate::openhuman::memory_store::namespace_store::fts5;
use crate::openhuman::memory_store::types::{NamespaceDocumentInput, GLOBAL_NAMESPACE};
use crate::openhuman::memory_store::unified::fts5;
use anyhow::Context;
use super::unified::UnifiedMemory;
use super::namespace_store::UnifiedMemory;
/// Convert a UNIX timestamp (f64) to RFC3339 string.
fn timestamp_to_rfc3339(ts: f64) -> String {
+8 -7
View File
@@ -11,7 +11,8 @@
//! ## Submodules
//!
//! - `types`: Common data structures and types used across the memory store.
//! - `unified`: The primary SQLite-based memory implementation.
//! - `namespace_store`: Host-retained SQLite namespace documents, graph,
//! episodic/event/segment/profile tables, and their query policy.
//! - `client`: High-level client interface for interacting with the memory system.
//! - `factories`: Factory functions for creating and initializing memory instances.
//! - `memory_trait`: Defines the `Memory` trait that all implementations must satisfy.
@@ -21,13 +22,13 @@ pub mod content;
pub mod entities;
pub mod kinds;
pub mod kv;
pub mod namespace_store;
pub mod retrieval;
pub mod safety;
pub mod tools;
pub mod traits;
pub mod trees;
pub mod types;
pub mod unified;
pub mod vectors;
mod client;
@@ -42,16 +43,16 @@ pub use factories::{
active_embedding_signature, create_memory, create_memory_for_migration,
create_memory_with_local_ai, effective_embedding_settings, effective_memory_backend_name,
};
pub use namespace_store::events;
pub use namespace_store::fts5;
pub use namespace_store::profile;
pub use namespace_store::segments;
pub use namespace_store::UnifiedMemory;
pub use types::{
GraphRelationRecord, MemoryItemKind, MemoryKvRecord, NamespaceDocumentInput,
NamespaceMemoryHit, NamespaceQueryResult, NamespaceRetrievalContext, RetrievalScoreBreakdown,
StoredMemoryDocument,
};
pub use unified::events;
pub use unified::fts5;
pub use unified::profile;
pub use unified::segments;
pub use unified::UnifiedMemory;
#[cfg(test)]
mod tests {
@@ -1,6 +1,11 @@
# Unified memory store
# Namespace memory store
SQLite-backed implementation of the memory store. One `UnifiedMemory` struct owns a WAL-mode connection plus the on-disk markdown sidecar tree and vector storage path; the rest of this directory adds capabilities to it via per-domain `impl` blocks.
Host-retained SQLite namespace/document tier. One `UnifiedMemory` struct owns
the shared connection plus the on-disk markdown sidecar and compatibility
embedding handle; the rest of this directory adds the product-owned document,
graph, episodic, event, segment, profile, and retrieval policy via `impl`
blocks. TinyCortex owns the generic chunk/vector/tree/queue engine beside this
tier; this directory is intentionally not migration staging.
## Files
@@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS conversation_segments (
status TEXT NOT NULL DEFAULT 'open',
created_at REAL NOT NULL,
updated_at REAL NOT NULL,
-- Per-session sequence numbers from memory_archivist::store, populated
-- Per-session sequence numbers from tinycortex::memory::archivist::store, populated
-- alongside start_episodic_id / end_episodic_id during the FTS5 -> md
-- migration. Once STM recall switches its segment-span dedup to use
-- (session_id, seq) the legacy episodic_id columns can be dropped.
@@ -108,7 +108,7 @@ pub struct ConversationSegment {
pub status: SegmentStatus,
pub created_at: f64,
pub updated_at: f64,
/// Per-session seq number assigned by `memory_archivist::store::record_turn`
/// Per-session seq number assigned by the TinyCortex archivist store.
/// for the user turn that opened this segment. `None` on legacy rows
/// written before the FTS5 -> md migration began.
#[serde(default)]
+21 -75
View File
@@ -1,82 +1,28 @@
# memory_sync
Every "pull data from upstream → land it in memory_store" pipeline in
one place, organised by the kind of upstream they talk to.
OpenHuman orchestration and product policy around the TinyCortex sync engine.
## Three pipeline kinds
TinyCortex owns generic Composio provider fetch/pagination, canonical memory
records, sync budgets/state, workspace reconciliation, and persistence traits.
The live host path calls it through `src/openhuman/tinycortex/sync.rs` from
`composio::run_connection_sync` and the default provider `sync()` method.
| Kind | Submodule | Owns |
| --- | --- | --- |
| **Composio** | [`composio/`](composio/) | Per-provider sync via the Composio Edge API: gmail, slack, github, notion, linear, clickup, … |
| **Workspace** | [`workspace/`](workspace/) | Vault file watch, harness turn capture, dictation transcripts — anything local. |
| **MCP** | [`mcp/`](mcp/) | Third-party MCP servers via `mcp_clients/` transport. |
OpenHuman retains:
## Trait
- periodic scheduling and connection selection;
- credentials and Composio action execution;
- source-scope and redaction policy;
- translation into host `DomainEvent`s;
- JSON-RPC/status/connect surfaces;
- agent-facing action tools and result post-processing;
- product task/profile projections for GitHub, Notion, Linear, and ClickUp;
- local workspace watching and MCP orchestration.
Every pipeline implements [`SyncPipeline`]:
The provider directories therefore are not alternate sync engines. Their
remaining `provider.rs`, `tools.rs`, `normalization.rs`, profile, catalog, and
post-processing files implement host product surfaces over the crate-backed
sync path. New generic parsing or persistence behavior belongs in
`vendor/tinycortex/src/memory/sync/`.
```rust
async fn init(&self, &Config) -> anyhow::Result<()>;
async fn tick(&self, &Config) -> anyhow::Result<SyncOutcome>;
fn id(&self) -> &str;
fn kind(&self) -> SyncPipelineKind;
```
`SyncOutcome { records_ingested, more_pending, note }` is the
orchestrator-facing result; pipelines own their own pagination cursors
and retry policy behind that.
## Layout
| Path | Role |
| --- | --- |
| [`mod.rs`](mod.rs) | Module root + re-exports. |
| [`traits.rs`](traits.rs) | `SyncPipeline`, `SyncOutcome`, `SyncPipelineKind`. |
| [`composio/`](composio/) | Per-provider pipelines (gmail, slack, github, notion, linear, clickup). |
| [`workspace/`](workspace/) | Vault, harness, dictation pipelines. |
| [`mcp/`](mcp/) | MCP-server pipelines (one per connected server). |
## Schedulers & self-healing
Three background loops keep memory synced and compressed without manual
"Sync now" clicks:
| Loop | Module | Cadence | Covers |
| --- | --- | --- | --- |
| Composio periodic | [`composio/periodic.rs`](composio/periodic.rs) | 20-min tick, per-connection interval | Composio connections (gmail, slack, …) |
| Workspace periodic | [`workspace/periodic.rs`](workspace/periodic.rs) | 20-min tick, `memory_sync_interval_secs` (24h default, `0` = manual only) | `github_repo`, `folder`, `rss_feed`, `web_page` sources |
| Queue scheduler | `memory_queue::scheduler` | 3 h | flush stale L0 buffers + requeue transient-failed jobs |
**Raw-archive coverage** ([`sources/rebuild.rs`](sources/rebuild.rs)): every
summary batch records its raw files in `mem_tree_ingested_sources`
(`source_kind = "raw_file"`). After each sync, `check_and_rebuild_tree`
diffs disk vs gate and summarises only the uncovered remainder — so an
interrupted sync self-heals on the next pass instead of stranding raw
files forever. Inspect/trigger over RPC with
`openhuman.memory_sources_reconcile`.
⚠️ A source's **tree scope** and **raw-archive id** slugify to different
directories for GitHub (`github:owner/repo` vs `github.com/owner/repo`) —
always carry both (`memory_sources::sync::SourceScope`).
## Status
**Pipelines: scaffold only.** Today's sync code still lives in:
- `composio/providers/<provider>/ingest.rs` + `bin/{slack_backfill,gmail_backfill_3d}.rs`
- `vault/sync.rs`, `agent_experience/`, `dictation_hotkeys/`
- `mcp_clients/` (transport only; no drain loop yet)
Each migrates here as its own per-pipeline PR. The job-queue orchestration
in `memory::jobs` stays put — it just gains the ability to iterate over a
registered `Vec<Box<dyn SyncPipeline>>`.
## Layer rules
- Sync writes go through `memory::ingest_pipeline` so every record
lands as raw md → chunks → tree leaves like any other ingest.
- No direct writes into trees or unified. No upstream-specific data
models leak past the pipeline boundary.
- One pipeline per upstream service. Composio's GitHub and MCP's GitHub
are distinct pipelines because they hit different surfaces with
different cadence and auth.
D4.1-D4.4 are closed in `docs/tinycortex-drift-ledger.md`. Gmail's bounded
25-message page is crate-owned and prevents Composio 413 responses.
@@ -6,15 +6,15 @@
//! re-learning a new shape:
//!
//! - `provider.rs` — `impl ComposioProvider for ClickUpProvider`
//! - `sync.rs` — payload-shape helpers (results extraction, title)
//! - `normalization.rs` — payload-shape helpers (results extraction, title)
//! - `ingest.rs` — memory_tree document ingest (issue #2885)
//! - `tools.rs` — `CLICKUP_CURATED` whitelist of Composio actions
//! - `tests.rs` — unit tests for the helpers + trait metadata
//!
//! Issue: #2288 (introduction); #2885 (memory_tree migration).
mod normalization;
mod provider;
mod sync;
#[cfg(test)]
mod tests;
pub mod tools;
@@ -1,4 +1,4 @@
//! ClickUp sync helpers — result extraction, task-title extraction,
//! ClickUp host normalization helpers — result extraction, task-title extraction,
//! and time utilities.
//!
//! ClickUp's REST API (and therefore Composio's wrapping of it) returns
@@ -27,7 +27,7 @@
use async_trait::async_trait;
use serde_json::json;
use super::sync;
use super::normalization;
use crate::openhuman::memory_sync::composio::providers::{
first_array_str, merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider,
CuratedTool, NormalizedTask, ProviderContext, ProviderUserProfile, TaskFetchFilter, TaskKind,
@@ -105,7 +105,7 @@ impl ComposioProvider for ClickUpProvider {
let data = &resp.data;
let display_name = pick_str(data, &["user.username", "data.user.username", "username"]);
let email = pick_str(data, &["user.email", "data.user.email", "email"]);
let username = sync::extract_user_id(data);
let username = normalization::extract_user_id(data);
let avatar_url = pick_str(
data,
&[
@@ -168,7 +168,7 @@ impl ComposioProvider for ClickUpProvider {
resp.error.unwrap_or_else(|| "provider failure".into())
));
}
sync::extract_workspace_ids(&resp.data)
normalization::extract_workspace_ids(&resp.data)
}
};
@@ -188,7 +188,7 @@ impl ComposioProvider for ClickUpProvider {
resp.error.unwrap_or_else(|| "provider failure".into())
));
}
let id = sync::extract_user_id(&resp.data).ok_or_else(|| {
let id = normalization::extract_user_id(&resp.data).ok_or_else(|| {
"[composio:clickup] CLICKUP_GET_AUTHORIZED_USER returned no user.id".to_string()
})?;
vec![id]
@@ -227,7 +227,7 @@ impl ComposioProvider for ClickUpProvider {
));
}
for task in sync::extract_tasks(&resp.data) {
for task in normalization::extract_tasks(&resp.data) {
if out.len() >= max {
break 'workspaces;
}
@@ -246,8 +246,8 @@ impl ComposioProvider for ClickUpProvider {
/// `None` only when the task has no extractable id (unroutable).
fn normalize_clickup_task(task: &serde_json::Value) -> Option<NormalizedTask> {
let external_id = pick_str(task, TASK_ID_PATHS)?;
let title =
sync::extract_task_name(task).unwrap_or_else(|| format!("ClickUp task {external_id}"));
let title = normalization::extract_task_name(task)
.unwrap_or_else(|| format!("ClickUp task {external_id}"));
Some(NormalizedTask {
external_id,
source_id: String::new(),
@@ -265,7 +265,7 @@ fn normalize_clickup_task(task: &serde_json::Value) -> Option<NormalizedTask> {
due: pick_str(task, &["due_date", "data.due_date"]),
labels: Vec::new(),
priority: pick_str(task, &["priority.priority", "data.priority.priority"]),
updated_at: sync::extract_task_updated(task),
updated_at: normalization::extract_task_updated(task),
raw: task.clone(),
})
}
@@ -1,6 +1,6 @@
//! Unit tests for the ClickUp provider.
use super::sync::{
use super::normalization::{
extract_task_name, extract_task_updated, extract_tasks, extract_user_id, extract_workspace_ids,
};
use super::ClickUpProvider;
@@ -6,14 +6,14 @@
//! re-learning a new shape:
//!
//! - `provider.rs` — `impl ComposioProvider for GitHubProvider`
//! - `sync.rs` — payload-shape helpers (result extraction, title, cursor)
//! - `normalization.rs` — payload-shape helpers (result extraction, title, cursor)
//! - `tools.rs` — `GITHUB_CURATED` whitelist of Composio actions
//! - `tests.rs` — unit tests for the helpers + trait metadata
//!
//! Issue: #2408.
mod normalization;
mod provider;
mod sync;
#[cfg(test)]
mod tests;
pub mod tools;
@@ -1,4 +1,4 @@
//! GitHub sync helpers — result extraction, identity helpers, and time utilities.
//! GitHub host normalization helpers — result extraction, identity helpers, and time utilities.
//!
//! GitHub's REST API (proxied through Composio) returns search results and
//! authenticated-user payloads in a small number of shapes. The functions here
@@ -23,7 +23,7 @@ use async_trait::async_trait;
use serde_json::{json, Value};
use std::time::Duration;
use super::sync;
use super::normalization;
use crate::openhuman::memory_sync::composio::providers::{
merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider, CuratedTool,
GithubFetchMode, NormalizedTask, ProviderContext, ProviderUserProfile, TaskFetchFilter,
@@ -94,7 +94,7 @@ impl ComposioProvider for GitHubProvider {
}
let data = &resp.data;
let login = sync::extract_user_login(data);
let login = normalization::extract_user_login(data);
let display_name = pick_str(data, &["name", "data.name"]).or_else(|| login.clone());
let email = pick_str(data, &["email", "data.email"]);
let avatar_url = pick_str(data, &["avatar_url", "data.avatar_url"]);
@@ -157,7 +157,7 @@ impl ComposioProvider for GitHubProvider {
};
let mut out: Vec<NormalizedTask> = Vec::new();
for issue in sync::extract_issues(&data) {
for issue in normalization::extract_issues(&data) {
if out.len() >= max {
break;
}
@@ -481,7 +481,7 @@ pub(super) fn normalize_github_repo_filter(raw: &str) -> String {
/// not depend on the fetch query), so even if a `closed` item slips through
/// the query bias it is dropped here.
pub(super) fn normalize_github_issue(issue: &serde_json::Value) -> Option<NormalizedTask> {
let external_id = sync::extract_issue_id(issue)?;
let external_id = normalization::extract_issue_id(issue)?;
let status = pick_str(issue, &["state", "data.state"]);
if status
.as_deref()
@@ -494,8 +494,8 @@ pub(super) fn normalize_github_issue(issue: &serde_json::Value) -> Option<Normal
);
return None;
}
let title =
sync::extract_issue_title(issue).unwrap_or_else(|| format!("GitHub issue {external_id}"));
let title = normalization::extract_issue_title(issue)
.unwrap_or_else(|| format!("GitHub issue {external_id}"));
let kind = if is_pull_request(issue) {
TaskKind::PullRequest
} else {
@@ -514,7 +514,7 @@ pub(super) fn normalize_github_issue(issue: &serde_json::Value) -> Option<Normal
due: None,
labels: extract_github_labels(issue),
priority: None,
updated_at: sync::extract_issue_updated_at(issue),
updated_at: normalization::extract_issue_updated_at(issue),
raw: issue.clone(),
})
}
@@ -1,14 +1,14 @@
//! Unit tests for the GitHub Composio provider.
use super::normalization::{
extract_issue_id, extract_issue_title, extract_issue_updated_at, extract_issues,
extract_user_login,
};
use super::provider::github_env_token;
use super::provider::{
build_fetch_query, build_search_query, github_search_arg_pairs, normalize_github_issue,
normalize_github_repo_filter, ACTION_GET_AUTHENTICATED_USER, ACTION_SEARCH_ISSUES,
};
use super::sync::{
extract_issue_id, extract_issue_title, extract_issue_updated_at, extract_issues,
extract_user_login,
};
use super::tools::GITHUB_CURATED;
use super::GitHubProvider;
use crate::openhuman::memory_sync::composio::providers::ComposioProvider;
@@ -1,6 +1,5 @@
mod post_process;
mod provider;
mod sync;
#[cfg(test)]
mod tests;
pub mod tools;
@@ -1,242 +0,0 @@
//! Gmail sync helpers — message extraction, pagination, cursor
//! conversion, and time utilities.
use serde_json::Value;
/// Walk the Composio response envelope and pull out message objects.
pub(crate) fn extract_messages(data: &Value) -> Vec<Value> {
let candidates = [
data.pointer("/data/messages"),
data.pointer("/messages"),
data.pointer("/data/data/messages"),
data.pointer("/data/items"),
data.pointer("/items"),
];
for cand in candidates.into_iter().flatten() {
if let Some(arr) = cand.as_array() {
return arr.clone();
}
}
Vec::new()
}
/// Try to extract a pagination token from the API response.
pub(crate) fn extract_page_token(data: &Value) -> Option<String> {
let candidates = [
data.pointer("/data/nextPageToken"),
data.pointer("/nextPageToken"),
data.pointer("/data/data/nextPageToken"),
];
for cand in candidates.into_iter().flatten() {
if let Some(s) = cand.as_str() {
let trimmed = s.trim();
if !trimmed.is_empty() {
return Some(trimmed.to_string());
}
}
}
None
}
/// Convert a cursor value (epoch millis or date string) into a Gmail
/// `after:YYYY/MM/DD` filter component. Day-level precision — kept as a
/// last-resort filter for non-numeric cursors and for back-compat with
/// the older day-cursor write path. New code should prefer
/// [`cursor_to_gmail_after_epoch_filter`] which produces a
/// second-precision `after:<unix>` filter and so avoids re-fetching
/// large same-day windows on every tick.
pub(crate) fn cursor_to_gmail_after_filter(cursor: &str) -> Option<String> {
let cursor = cursor.trim();
// Try parsing as epoch millis first (Gmail's internalDate).
if let Ok(millis) = cursor.parse::<i64>() {
let secs = millis / 1000;
if let Some(dt) = chrono::DateTime::from_timestamp(secs, 0) {
return Some(dt.format("%Y/%m/%d").to_string());
}
}
// Try parsing as an ISO date/datetime.
if let Ok(dt) = chrono::NaiveDate::parse_from_str(cursor, "%Y-%m-%d") {
return Some(dt.format("%Y/%m/%d").to_string());
}
if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(cursor) {
return Some(dt.format("%Y/%m/%d").to_string());
}
None
}
/// Convert a cursor value (epoch millis or date string) into a Gmail
/// `after:<unix_seconds>` filter component. Gmail's search syntax
/// accepts a bare unix-seconds value for `after:` / `before:`, so the
/// filter is second-precision rather than the day-level
/// `after:YYYY/MM/DD` form. Used by the incremental sync path so a
/// same-day re-tick does not re-fetch every message Gmail has filed
/// today.
///
/// Returns `None` when the cursor cannot be parsed; callers should
/// fall back to the coarse day filter to avoid sending an unbounded
/// query.
pub(crate) fn cursor_to_gmail_after_epoch_filter(cursor: &str) -> Option<String> {
let secs = parse_cursor_to_epoch_secs(cursor)?;
Some(secs.to_string())
}
/// Parse a cursor (epoch millis as string, `YYYY-MM-DD`, or RFC3339)
/// into unix-seconds. Shared by the epoch filter and by the adaptive
/// page-cap recency check.
pub(crate) fn parse_cursor_to_epoch_secs(cursor: &str) -> Option<i64> {
let cursor = cursor.trim();
if let Ok(millis) = cursor.parse::<i64>() {
return Some(millis / 1000);
}
if let Ok(date) = chrono::NaiveDate::parse_from_str(cursor, "%Y-%m-%d") {
return date.and_hms_opt(0, 0, 0).map(|dt| dt.and_utc().timestamp());
}
if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(cursor) {
return Some(dt.timestamp());
}
None
}
pub(crate) fn now_ms() -> u64 {
use std::time::{SystemTime, UNIX_EPOCH};
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_millis() as u64)
.unwrap_or(0)
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn extract_messages_from_data_messages() {
let data = json!({"data": {"messages": [{"id": "1"}, {"id": "2"}]}});
let msgs = extract_messages(&data);
assert_eq!(msgs.len(), 2);
}
#[test]
fn extract_messages_from_top_level() {
let data = json!({"messages": [{"id": "1"}]});
let msgs = extract_messages(&data);
assert_eq!(msgs.len(), 1);
}
#[test]
fn extract_messages_from_data_items() {
let data = json!({"data": {"items": [{"id": "a"}]}});
let msgs = extract_messages(&data);
assert_eq!(msgs.len(), 1);
}
#[test]
fn extract_messages_empty_when_no_match() {
let data = json!({"foo": "bar"});
assert!(extract_messages(&data).is_empty());
}
#[test]
fn extract_page_token_from_data() {
let data = json!({"data": {"nextPageToken": "abc123"}});
assert_eq!(extract_page_token(&data), Some("abc123".into()));
}
#[test]
fn extract_page_token_from_top_level() {
let data = json!({"nextPageToken": "tok"});
assert_eq!(extract_page_token(&data), Some("tok".into()));
}
#[test]
fn extract_page_token_none_when_empty() {
let data = json!({"data": {"nextPageToken": " "}});
assert_eq!(extract_page_token(&data), None);
}
#[test]
fn extract_page_token_none_when_missing() {
let data = json!({"data": {}});
assert_eq!(extract_page_token(&data), None);
}
#[test]
fn cursor_to_filter_epoch_millis() {
let filter = cursor_to_gmail_after_filter("1700000000000").unwrap();
assert!(filter.contains('/'));
assert_eq!(filter, "2023/11/14");
}
#[test]
fn cursor_to_filter_iso_date() {
let filter = cursor_to_gmail_after_filter("2024-01-15").unwrap();
assert_eq!(filter, "2024/01/15");
}
#[test]
fn cursor_to_filter_rfc3339() {
let filter = cursor_to_gmail_after_filter("2024-06-01T12:00:00Z").unwrap();
assert_eq!(filter, "2024/06/01");
}
#[test]
fn cursor_to_filter_invalid_returns_none() {
assert!(cursor_to_gmail_after_filter("not-a-date").is_none());
}
#[test]
fn cursor_to_filter_trims_whitespace() {
let filter = cursor_to_gmail_after_filter(" 2024-01-15 ").unwrap();
assert_eq!(filter, "2024/01/15");
}
#[test]
fn now_ms_returns_nonzero() {
assert!(now_ms() > 0);
}
// ── second-precision cursor ──────────────────────────────────
#[test]
fn epoch_filter_emits_unix_seconds_for_internal_date_millis() {
let filter = cursor_to_gmail_after_epoch_filter("1700000000000").unwrap();
assert_eq!(filter, "1700000000");
}
#[test]
fn epoch_filter_handles_iso_date() {
let filter = cursor_to_gmail_after_epoch_filter("2024-01-15").unwrap();
// 2024-01-15 00:00:00 UTC == 1705276800.
assert_eq!(filter, "1705276800");
}
#[test]
fn epoch_filter_handles_rfc3339() {
let filter = cursor_to_gmail_after_epoch_filter("2024-06-01T12:00:00Z").unwrap();
assert_eq!(filter, "1717243200");
}
#[test]
fn epoch_filter_returns_none_for_garbage() {
assert!(cursor_to_gmail_after_epoch_filter("not-a-date").is_none());
}
#[test]
fn epoch_filter_trims_whitespace() {
let filter = cursor_to_gmail_after_epoch_filter(" 2024-01-15 ").unwrap();
assert_eq!(filter, "1705276800");
}
#[test]
fn parse_cursor_round_trip_matches_epoch_filter() {
// The adaptive page-cap relies on parse_cursor_to_epoch_secs
// agreeing with cursor_to_gmail_after_epoch_filter — both must
// emit the same seconds value for any given input.
for cursor in ["1700000000000", "2024-01-15", "2024-06-01T12:00:00Z"] {
let secs = parse_cursor_to_epoch_secs(cursor).unwrap();
let filter = cursor_to_gmail_after_epoch_filter(cursor).unwrap();
assert_eq!(filter, secs.to_string(), "cursor `{cursor}`");
}
}
}
@@ -1,301 +1,46 @@
//! Unit tests for the Gmail provider.
//! Host-owned Gmail provider surface tests.
//!
//! Pagination, cursor, envelope parsing, and ingest behavior are owned and
//! tested by `tinycortex::memory::sync::GmailSyncPipeline`.
use super::provider::{BASE_QUERY, SENT_QUERIES};
use super::sync::{
cursor_to_gmail_after_epoch_filter, cursor_to_gmail_after_filter, extract_messages,
extract_page_token, now_ms, parse_cursor_to_epoch_secs,
};
use super::GmailProvider;
use crate::openhuman::memory_sync::composio::providers::ComposioProvider;
use serde_json::json;
#[test]
fn extract_messages_finds_data_messages() {
let v = json!({
"data": { "messages": [{"id": "m1"}, {"id": "m2"}] },
"successful": true,
});
assert_eq!(extract_messages(&v).len(), 2);
}
#[test]
fn extract_messages_finds_top_level_messages() {
let v = json!({ "messages": [{"id": "m1"}] });
assert_eq!(extract_messages(&v).len(), 1);
}
#[test]
fn extract_messages_returns_empty_when_missing() {
let v = json!({ "data": { "other": [] } });
assert_eq!(extract_messages(&v).len(), 0);
}
#[test]
fn extract_page_token_finds_nested() {
let v = json!({ "data": { "nextPageToken": "tok123" } });
assert_eq!(extract_page_token(&v), Some("tok123".to_string()));
}
#[test]
fn extract_page_token_none_when_missing() {
let v = json!({ "data": {} });
assert_eq!(extract_page_token(&v), None);
}
#[test]
fn cursor_to_filter_from_epoch_millis() {
// 1774915200000 ms = 2026-03-31 UTC
let millis = "1774915200000";
assert_eq!(
cursor_to_gmail_after_filter(millis),
Some("2026/03/31".to_string())
);
}
#[test]
fn cursor_to_filter_from_iso_date() {
assert_eq!(
cursor_to_gmail_after_filter("2026-03-15"),
Some("2026/03/15".to_string())
);
}
#[test]
fn cursor_to_filter_from_rfc3339() {
let f = cursor_to_gmail_after_filter("2026-03-15T12:00:00Z");
assert_eq!(f, Some("2026/03/15".to_string()));
}
#[test]
fn cursor_to_filter_returns_none_for_garbage() {
assert_eq!(cursor_to_gmail_after_filter("not-a-date"), None);
}
#[test]
fn provider_metadata_is_stable() {
let p = GmailProvider::new();
assert_eq!(p.toolkit_slug(), "gmail");
assert_eq!(p.sync_interval_secs(), Some(15 * 60));
let provider = GmailProvider::new();
assert_eq!(provider.toolkit_slug(), "gmail");
assert_eq!(provider.sync_interval_secs(), Some(15 * 60));
}
#[test]
fn default_impl_matches_new() {
let _a = GmailProvider::new();
let _b = GmailProvider::default();
// Both are unit structs — constructing via Default is the cover target.
let _new = GmailProvider::new();
let _default = GmailProvider::default();
}
#[test]
fn epoch_filter_is_preferred_over_day_filter_for_typical_internal_date() {
// The provider tries `cursor_to_gmail_after_epoch_filter` first
// and only falls back to the day filter if the parse fails. Both
// helpers must accept the same internalDate (epoch millis) input
// so the fallback path is genuinely a fallback, not a divergence.
let internal_date = "1774915200000"; // 2026-03-31 UTC
let epoch = cursor_to_gmail_after_epoch_filter(internal_date).unwrap();
let day = cursor_to_gmail_after_filter(internal_date).unwrap();
assert_eq!(epoch, "1774915200");
assert_eq!(day, "2026/03/31");
// Sanity bound: the epoch filter must be after 2020 and before
// year 2100, otherwise we shipped a regression in the cursor
// converter that would silently let queries land on year 1970.
let secs: i64 = epoch.parse().unwrap();
assert!(
secs > 1_577_836_800,
"epoch filter must be after 2020-01-01"
);
assert!(
secs < 4_102_444_800,
"epoch filter must be before 2100-01-01"
);
}
// ── Adaptive page cap and early-stop helpers (issue#1404, pr#1474) ──────────
//
// The full `sync()` path needs a live ComposioClient + MemoryClient, so
// we test the helper functions that gate the adaptive cap and early-stop
// decisions:
//
// * `parse_cursor_to_epoch_secs` — used to decide whether `last_sync_at_ms`
// falls within `RECENT_SYNC_WINDOW_MS` (5 min) for the adaptive page cap.
// * `now_ms` — sanity check: must not return 0 and must be within a plausible
// range so the adaptive window comparison never produces pathological results.
// * early-stop guard: when `last_seen_id` matches the first page's head id
// the sync loop breaks with `stop_reason = "head_unchanged"`. We pin the
// helper logic that feeds this decision.
#[test]
fn parse_cursor_to_epoch_secs_handles_epoch_millis() {
// Gmail internalDate is epoch milliseconds as a numeric string.
// 1774915200000 ms = 1774915200 s (2026-03-31 00:00:00 UTC).
assert_eq!(
parse_cursor_to_epoch_secs("1774915200000"),
Some(1774915200)
);
}
#[test]
fn parse_cursor_to_epoch_secs_handles_iso_date() {
// YYYY-MM-DD date cursor produced by the older day-cursor write path.
let secs = parse_cursor_to_epoch_secs("2024-01-15").unwrap();
// 2024-01-15 00:00:00 UTC = 1705276800
assert_eq!(secs, 1705276800);
}
#[test]
fn parse_cursor_to_epoch_secs_handles_rfc3339() {
let secs = parse_cursor_to_epoch_secs("2024-01-15T00:00:00Z").unwrap();
assert_eq!(secs, 1705276800);
}
#[test]
fn parse_cursor_to_epoch_secs_returns_none_for_garbage() {
assert_eq!(parse_cursor_to_epoch_secs("not-a-timestamp"), None);
assert_eq!(parse_cursor_to_epoch_secs(""), None);
assert_eq!(parse_cursor_to_epoch_secs(" "), None);
}
/// The adaptive page cap relies on `parse_cursor_to_epoch_secs` and `now_ms`
/// agreeing on a common epoch so the "less than 5 min ago" comparison works.
/// `now_ms()` must return epoch-milliseconds (not zero, not micros). If it
/// returned microseconds, every sync would appear "recent" (delta < 300_000 ms
/// vs delta actually being ~ 1e12 µs); if it returned seconds, every sync
/// would appear "old" (delta > 300_000 ms trivially).
#[test]
fn now_ms_is_in_epoch_milliseconds_range() {
let ms = now_ms();
// Must be strictly positive.
assert!(ms > 0, "now_ms must not return zero");
// Must be > 2024-01-01 00:00:00 UTC in milliseconds so it's clearly
// millisecond-epoch and not seconds-epoch (which would be ~1.7e9, much
// smaller than 1.7e12).
let jan_2024_ms: u64 = 1_704_067_200_000;
assert!(
ms > jan_2024_ms,
"now_ms ({ms}) must be above 2024-01-01 in epoch-millisecond scale"
);
// Must be < year 2100 in milliseconds — rules out microseconds/nanoseconds.
let year_2100_ms: u64 = 4_102_444_800_000;
assert!(
ms < year_2100_ms,
"now_ms ({ms}) must be below year 2100 in epoch-millisecond scale"
);
}
/// The early-stop optimisation fires when `last_seen_id` equals the first
/// message id on the first page. We test the helper that extracts message ids
/// — `extract_messages` — to verify it correctly surfaces the `id` field so
/// the comparison in the sync loop gets the right value.
///
/// The early-stop check uses `messages.first()` with the `MESSAGE_ID_PATHS`
/// extractor. We can't call the private extractor, but we can pin
/// `extract_messages` to return messages with their `id` intact so the
/// sync loop can compare them to `state.last_seen_id`.
#[test]
fn extract_messages_preserves_id_field_for_early_stop() {
// The early-stop check reads `m["id"]` via `extract_item_id`. Verify
// `extract_messages` doesn't strip or transform the field.
let v = json!({
"data": {
"messages": [
{"id": "msg_abc", "internalDate": "1774915200000"},
{"id": "msg_def", "internalDate": "1774915100000"}
]
},
"successful": true
});
let msgs = extract_messages(&v);
assert_eq!(msgs.len(), 2);
assert_eq!(
msgs[0]["id"], "msg_abc",
"first message id must be preserved"
);
assert_eq!(
msgs[1]["id"], "msg_def",
"second message id must be preserved"
);
}
/// Variant: messages embedded in `data.data.messages` (deeper nesting
/// seen in some Composio provider responses) — the extractor must still
/// find them so the early-stop comparison has data to work with.
#[test]
fn extract_messages_handles_deep_nesting() {
let v = json!({
"data": {
"data": {
"messages": [
{"id": "deep_msg_1"}
]
}
}
});
let msgs = extract_messages(&v);
assert_eq!(msgs.len(), 1);
assert_eq!(msgs[0]["id"], "deep_msg_1");
}
// Note: full `sync` / `fetch_user_profile` / `on_trigger` paths require a
// live `ComposioClient` (HTTP) plus the global `MemoryClient` singleton.
// Those go through the integration test suite. Here we just lock in
// the provider's identity surface and helpers.
// ── Regression tests for issue #1713: sent-mail retrieval ───────────────────
//
// Before the fix the sync query was `in:inbox -in:spam -in:trash`, which meant
// sent emails (label:SENT) were never fetched or ingested into the memory tree.
// The fix removes `in:inbox` so both inbox and sent mail are fetched.
/// Guard: the provider source must NOT contain the `in:inbox` restriction.
/// If this test fails, someone reintroduced the inbox-only query that caused
/// issue #1713.
#[test]
fn provider_source_does_not_restrict_to_inbox() {
let source = include_str!("provider.rs");
// The old restriction started with `"in:inbox`. The double-quote is part
// of the Rust string literal, so this catches the exact regression pattern.
assert!(
!source.contains("\"in:inbox"),
"provider.rs sync query must NOT start with 'in:inbox' — this \
restriction prevents sent emails from being ingested (issue #1713). \
Use '-in:spam -in:trash' to allow both inbox and sent mail."
"provider query must not exclude sent mail"
);
}
/// The base sync query must exclude spam and trash but NOT restrict to inbox.
/// Asserts against the canonical `BASE_QUERY` constant from provider.rs so
/// any change to the production value is caught immediately.
#[test]
fn sync_base_query_excludes_spam_and_trash_without_inbox_restriction() {
assert!(
BASE_QUERY.contains("-in:spam"),
"BASE_QUERY must exclude spam (got: {BASE_QUERY:?})"
);
assert!(
BASE_QUERY.contains("-in:trash"),
"BASE_QUERY must exclude trash (got: {BASE_QUERY:?})"
);
assert!(
!BASE_QUERY.contains("in:inbox"),
"BASE_QUERY must NOT restrict to inbox — omitting this allows sent \
mail to be fetched and ingested (issue #1713). Got: {BASE_QUERY:?}"
);
fn base_query_excludes_spam_and_trash_without_inbox_restriction() {
assert!(BASE_QUERY.contains("-in:spam"));
assert!(BASE_QUERY.contains("-in:trash"));
assert!(!BASE_QUERY.contains("in:inbox"));
}
/// Sent-mail query strings must be non-empty and must not restrict to inbox.
/// Iterates `SENT_QUERIES` from provider.rs — the canonical list of query
/// strings a user or agent can pass to GMAIL_FETCH_EMAILS for sent mail.
#[test]
fn sent_mail_query_strings_are_well_formed() {
assert!(!SENT_QUERIES.is_empty(), "SENT_QUERIES must not be empty");
for q in SENT_QUERIES {
assert!(
!q.is_empty(),
"sent-mail query must not be empty, got empty string in SENT_QUERIES"
);
assert!(
!q.starts_with("in:inbox"),
"sent-mail query '{q}' must not restrict to inbox"
);
assert!(!SENT_QUERIES.is_empty());
for query in SENT_QUERIES {
assert!(!query.is_empty());
assert!(!query.starts_with("in:inbox"));
}
}
@@ -3,8 +3,8 @@
//!
//! Issue: #2400.
mod normalization;
mod provider;
mod sync;
#[cfg(test)]
mod tests;
pub mod tools;
@@ -1,4 +1,4 @@
//! Linear sync helpers — result extraction, issue-title extraction,
//! Linear host normalization helpers — result extraction, issue-title extraction,
//! viewer identity, cursor extraction, and time utilities.
//!
//! Linear's GraphQL API (and therefore Composio's wrapping of it) returns
@@ -21,7 +21,7 @@
use async_trait::async_trait;
use serde_json::json;
use super::sync;
use super::normalization;
use crate::openhuman::memory_sync::composio::providers::{
merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider, CuratedTool,
NormalizedTask, ProviderContext, ProviderUserProfile, TaskFetchFilter, TaskKind,
@@ -86,7 +86,7 @@ impl ComposioProvider for LinearProvider {
}
let data = &resp.data;
let viewer = sync::extract_viewer(data);
let viewer = normalization::extract_viewer(data);
let viewer_ref = viewer.as_ref().unwrap_or(data);
let display_name = pick_str(viewer_ref, &["name", "data.name", "displayName"]);
@@ -143,7 +143,7 @@ impl ComposioProvider for LinearProvider {
resp.error.unwrap_or_else(|| "provider failure".into())
));
}
let viewer_id = sync::extract_viewer_id(&resp.data).ok_or_else(|| {
let viewer_id = normalization::extract_viewer_id(&resp.data).ok_or_else(|| {
"[composio:linear] LINEAR_LIST_LINEAR_USERS returned no viewer id".to_string()
})?;
args["assigneeId"] = json!(viewer_id);
@@ -176,7 +176,7 @@ impl ComposioProvider for LinearProvider {
.filter(|s| !s.is_empty());
let mut out: Vec<NormalizedTask> = Vec::new();
for issue in sync::extract_issues(&resp.data) {
for issue in normalization::extract_issues(&resp.data) {
if out.len() >= max {
break;
}
@@ -203,8 +203,8 @@ impl ComposioProvider for LinearProvider {
/// Map a raw Linear issue payload into a [`NormalizedTask`].
fn normalize_linear_issue(issue: &serde_json::Value) -> Option<NormalizedTask> {
let external_id = pick_str(issue, ISSUE_ID_PATHS)?;
let title =
sync::extract_issue_title(issue).unwrap_or_else(|| format!("Linear issue {external_id}"));
let title = normalization::extract_issue_title(issue)
.unwrap_or_else(|| format!("Linear issue {external_id}"));
Some(NormalizedTask {
external_id,
source_id: String::new(),
@@ -218,7 +218,7 @@ fn normalize_linear_issue(issue: &serde_json::Value) -> Option<NormalizedTask> {
due: pick_str(issue, &["dueDate", "data.dueDate"]),
labels: extract_linear_labels(issue),
priority: pick_str(issue, &["priorityLabel", "data.priorityLabel"]),
updated_at: sync::extract_issue_updated(issue),
updated_at: normalization::extract_issue_updated(issue),
raw: issue.clone(),
})
}
@@ -1,6 +1,6 @@
//! Unit tests for the Linear provider.
use super::sync::{
use super::normalization::{
extract_issue_title, extract_issue_updated, extract_issues, extract_pagination_cursor,
extract_viewer, extract_viewer_id,
};
@@ -1,5 +1,5 @@
mod normalization;
mod provider;
mod sync;
#[cfg(test)]
mod tests;
pub mod tools;
@@ -1,4 +1,4 @@
//! Notion sync helpers — result extraction, pagination cursor,
//! Notion host normalization helpers — result extraction, pagination cursor,
//! page title extraction, and time utilities.
use serde_json::Value;
@@ -18,7 +18,7 @@
use async_trait::async_trait;
use serde_json::{json, Value};
use super::sync;
use super::normalization;
use crate::openhuman::memory_sync::composio::providers::{
first_array_str, merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider,
CuratedTool, NormalizedTask, ProviderContext, ProviderUserProfile, TaskContainer,
@@ -194,7 +194,7 @@ impl ComposioProvider for NotionProvider {
.filter(|s| !s.is_empty());
let mut out: Vec<NormalizedTask> = Vec::new();
for page in sync::extract_results(&resp.data) {
for page in normalization::extract_results(&resp.data) {
if out.len() >= max {
break;
}
@@ -302,8 +302,8 @@ impl ComposioProvider for NotionProvider {
/// preserved for enrichment.
fn normalize_notion_page(page: &serde_json::Value) -> Option<NormalizedTask> {
let external_id = pick_str(page, PAGE_ID_PATHS)?;
let title =
sync::extract_page_title(page).unwrap_or_else(|| format!("Notion page {external_id}"));
let title = normalization::extract_page_title(page)
.unwrap_or_else(|| format!("Notion page {external_id}"));
Some(NormalizedTask {
external_id,
source_id: String::new(),
@@ -358,7 +358,7 @@ fn normalize_notion_page(page: &serde_json::Value) -> Option<NormalizedTask> {
/// "keep only object==database" check silently dropped every database.
/// Pure (no I/O) so it is unit-testable.
pub(super) fn parse_database_results(data: &serde_json::Value) -> Vec<TaskContainer> {
let results = sync::extract_results(data);
let results = normalization::extract_results(data);
let mut kinds: std::collections::BTreeMap<String, usize> = std::collections::BTreeMap::new();
let mut out: Vec<TaskContainer> = Vec::new();
for item in &results {
@@ -1,6 +1,6 @@
//! Unit tests for the Notion provider.
use super::sync::{extract_notion_cursor, extract_page_title, extract_results};
use super::normalization::{extract_notion_cursor, extract_page_title, extract_results};
use super::NotionProvider;
use crate::openhuman::memory_sync::composio::providers::ComposioProvider;
use serde_json::json;
+1 -1
View File
@@ -6,7 +6,7 @@
//! | Submodule | Source | Notes |
//! | --- | --- | --- |
//! | `folder` | Files under a user-added folder memory source | Watch + diff |
//! | `harness` | Agent harness turns (memory_archivist's caller side) | Push-based |
//! | `harness` | Agent harness turns (TinyCortex archivist caller side) | Push-based |
//! | `dictation` | Local audio capture transcripts | Push-based |
//!
//! ## Status
+5 -7
View File
@@ -5,16 +5,15 @@ from generic namespace memory and from `learning::tool_tracker` statistics.
## Namespace convention
Each tool gets its own namespace `tool-{tool_name}`. Build the string via
[`types::tool_memory_namespace`] — never hard-code it.
Each tool gets its own namespace `tool-{tool_name}`. Build the string via the
`tool_memory_namespace` re-export — never hard-code it.
## Layout
| Path | Role |
| --- | --- |
| [`mod.rs`](mod.rs) | Module root + public re-exports. |
| [`types.rs`](types.rs) | **Shim** — re-exports `ToolMemoryRule` / `ToolMemoryPriority` / `ToolMemorySource` / `tool_memory_namespace` from `tinycortex::memory::tool_memory::types` (W7). |
| [`store.rs`](store.rs) | **Shim** — re-exports the crate `ToolMemoryStore` (`put_rule`, `get_rule`, `list_rules`, `delete_rule`, `rules_for_prompt`, `list_tool_names`, `record`, `list_rules_json`) + `tool_memory_store(Arc<dyn Memory>)`. The trait and engine are both crate-owned. |
| [`mod.rs`](mod.rs) | Re-exports crate types/store and defines `tool_memory_store(Arc<dyn Memory>)`. |
| [`capture.rs`](capture.rs) | `ToolMemoryCaptureHook``PostTurnHook` impl that captures user edicts and repeated tool failures into the store (host-retained). |
| [`prompt.rs`](prompt.rs) | **Shim** — re-exports the crate `ToolMemoryRulesSection` + `render_tool_memory_rules` + `TOOL_MEMORY_HEADING`, and keeps the host `PromptSection` impl that plugs the section into the system-prompt builder. |
| [`tools/`](tools/) | Agent-facing read/write tools: `MemoryToolsListTool` (list rules for a tool), `MemoryToolsPutTool` (upsert a rule). |
@@ -35,6 +34,5 @@ The agent harness:
- No upward dependencies — only `memory::Memory` trait (via `Arc<dyn Memory>`)
and project-wide primitives (`tools::traits::Tool`, `serde_json`).
- `MockMemory` is `#[cfg(test)]`-only — never available outside test builds.
- Re-exports in `mod.rs` are the public surface; the underlying submodules
are `pub` so test code can reach in but consumers should go through the
re-exports.
- Re-exports in `mod.rs` are the public surface. Crate-owned type and store
forwarding files were removed; consumers should use the domain root.
+2 -3
View File
@@ -33,8 +33,7 @@ use std::sync::Arc;
use async_trait::async_trait;
use super::store::{tool_memory_store, ToolMemoryStore};
use super::types::{ToolMemoryPriority, ToolMemorySource};
use super::{tool_memory_store, ToolMemoryPriority, ToolMemorySource, ToolMemoryStore};
use crate::openhuman::agent::hooks::{PostTurnHook, ToolCallRecord, TurnContext};
use crate::openhuman::memory::Memory;
@@ -291,8 +290,8 @@ fn tool_aliases(tool_name: &str) -> Vec<&'static str> {
mod tests {
use super::*;
use crate::openhuman::agent::hooks::ToolCallRecord;
use crate::openhuman::memory_tools::store::tool_memory_store;
use crate::openhuman::memory_tools::test_helpers::MockMemory;
use crate::openhuman::memory_tools::tool_memory_store;
fn ctx_with(message: &str, tool_calls: Vec<ToolCallRecord>) -> TurnContext {
TurnContext {
+1 -3
View File
@@ -46,9 +46,7 @@ mod tests {
use crate::openhuman::agent::prompts::types::{
LearnedContextData, PromptContext, ToolCallFormat,
};
use crate::openhuman::memory_tools::types::{
ToolMemoryPriority, ToolMemoryRule, ToolMemorySource,
};
use crate::openhuman::memory_tools::{ToolMemoryPriority, ToolMemoryRule, ToolMemorySource};
fn rule(tool: &str, body: &str, priority: ToolMemoryPriority) -> ToolMemoryRule {
ToolMemoryRule {
+1 -7
View File
@@ -1,16 +1,10 @@
//! Storage layer for tool-scoped rules — thin host shim over
//! `tinycortex::memory::tool_memory::store` (W7).
//!
//! The store engine (put / get / list / delete / prompt over an
//! `Arc<dyn Memory>`) and the `Memory` trait are both owned by tinycortex.
use std::sync::Arc;
use crate::openhuman::memory::Memory;
pub use tinycortex::memory::tool_memory::store::{ToolMemoryStore, TOOL_MEMORY_PROMPT_CAP};
/// Build a crate [`ToolMemoryStore`] over the shared tinycortex memory trait.
/// Build the crate-owned store over OpenHuman's shared memory object.
pub fn tool_memory_store(memory: Arc<dyn Memory>) -> ToolMemoryStore {
ToolMemoryStore::new(memory)
}
+3 -5
View File
@@ -1,10 +1,8 @@
//! Domain types for the tool-scoped memory layer — thin host re-export of
//! `tinycortex::memory::tool_memory::types` (W7).
//! `tinycortex::memory::tool_memory::types`.
//!
//! [`ToolMemoryRule`] / [`ToolMemoryPriority`] / [`ToolMemorySource`] and the
//! [`tool_memory_namespace`] helper are the crate's (a byte-identical port,
//! preserving the serde wire strings + `rule/{id}` storage keys). Host consumers
//! keep their `memory_tools::types::*` import paths unchanged.
//! This module preserves the public `memory_tools::types::*` import path while
//! TinyCortex remains the single implementation and wire-format authority.
pub use tinycortex::memory::tool_memory::types::{
tool_memory_namespace, ToolMemoryPriority, ToolMemoryRule, ToolMemorySource,
+1 -2
View File
@@ -31,8 +31,7 @@ memory_store::trees (persistence: one Tree table, one schema)
| [`tree/`](tree/) | `bucket_seal` (append leaf + cascade seal), `flush` (time-based partial seal), `registry` (kind-parameterized `get_or_create_tree` with UNIQUE-race recovery), `mod.rs` (re-exports + `memory_store::trees` shims for legacy paths). |
| [`summarise.rs`](summarise.rs) | One function: produce the next-level summary text for a bucket. Wraps the chat model with a fixed prompt and token budget. |
| [`retrieval/`](retrieval/) | Agent-facing tools. Read: `walk` (agentic), `drill_down`, `fetch_leaves`, `query_{source,global,topic}`, `search_entities`. Write: `ingest_document` (orchestrator-facing). |
| [`score/`](score/) | Scoring signals, embedding (cloud/ollama/inert), entity extraction (regex/LLM), canonical resolver, entity index store. |
| [`tools.rs`](tools.rs) | Re-exports from `memory::query` for backward compatibility. |
| [`score/`](score/) | Product adapters over TinyCortex scoring and TinyAgents embedding models, plus entity extraction and the entity index store. |
## Layer rules
-1
View File
@@ -13,7 +13,6 @@ pub mod nlp;
pub mod retrieval;
pub mod score;
pub mod summarise;
pub mod tools;
pub mod tree;
pub mod tree_runtime;
+14 -15
View File
@@ -1,18 +1,17 @@
# Memory tree — score embed (Phase 4 / #710)
# Memory tree embedding bridge
Vector embedder for chunks and summaries. Produces a fixed-dimension (`EMBEDDING_DIM = 768`) `Vec<f32>` per text so retrieval can rerank candidates by semantic similarity. Default backend is local Ollama running `nomic-embed-text`; tests use the deterministic `InertEmbedder` so no network is required.
The memory tree stores fixed 1024-dimensional vectors. Concrete provider
transports live in TinyAgents; this directory keeps only memory-tree policy and
compatibility:
## Public surface
- `factory.rs`: read/write provider resolution, cloud-session policy, and
degraded-state behavior;
- `openai_compat.rs`: OpenHuman config/credential and custom-slug resolution;
- `inert.rs`: deterministic 1024-element zero vectors for opt-out/tests;
- `mod.rs`: the legacy `Embedder` contract, `ProviderEmbedder` bridge, batch
fallback/dimension checks, cosine math, and SQLite f32 packing helpers.
- `pub trait Embedder``mod.rs``embed(text) -> Vec<f32>` contract; impls must return exactly `EMBEDDING_DIM` floats.
- `pub fn build_embedder_from_config``factory.rs` — returns `OllamaEmbedder` when configured, otherwise `InertEmbedder` (or bails when `embedding_strict = true`).
- `pub struct OllamaEmbedder``ollama.rs` — HTTP client posting to `{endpoint}/api/embeddings`.
- `pub struct InertEmbedder``inert.rs` — zero-vector embedder for tests.
- `pub fn cosine_similarity` / `pack_embedding` / `unpack_embedding` / `pack_checked` / `decode_optional_blob``mod.rs` — math + SQLite BLOB packing helpers.
## Files
- `mod.rs` — trait, `EMBEDDING_DIM`, math + pack/unpack helpers, write-time / read-time semantics.
- `factory.rs``Config::memory_tree`-driven embedder selection with `embedding_strict` opt-in.
- `ollama.rs` — Ollama `/api/embeddings` client; defaults at `http://localhost:11434` / `nomic-embed-text` / 10s timeout.
- `inert.rs` — zero-vector embedder; cosine similarity between any two inert vectors is 0.0 (zero-magnitude short-circuit), so retrieval tests that need real reranking should hand-stitch embeddings instead of relying on this path.
Ollama uses TinyAgents `OllamaEmbeddingModel` and `/api/embed`, with the shared
8192-token context and batch window. Managed cloud uses the host credential and
privacy wrapper around TinyAgents `CloudEmbeddingModel`. Do not add provider
HTTP clients in this directory.
@@ -1,109 +0,0 @@
//! Cloud (Voyage-backed) embedder for the memory tree.
//!
//! Adapts the OpenHuman backend's `POST /openai/v1/embeddings` surface
//! (Voyage `voyage-3.5`, 1024 dims) to the memory_tree [`Embedder`] trait
//! so Phase 4 ingest / bucket-seal can vectorize chunks without a local
//! Ollama install.
//!
//! The 1024-dim output matches existing on-disk blobs (which were
//! produced by `bge-m3`, also 1024-dim), so this is a drop-in replacement
//! for the Ollama path — no migration of `mem_tree_chunks.embedding`
//! required.
//!
//! Auth: the cloud embedder resolves the session JWT per call via
//! [`OpenHumanCloudEmbedding`], so a session refresh between batches is
//! picked up transparently. When the user is unauthenticated the first
//! `embed()` returns an error; ingest treats that the same as any other
//! embedder failure (don't persist the row, let job retry).
use anyhow::{Context, Result};
use async_trait::async_trait;
use super::{Embedder, EMBEDDING_DIM};
use crate::openhuman::config::Config;
use crate::openhuman::embeddings::cloud::{
OpenHumanCloudEmbedding, DEFAULT_CLOUD_EMBEDDING_DIMENSIONS, DEFAULT_CLOUD_EMBEDDING_MODEL,
};
use crate::openhuman::embeddings::EmbeddingProvider;
/// Cloud-backed memory_tree embedder.
///
/// Wraps [`OpenHumanCloudEmbedding`] (which speaks the OpenAI-compatible
/// `/openai/v1/embeddings` shape backed by Voyage on the OpenHuman
/// backend) and adapts it to the memory_tree [`Embedder`] trait.
pub struct CloudEmbedder {
inner: OpenHumanCloudEmbedding,
}
impl CloudEmbedder {
/// Build a cloud embedder using the same backend resolution as the
/// main embeddings path: `api_url` falls back to
/// [`effective_api_url`](crate::api::config::effective_api_url) and
/// the workspace dir comes from `config.workspace_dir` so the auth
/// service finds the user's session JWT.
pub fn new(config: &Config) -> Self {
let openhuman_dir = config.config_path.parent().map(std::path::PathBuf::from);
Self {
inner: OpenHumanCloudEmbedding::new(
None,
openhuman_dir,
config.secrets.encrypt,
DEFAULT_CLOUD_EMBEDDING_MODEL,
DEFAULT_CLOUD_EMBEDDING_DIMENSIONS,
),
}
}
}
#[async_trait]
impl Embedder for CloudEmbedder {
fn name(&self) -> &'static str {
"cloud"
}
async fn embed(&self, text: &str) -> Result<Vec<f32>> {
let v = self
.inner
.embed_one(text)
.await
.context("cloud embeddings failed")?;
if v.len() != EMBEDDING_DIM {
anyhow::bail!(
"cloud embedder returned {} dims, expected {}",
v.len(),
EMBEDDING_DIM
);
}
Ok(v)
}
/// Collapse N per-text round-trips into a single batched Voyage request by
/// delegating to the inner provider's native batch `embed`. Falls back to
/// per-text embedding (preserving per-position error attribution) on a
/// whole-batch failure or a length mismatch.
async fn embed_batch(&self, texts: &[&str]) -> Vec<Result<Vec<f32>>> {
super::embed_batch_via_provider(&self.inner, "cloud", texts).await
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::openhuman::config::Config;
use tempfile::TempDir;
fn test_config() -> (TempDir, Config) {
let tmp = TempDir::new().unwrap();
let mut cfg = Config::default();
cfg.workspace_dir = tmp.path().to_path_buf();
cfg.config_path = tmp.path().join("config.toml");
(tmp, cfg)
}
#[test]
fn name_is_cloud() {
let (_tmp, cfg) = test_config();
let e = CloudEmbedder::new(&cfg);
assert_eq!(e.name(), "cloud");
}
}
@@ -29,11 +29,14 @@
//! - `OPENHUMAN_MEMORY_EMBED_MODEL`
//! - `OPENHUMAN_MEMORY_EMBED_TIMEOUT_MS`
use anyhow::Result;
use anyhow::{Context, Result};
use super::{CloudEmbedder, Embedder, InertEmbedder, OllamaEmbedder};
use std::time::Duration;
use super::{Embedder, InertEmbedder, ProviderEmbedder, EMBEDDING_DIM};
use crate::openhuman::config::Config;
use crate::openhuman::inference::local::ollama_base_url;
use tinyagents::harness::embeddings::{OllamaEmbeddingModel, RECOMMENDED_OLLAMA_CONTEXT_TOKENS};
/// Cheap heuristic for "is a backend session reachable?" — the cloud
/// embedder needs one and bails on first embed call without it. We use
@@ -69,7 +72,7 @@ pub fn build_embedder_from_config(config: &Config) -> Result<Box<dyn Embedder>>
log::debug!(
"[memory_tree::embed::factory] read → Ollama endpoint={endpoint} model={model} timeout_ms={timeout_ms}"
);
Box::new(OllamaEmbedder::new(endpoint, model, timeout_ms))
Box::new(build_ollama_embedder(&endpoint, &model, timeout_ms)?)
}
EmbedderChoice::OptOut => {
log::info!(
@@ -90,7 +93,7 @@ pub fn build_embedder_from_config(config: &Config) -> Result<Box<dyn Embedder>>
"[memory_tree::embed::factory] read → cloud (Voyage) — flip \
'Memory embeddings' in Local AI Settings to switch to local"
);
Box::new(CloudEmbedder::new(config))
Box::new(build_cloud_embedder(config))
}
EmbedderChoice::NoProvider => {
log::warn!(
@@ -215,7 +218,9 @@ pub fn build_write_embedder(config: &Config) -> Result<Option<Box<dyn Embedder>>
timeout_ms,
} => {
clear_semantic_recall_degraded();
Some(Box::new(OllamaEmbedder::new(endpoint, model, timeout_ms)))
Some(Box::new(build_ollama_embedder(
&endpoint, &model, timeout_ms,
)?))
}
EmbedderChoice::OptOut => {
clear_semantic_recall_degraded();
@@ -231,7 +236,7 @@ pub fn build_write_embedder(config: &Config) -> Result<Option<Box<dyn Embedder>>
}
EmbedderChoice::Cloud => {
clear_semantic_recall_degraded();
Some(Box::new(CloudEmbedder::new(config)))
Some(Box::new(build_cloud_embedder(config)))
}
EmbedderChoice::NoProvider => {
log::warn!(
@@ -245,6 +250,36 @@ pub fn build_write_embedder(config: &Config) -> Result<Option<Box<dyn Embedder>>
})
}
fn build_ollama_embedder(endpoint: &str, model: &str, timeout_ms: u64) -> Result<ProviderEmbedder> {
let timeout = Duration::from_millis(if timeout_ms == 0 { 10_000 } else { timeout_ms });
let client = reqwest::Client::builder()
.connect_timeout(timeout)
.build()
.context("build Ollama embeddings HTTP client")?;
let model = OllamaEmbeddingModel::try_new(endpoint, model, EMBEDDING_DIM)?
.with_context_options(
RECOMMENDED_OLLAMA_CONTEXT_TOKENS,
RECOMMENDED_OLLAMA_CONTEXT_TOKENS,
)
.with_client(client);
Ok(ProviderEmbedder::new(
crate::openhuman::embeddings::TinyAgentsEmbeddingProvider::boxed(model),
"ollama",
))
}
fn build_cloud_embedder(config: &Config) -> ProviderEmbedder {
let openhuman_dir = config.config_path.parent().map(std::path::PathBuf::from);
let provider = crate::openhuman::embeddings::cloud::OpenHumanCloudEmbedding::new(
None,
openhuman_dir,
config.secrets.encrypt,
crate::openhuman::embeddings::cloud::DEFAULT_CLOUD_EMBEDDING_MODEL,
crate::openhuman::embeddings::cloud::DEFAULT_CLOUD_EMBEDDING_DIMENSIONS,
);
ProviderEmbedder::new(Box::new(provider), "cloud")
}
#[cfg(test)]
mod tests {
use super::*;
+37 -4
View File
@@ -29,16 +29,12 @@
use anyhow::{Context, Result};
use async_trait::async_trait;
pub mod cloud;
pub mod factory;
pub mod inert;
pub mod ollama;
pub mod openai_compat;
pub use cloud::CloudEmbedder;
pub use factory::{build_embedder_from_config, build_write_embedder};
pub use inert::InertEmbedder;
pub use ollama::OllamaEmbedder;
pub use openai_compat::OpenAiCompatEmbedder;
/// Embedding dimensionality used across the memory tree.
@@ -89,6 +85,43 @@ pub trait Embedder: Send + Sync {
}
}
/// Adapts the canonical host embedding-provider contract to the legacy
/// memory-tree embedder shape. Concrete network implementations live in
/// `tinyagents::harness::embeddings`; this bridge owns only dimension checks
/// and the memory tree's per-position batch fallback contract.
pub struct ProviderEmbedder {
inner: Box<dyn crate::openhuman::embeddings::EmbeddingProvider>,
label: &'static str,
}
impl ProviderEmbedder {
pub fn new(
inner: Box<dyn crate::openhuman::embeddings::EmbeddingProvider>,
label: &'static str,
) -> Self {
Self { inner, label }
}
}
#[async_trait]
impl Embedder for ProviderEmbedder {
fn name(&self) -> &'static str {
self.label
}
async fn embed(&self, text: &str) -> Result<Vec<f32>> {
self.inner
.embed_one(text)
.await
.with_context(|| format!("{} embeddings failed", self.label))
.and_then(|vector| check_embed_dim(vector, self.label))
}
async fn embed_batch(&self, texts: &[&str]) -> Vec<Result<Vec<f32>>> {
embed_batch_via_provider(self.inner.as_ref(), self.label, texts).await
}
}
/// Validate that a freshly-produced embedding has exactly [`EMBEDDING_DIM`]
/// floats, returning a labelled error otherwise. Shared by the per-text and
/// batched provider adapters so the "wrong dims" diagnostic is identical
@@ -1,384 +0,0 @@
//! Ollama-backed embedder for Phase 4 (#710).
//!
//! Posts `{model, prompt, options: {num_ctx}}` to
//! `{endpoint}/api/embeddings` and expects
//! `{"embedding": [f32; EMBEDDING_DIM]}` back. Designed for a local
//! `ollama serve` hosting `bge-m3`.
//!
//! This is intentionally a tiny HTTP client — no retry, no pool caching,
//! no streaming. Phase 4 wants the simplest thing that works so we can
//! land embedding end-to-end and iterate once baseline retrieval quality
//! is measurable. Timeouts, parallelism, and caching are explicit
//! follow-ups.
use std::time::Duration;
use anyhow::{Context, Result};
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use super::{Embedder, EMBEDDING_DIM};
/// Default Ollama endpoint. Matches the local-install default from the
/// `local_ai` subsystem and the Ollama defaults.
pub const DEFAULT_ENDPOINT: &str = "http://localhost:11434";
/// Default embedding model — must output exactly [`EMBEDDING_DIM`]
/// (1024) dims. `bge-m3` is a multilingual BERT-family encoder with
/// native 8192-token context and 1024-dim output.
pub const DEFAULT_MODEL: &str = "bge-m3";
/// Default request timeout. Ollama's first-use latency is a few hundred
/// ms on a warm model; 10s absorbs a cold-model load on commodity
/// hardware without stalling ingest on a broken backend.
pub const DEFAULT_TIMEOUT_MS: u64 = 10_000;
/// HTTP client wrapping a single Ollama endpoint + model pair.
///
/// Cloneable — `reqwest::Client` shares a connection pool under the hood
/// so cloning the wrapper stays cheap across seal / ingest call sites.
#[derive(Clone)]
pub struct OllamaEmbedder {
endpoint: String,
model: String,
timeout: Duration,
client: reqwest::Client,
}
impl OllamaEmbedder {
/// Build a new embedder. `endpoint` is trimmed of trailing slashes
/// so callers don't have to worry about mixing `http://host` and
/// `http://host/`. Empty values fall back to the public defaults.
pub fn new(endpoint: String, model: String, timeout_ms: u64) -> Self {
let endpoint = if endpoint.trim().is_empty() {
DEFAULT_ENDPOINT.to_string()
} else {
endpoint.trim().trim_end_matches('/').to_string()
};
let model = if model.trim().is_empty() {
DEFAULT_MODEL.to_string()
} else {
model.trim().to_string()
};
let timeout_ms = if timeout_ms == 0 {
DEFAULT_TIMEOUT_MS
} else {
timeout_ms
};
let timeout = Duration::from_millis(timeout_ms);
// No body-read timeout. Ollama is local — slow responses mean
// the model is genuinely processing, not that the network
// broke. A body-read timeout here would cancel mid-stream and
// force retries against the same slow model. `timeout` is
// repurposed as the TCP connect timeout (fast-fail when
// Ollama is actually down).
let client = reqwest::Client::builder()
.connect_timeout(timeout)
.build()
.unwrap_or_else(|e| {
log::warn!(
"[memory_tree::embed::ollama] failed to build client \
with connect_timeout using default: {e}"
);
reqwest::Client::new()
});
log::debug!(
"[memory_tree::embed::ollama] created endpoint={endpoint} \
model={model} timeout_ms={timeout_ms}"
);
Self {
endpoint,
model,
timeout,
client,
}
}
/// Convenience constructor using all defaults ([`DEFAULT_ENDPOINT`],
/// [`DEFAULT_MODEL`], [`DEFAULT_TIMEOUT_MS`]).
pub fn default_new() -> Self {
Self::new(String::new(), String::new(), 0)
}
fn embed_url(&self) -> String {
format!("{}/api/embeddings", self.endpoint)
}
}
fn is_missing_model_response(status: reqwest::StatusCode, body: &str) -> bool {
if status != reqwest::StatusCode::NOT_FOUND {
return false;
}
let normalized = body.to_ascii_lowercase();
normalized.contains("model") && normalized.contains("not found")
}
fn format_embedding_status_error(
status: reqwest::StatusCode,
body: &str,
endpoint: &str,
model: &str,
) -> String {
let trimmed_body = body.trim();
if is_missing_model_response(status, trimmed_body) {
return format!(
"Ollama embedding model `{model}` is not installed at {endpoint}. \
Run `ollama pull {model}` or choose an installed embedding model in \
Connections API keys Embeddings. status={status} body={trimmed_body}"
);
}
format!("ollama embeddings failed status={status} body={trimmed_body}")
}
/// Override Ollama's per-model `num_ctx` default. Ollama loads
/// embedding models with `num_ctx = 4096` (or whatever default the
/// model's modelfile carries) unless the request explicitly asks for
/// more. `bge-m3` natively supports 8192 tokens, and chunker-token
/// counts undercount BERT-WordPiece tokens by ~1.5-2× for HTML-derived
/// markdown — so a 1500-chunker-token chunk routinely produces 2500+
/// real tokens at embed time. Asking for 8192 unconditionally avoids
/// silent prompt truncation; on models that natively support less,
/// Ollama clamps `num_ctx` to the model's actual maximum, so this is
/// safe to over-request.
///
/// This is also the **single source of truth for the memory layer's
/// minimum model context window**: a local model whose native context is
/// below this floor silently truncates chunks/summaries and corrupts
/// recall. `inference::local::model_requirements::MIN_CONTEXT_TOKENS`
/// re-exports this value so the model-acceptance gate can never drift
/// from what the embedder actually requests.
pub(crate) const EMBED_NUM_CTX: u32 = 8192;
#[derive(Serialize)]
struct EmbedRequest<'a> {
model: &'a str,
prompt: &'a str,
options: EmbedOptions,
}
#[derive(Serialize)]
struct EmbedOptions {
num_ctx: u32,
// Ollama/llama.cpp evaluates an embedding prompt in a SINGLE batch and
// rejects (HTTP 500 "input length exceeds the context length") any prompt
// longer than `n_batch` — whose default is 2048, far below `num_ctx`.
// Raising `num_ctx` alone does NOT lift this; `num_batch` must match, or
// chunks between ~2k and 8k real tokens fail to embed even though they fit
// the context window. Set equal to `num_ctx` so the batch limit never bites
// before the real context limit.
num_batch: u32,
}
#[derive(Deserialize)]
struct EmbedResponse {
#[serde(default)]
embedding: Vec<f32>,
}
#[async_trait]
impl Embedder for OllamaEmbedder {
fn name(&self) -> &'static str {
"ollama"
}
async fn embed(&self, text: &str) -> Result<Vec<f32>> {
log::debug!(
"[memory_tree::embed::ollama] embed endpoint={} model={} bytes={}",
self.endpoint,
self.model,
text.len()
);
let req = EmbedRequest {
model: &self.model,
prompt: text,
options: EmbedOptions {
num_ctx: EMBED_NUM_CTX,
num_batch: EMBED_NUM_CTX,
},
};
let resp = self
.client
// No per-request body-read timeout — see `OllamaEmbedder::new`
// for rationale. The Client's `connect_timeout` still applies
// and fails fast if Ollama isn't reachable.
.post(self.embed_url())
.json(&req)
.send()
.await
.with_context(|| {
format!(
"ollama embeddings request failed (is Ollama running at {}?)",
self.endpoint
)
})?;
if !resp.status().is_success() {
let status = resp.status();
let body = resp.text().await.unwrap_or_default();
let msg = format_embedding_status_error(status, &body, &self.endpoint, &self.model);
// Log at WARN so missing-model failures surface in traces without
// requiring debug-level logging to be enabled. Missing-model 404s
// include the `ollama pull` remediation hint from
// `format_embedding_status_error`.
log::warn!("[embeddings] {msg}");
anyhow::bail!(msg);
}
let payload: EmbedResponse = resp
.json()
.await
.context("ollama embeddings response parse failed")?;
if payload.embedding.len() != EMBEDDING_DIM {
anyhow::bail!(
"ollama embeddings returned {} dims, expected {EMBEDDING_DIM} \
(check model={})",
payload.embedding.len(),
self.model
);
}
Ok(payload.embedding)
}
}
#[cfg(test)]
mod tests {
use super::*;
use axum::{extract::Json, http::StatusCode, routing::post, Router};
use std::net::SocketAddr;
/// Spin up a local axum server and return its base URL.
async fn start_mock(app: Router) -> String {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr: SocketAddr = listener.local_addr().unwrap();
tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
format!("http://127.0.0.1:{}", addr.port())
}
fn fixed_vec(val: f32) -> Vec<f32> {
vec![val; EMBEDDING_DIM]
}
#[test]
fn defaults_applied_on_empty_input() {
let e = OllamaEmbedder::new(String::new(), String::new(), 0);
assert_eq!(e.endpoint, DEFAULT_ENDPOINT);
assert_eq!(e.model, DEFAULT_MODEL);
assert_eq!(e.timeout, Duration::from_millis(DEFAULT_TIMEOUT_MS));
}
#[test]
fn trailing_slash_trimmed() {
let e = OllamaEmbedder::new("http://host:1234/".into(), "m".into(), 1000);
assert_eq!(e.endpoint, "http://host:1234");
}
#[test]
fn embed_url_format() {
let e = OllamaEmbedder::default_new();
assert_eq!(e.embed_url(), "http://localhost:11434/api/embeddings");
}
#[test]
fn name_is_ollama() {
assert_eq!(OllamaEmbedder::default_new().name(), "ollama");
}
#[tokio::test]
async fn happy_path_returns_embedding() {
let v = fixed_vec(0.25);
let v_clone = v.clone();
let app = Router::new().route(
"/api/embeddings",
post(move |Json(body): Json<serde_json::Value>| {
let v = v_clone.clone();
async move {
assert_eq!(body["model"], "bge-m3");
assert_eq!(body["prompt"], "hello world");
assert_eq!(body["options"]["num_ctx"], 8192);
assert_eq!(body["options"]["num_batch"], 8192);
Json(serde_json::json!({ "embedding": v }))
}
}),
);
let url = start_mock(app).await;
let e = OllamaEmbedder::new(url, String::new(), 0);
let out = e.embed("hello world").await.unwrap();
assert_eq!(out.len(), EMBEDDING_DIM);
assert!((out[0] - 0.25).abs() < 1e-6);
}
#[tokio::test]
async fn server_error_bubbles_up() {
let app = Router::new().route(
"/api/embeddings",
post(|| async { (StatusCode::INTERNAL_SERVER_ERROR, "model crashed") }),
);
let url = start_mock(app).await;
let e = OllamaEmbedder::new(url, String::new(), 0);
let err = e.embed("hello").await.unwrap_err().to_string();
assert!(err.contains("500"), "msg: {err}");
assert!(err.contains("model crashed"), "msg: {err}");
}
#[tokio::test]
async fn missing_model_404_mentions_pull_command() {
let app = Router::new().route(
"/api/embeddings",
post(|| async { (StatusCode::NOT_FOUND, "{\"error\":\"model not found\"}") }),
);
let url = start_mock(app).await;
let e = OllamaEmbedder::new(url, "custom-embed".into(), 0);
let err = e.embed("hello").await.unwrap_err().to_string();
assert!(
err.contains("embedding model `custom-embed` is not installed"),
"msg: {err}"
);
assert!(err.contains("ollama pull custom-embed"), "msg: {err}");
}
#[tokio::test]
async fn dim_mismatch_rejected() {
let app = Router::new().route(
"/api/embeddings",
post(|| async {
// Return a 3-dim vector — must fail validation.
Json(serde_json::json!({ "embedding": [0.1, 0.2, 0.3] }))
}),
);
let url = start_mock(app).await;
let e = OllamaEmbedder::new(url, String::new(), 0);
let err = e.embed("hi").await.unwrap_err().to_string();
assert!(err.contains("3 dims"), "msg: {err}");
assert!(err.contains("expected 1024"), "msg: {err}");
}
#[tokio::test]
async fn malformed_json_response_rejected() {
let app = Router::new().route(
"/api/embeddings",
post(|| async { (StatusCode::OK, "not even json") }),
);
let url = start_mock(app).await;
let e = OllamaEmbedder::new(url, String::new(), 0);
let err = e.embed("hi").await.unwrap_err().to_string();
assert!(err.contains("parse failed"), "msg: {err}");
}
#[tokio::test]
async fn request_failure_is_descriptive() {
let e = OllamaEmbedder::new("http://%".into(), String::new(), 500);
let err = e.embed("hi").await.unwrap_err().to_string();
assert!(
err.contains("is Ollama running"),
"should mention Ollama: {err}"
);
}
}
-4
View File
@@ -1,4 +0,0 @@
//! Compatibility shim: high-level memory query tools now live in
//! `crate::openhuman::memory::query`.
pub use crate::openhuman::memory::query::*;
-1
View File
@@ -84,7 +84,6 @@ pub mod medulla_local;
pub mod meet;
pub mod meet_agent;
pub mod memory;
pub mod memory_archivist;
pub mod memory_conversations;
pub mod memory_diff;
pub mod memory_goals;
+1 -1
View File
@@ -24,7 +24,7 @@ use crate::openhuman::scheduler_gate::signals::Signals;
/// simultaneous Ollama requests have crashed the user's laptop twice.
///
/// Cloud-backend LLM calls bypass this semaphore at the worker layer
/// (see `memory::jobs::worker::run_once`) because they're
/// (see `memory_queue::worker::run_once`) because they're
/// bandwidth-bound, not RAM-bound, and the worker pool itself bounds
/// concurrency upstream. Keeping this at 1 preserves the laptop-RAM
/// contract regardless of backend.
+10 -18
View File
@@ -5,7 +5,7 @@
//! chunks / tree / retrieval / queue / ingest / score + the long tail). This
//! module is the **adapter seam**, mirroring `src/openhuman/tinyagents/`: it
//! implements the crate's engine traits over OpenHuman services and derives the
//! engine's [`MemoryConfig`] from the host [`Config`]. Nothing here contains
//! engine's [`tinycortex::memory::MemoryConfig`] from the host [`Config`]. Nothing here contains
//! engine logic — that lives in the crate.
//!
//! ## Ownership boundary (the seam contract)
@@ -16,19 +16,20 @@
//! archivist/tool-memory/conversations long tail.
//!
//! **Product (host, stays in OpenHuman):** JSON-RPC schemas/ops/read_rpc, agent
//! tools + `SecurityPolicy` gating, live sync (`memory_sync`), the event bus,
//! preferences, `source_scope` per-turn allowlist, redaction, the global
//! singleton + background queue worker, embedding/LLM **compute**, and the
//! tools + `SecurityPolicy` gating, sync scheduling/credentials/events, the
//! event bus, preferences, `source_scope` per-turn allowlist, redaction, the
//! global singleton + background queue worker, embedding/LLM **compute**, and the
//! host-retained `UnifiedMemory` namespace-document tier (episodic/event/
//! segment/doc/graph/profile tables) plus the `wiki_git`/`obsidian` content
//! surfaces the crate deliberately excludes.
//!
//! The crate never makes a network call or owns a worker pool: LLM/embedding
//! compute is injected through `EmbeddingBackend`, `ChatProvider`, `Summariser`,
//! and `EntityExtractor`; the job queue is driven by the host worker loop via
//! Network-capable sync providers are feature-gated in the crate; their
//! credentials and product policy stay in the host. LLM/embedding compute is
//! injected through `EmbeddingBackend`, `ChatProvider`, `Summariser`, and
//! `EntityExtractor`; the job queue is driven by the host worker loop via
//! `queue::run_once` / `drain_until_idle`. Those adapters live beside this file
//! (`embeddings.rs`, `chat.rs`, `queue_driver.rs`, `sinks.rs`, `bus.rs`) and are
//! added workstream by workstream.
//! (`embeddings.rs`, `chat.rs`, `queue_driver.rs`, `ingest.rs`, `seal.rs`, and
//! `sync.rs`).
//!
//! See `docs/tinycortex-migration-spec.md` for the full ownership split,
//! drift/gap/parity ledgers, and the workstream order.
@@ -66,12 +67,3 @@ pub use sync::{
run_gmail_backfill, run_slack_search_backfill, run_source_pipeline, sync_context,
HostSyncAdapter, SourcePipelineFailure, HOST_SYNC_STATE_NAMESPACE,
};
// Facade re-exports — the rest of the host imports memory-engine types through
// this one seam so consumer import paths stay stable as the internals flip to
// the crate (the type-unification decision, spec §0.5). `MemoryTaint` is the
// security-critical provenance type; it is proven byte-identical to the host's
// (fail-closed to `ExternalSync`) before re-exporting.
pub use tinycortex::memory::{
MemoryCategory, MemoryConfig, MemoryEntry, MemoryTaint, NamespaceSummary, RecallOpts,
WeightProfile,
};
+4 -6
View File
@@ -18,12 +18,10 @@
//! reproduces it exactly. It is a pure function so the policy is unit-tested
//! without spinning a live loop.
//!
//! Still to land in W4 (tracked in the migration spec): the real
//! `HostQueueDelegates` that bridges the 8 delegate methods to the host
//! `memory_tree` engine (paired with W5), the tokio worker-loop + scheduler that
//! applies this policy while driving `run_once`, re-pointing `memory/global.rs`
//! and the enqueue call sites onto `queue::store`, and deleting the legacy
//! `memory_queue` engine. This brick is additive: nothing is flipped yet.
//! The host worker is flipped to `tinycortex::memory::queue::run_once` through
//! `HostQueueDelegates`. The adapter preserves host-owned scheduling, health
//! reporting, event-bus publishing, and product-policy hooks while the crate
//! owns claim/dispatch/settle.
use std::time::Duration;
+1 -1
View File
@@ -22,7 +22,7 @@
use chrono::{TimeZone, Utc};
use openhuman_core::openhuman::config::Config;
use openhuman_core::openhuman::memory::ingest_pipeline::{ingest_chat, ingest_email};
use openhuman_core::openhuman::memory::jobs::drain_until_idle;
use openhuman_core::openhuman::memory_queue::drain_until_idle;
use openhuman_core::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage};
use openhuman_core::openhuman::memory_sync::canonicalize::email::{EmailMessage, EmailThread};
use openhuman_core::openhuman::tools::{
+2 -2
View File
@@ -10,7 +10,7 @@
//! 1. the **crate-owned substrate** the `tinycortex` chunk DB creates
//! (`init_db` → `chunks/schema.rs`), and
//! 2. the **host-retained `UnifiedMemory` namespace-document tier**
//! (`memory_store/unified/*`),
//! (`memory_store/namespace_store/*`),
//!
//! coexisting without collision (parity checklist P3/P5/P11/P12 — the W3 gate).
//! A store/tree cutover that reshaped, renamed, or dropped a table would strand
@@ -105,7 +105,7 @@ const CRATE_CHUNK_SCHEMA_TABLES: &[&str] = &[
];
/// The host-retained `UnifiedMemory` namespace-document tier
/// (`memory_store/unified/*`) — stays host, coexists in the shared workspace.
/// (`memory_store/namespace_store/*`) — stays host, coexists in the shared workspace.
const HOST_UNIFIED_TABLES: &[&str] = &[
"memory_docs",
"graph_global",
@@ -308,7 +308,7 @@ async fn round15_composio_agent_tools_backend_cache_and_trigger_history_edges()
assert_eq!(action_tool.name(), "GMAIL_FETCH_EMAILS");
assert_eq!(action_tool.category().to_string(), "skill");
let contract_result = action_tool
.execute(json!({ "query": "from:me" }))
.execute(json!({ "invented_filter": "from:me" }))
.await
.expect("per-action contract gate");
assert!(contract_result.is_error);

Some files were not shown because too many files have changed in this diff Show More