diff --git a/.github/workflows/ci-lite.yml b/.github/workflows/ci-lite.yml index 4b71a31e6..4e2afa1c8 100644 --- a/.github/workflows/ci-lite.yml +++ b/.github/workflows/ci-lite.yml @@ -856,7 +856,7 @@ jobs: key: tinycortex - name: Run TinyCortex engine and Composio sync tests - run: bash scripts/ci-cancel-aware.sh cargo test --manifest-path vendor/tinycortex/Cargo.toml --features git-diff,sync + run: bash scripts/ci-cancel-aware.sh cargo test --manifest-path vendor/tinycortex/Cargo.toml --features git-diff,sync,persona pr-ci-gate: name: PR CI Gate diff --git a/Cargo.lock b/Cargo.lock index 2e46dbb88..bfdee024d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3079,7 +3079,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.62.2", + "windows-core 0.57.0", ] [[package]] @@ -4746,7 +4746,7 @@ dependencies = [ "tar", "tempfile", "thiserror 2.0.18", - "tinyagents 2.1.0", + "tinyagents", "tinychannels", "tinycortex", "tinyflows", @@ -7332,23 +7332,6 @@ dependencies = [ "crunchy", ] -[[package]] -name = "tinyagents" -version = "2.0.0" -dependencies = [ - "async-trait", - "bytes", - "chrono", - "futures", - "reqwest 0.12.28", - "serde", - "serde_json", - "sha2 0.11.0", - "thiserror 2.0.18", - "tokio", - "tracing", -] - [[package]] name = "tinyagents" version = "2.1.0" @@ -7433,7 +7416,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "thiserror 2.0.18", - "tinyagents 2.0.0", + "tinyagents", "tokio", "toml 0.8.23", "tracing", @@ -7453,7 +7436,7 @@ dependencies = [ "serde", "serde_json", "thiserror 2.0.18", - "tinyagents 2.1.0", + "tinyagents", "tracing", ] @@ -8864,7 +8847,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d004bd392..6b023d390 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -109,10 +109,12 @@ tinyagents = { version = "2.1", features = ["sqlite"] } # queue/ingest/score + long tail), vendored as a git submodule and patched # below to `vendor/tinycortex`. OpenHuman's memory subsystem migrates onto this # crate through the adapter seam in `src/openhuman/tinycortex/` (mirroring the -# tinyagents seam): engine logic in the crate; RPC, agent tools, live sync, -# security gating, and the global singleton stay host-side. rusqlite/git2 are +# tinyagents seam): engine logic (including provider sync pipelines) in the +# crate; RPC, agent tools, sync scheduling/credentials/events, security gating, +# and the global singleton stay host-side. rusqlite/git2 are # aligned to the host pins (=0.40 / 0.21) so one bundled SQLite + one libgit2 -# link. Keep the version pin in lockstep with the submodule tag. +# link. The submodule intentionally tracks reviewed upstream main commits; +# keep this semver requirement compatible with the vendored crate version. tinycortex = { version = "0.1", features = ["git-diff", "persona", "sync"] } tinychannels = { version = "0.1", features = ["relay-websocket"] } serde = { version = "1", features = ["derive"] } @@ -704,12 +706,6 @@ tinyjuice = { path = "vendor/tinyjuice" } tinychannels = { path = "vendor/tinychannels" } tinyplace = { path = "vendor/tinyplace/sdk/rust" } -# TinyCortex temporarily pins the embedding API port while tinyagents #58 is -# pending. Resolve that git source to the same vendored crate so host adapters -# and TinyCortex share one `EmbeddingModel` trait identity. -[patch."https://github.com/senamakel/tinyagents"] -tinyagents = { path = "vendor/tinyagents" } - # Emit just enough DWARF in release builds for Sentry to symbolicate Rust # panics + render surrounding source lines. `line-tables-only` keeps the # binary small (only file+line tables, no full type info) while still diff --git a/app/src-tauri/Cargo.lock b/app/src-tauri/Cargo.lock index d3e7cce18..e1aa752e4 100644 --- a/app/src-tauri/Cargo.lock +++ b/app/src-tauri/Cargo.lock @@ -205,7 +205,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -216,7 +216,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -2061,7 +2061,7 @@ dependencies = [ "libc", "option-ext", "redox_users 0.5.2", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -2479,7 +2479,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -3706,7 +3706,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.62.2", + "windows-core 0.57.0", ] [[package]] @@ -4107,7 +4107,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -4991,7 +4991,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -5603,7 +5603,7 @@ dependencies = [ "tar", "tempfile", "thiserror 2.0.18", - "tinyagents 2.1.0", + "tinyagents", "tinychannels", "tinycortex", "tinyflows", @@ -7133,7 +7133,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -7192,7 +7192,7 @@ dependencies = [ "security-framework 3.7.0", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -7949,7 +7949,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -8809,7 +8809,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -8968,23 +8968,6 @@ dependencies = [ "strict-num", ] -[[package]] -name = "tinyagents" -version = "2.0.0" -dependencies = [ - "async-trait", - "bytes", - "chrono", - "futures", - "reqwest 0.12.28", - "serde", - "serde_json", - "sha2 0.11.0", - "thiserror 2.0.18", - "tokio", - "tracing", -] - [[package]] name = "tinyagents" version = "2.1.0" @@ -9064,7 +9047,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "thiserror 2.0.18", - "tinyagents 2.0.0", + "tinyagents", "tokio", "toml 0.8.2", "tracing", @@ -9084,7 +9067,7 @@ dependencies = [ "serde", "serde_json", "thiserror 2.0.18", - "tinyagents 2.1.0", + "tinyagents", "tracing", ] @@ -9670,7 +9653,7 @@ checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" dependencies = [ "memoffset", "tempfile", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -10466,7 +10449,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/docs/TEST-COVERAGE-MATRIX.md b/docs/TEST-COVERAGE-MATRIX.md index 1153bc166..717434d05 100644 --- a/docs/TEST-COVERAGE-MATRIX.md +++ b/docs/TEST-COVERAGE-MATRIX.md @@ -366,10 +366,10 @@ End-to-end coverage of the agent harness via the web-chat RPC surface against an | ----- | ------------------------------------------ | ----- | -------------------------------------------------------------------------------------------------------------- | ------ | ----------------------------------------------------------------------- | | 8.4.1 | Save Preference (general / situational) | RU | `src/openhuman/agent/tools/save_preference_tests.rs` | ✅ | `save_preference` tool → `user_pref_{general,situational}`, topic-keyed | | 8.4.2 | Lane A — Standing Prefs in System Prompt | RU | `src/openhuman/learning/prompt_sections.rs`, `src/openhuman/agent/harness/session/turn_tests.rs` | ✅ | General prefs rendered into the system prompt at thread start | -| 8.4.3 | Lane B — Situational Recall (vector-gated) | RU | `src/openhuman/memory/store/unified/query_tests.rs::recall_relevant_by_vector_gates_on_similarity` | ✅ | Per-turn; relevant query injects, unrelated suppresses | +| 8.4.3 | Lane B — Situational Recall (vector-gated) | RU | `src/openhuman/memory_store/namespace_store/query_tests.rs::recall_relevant_by_vector_gates_on_similarity` | ✅ | Per-turn; relevant query injects, unrelated suppresses | | 8.4.4 | Same-Topic Contradiction (replace) | RU | `src/openhuman/agent/tools/save_preference_tests.rs::recategorising_moves_pref_between_namespaces` | ✅ | `ON CONFLICT REPLACE`; a topic lives in exactly one scope | | 8.4.5 | Cross-Topic Contradiction Surfacing | RU | `src/openhuman/agent/tools/save_preference_tests.rs::save_surfaces_related_preference_for_contradiction_check` | ✅ | Related prefs surfaced in the tool result for the chat agent to resolve | -| 8.4.6 | vector_chunks Model-Signature Recall Guard | RU | `src/openhuman/memory/store/unified/query_tests.rs::vector_recall_excludes_other_model_signature` | ✅ | Excludes cross-model vectors; dim-guards legacy rows | +| 8.4.6 | vector_chunks Model-Signature Recall Guard | RU | `src/openhuman/memory_store/namespace_store/query_tests.rs::vector_recall_excludes_other_model_signature` | ✅ | Excludes cross-model vectors; dim-guards legacy rows | ### 8.5 Long-term Goals diff --git a/docs/tinycortex-api-gap-audit.md b/docs/tinycortex-api-gap-audit.md index 1272f307c..2cf269eae 100644 --- a/docs/tinycortex-api-gap-audit.md +++ b/docs/tinycortex-api-gap-audit.md @@ -1,5 +1,10 @@ # TinyCortex API Gap Audit (Phase 0.2) +**Status (2026-07-22):** Historical gap inventory for the completed engine +cutover. Counts and source anchors below describe the audit base; unresolved +host consolidation work, including G1 re-homing and sync dedupe, is tracked in +`tinycortex-migration-plan-2026-07-22.md`. + **Purpose.** Map every host call site into the engine-mapping modules (`memory_store`, `memory_tree`, `memory_queue`, and the long tail) onto a TinyCortex public API. Where no crate API covers a host need, record a **gap** — each gap becomes a @@ -48,16 +53,16 @@ The crate exposes every seam the plan expects, plus more. Host implements these: | `Summariser` (time-tree, string) | `tree/runtime/engine.rs:29` | **divergent — see G6** | | `EntityExtractor` (regex + LLM) | `score/extract/composite.rs:16` | `tinycortex/chat.rs` (LLM variant) | | `QueueDelegates` | `queue/handlers.rs:96` | `tinycortex/queue_driver.rs` | -| `TreeJobSink` | `ingest/types.rs:15` | `tinycortex/sinks.rs` | -| `TreeLeafSink` | `archivist/sink.rs:44` | `tinycortex/sinks.rs` | -| `SnapshotItemSource` | `diff/source.rs:28` | `tinycortex/sinks.rs` | -| `EntityOccurrenceIndex` | `graph/types.rs:44` | `tinycortex/sinks.rs` | +| `TreeJobSink` | `ingest/types.rs:15` | `tinycortex/ingest.rs` | +| `TreeLeafSink` | `archivist/sink.rs:44` | `tinycortex/ingest.rs` | +| `SnapshotItemSource` | `diff/source.rs:28` | host diff adapter | +| `EntityOccurrenceIndex` | `graph/types.rs:44` | host graph adapter | | `SelfIdentity` | `store/entity_index/store.rs:25` | host identity registry | | `GoalsGenerator` | `goals/reflect.rs:57` | `tinycortex/chat.rs` | | `SourceReader` | `sources/readers/mod.rs:1732` | local readers only *(amended: live-sync readers join the crate in W-SYNC, plan §8)* | | `SyncEventSink` *(new, W-SYNC)* | `sync/traits.rs` (planned) | `tinycortex/sync_sink.rs` → `MemorySyncStage` bus events | | `SkillDocSink` *(new, W-SYNC)* | `sync/traits.rs` (planned) | forwards to `MemoryClient::store_skill_sync` (host-retained unified tier) | -| `ConversationEventBus` / `ChannelEventHandler` | `conversations/bus.rs:106/95` | `tinycortex/bus.rs` (translate to `DomainEvent`) | +| `ConversationEventBus` / `ChannelEventHandler` | `conversations/bus.rs:106/95` | host `memory_conversations/bus.rs` (translate to `DomainEvent`) | Config seam: `MemoryConfig{workspace, embedding: EmbeddingConfig{dim,model,strict}, tree: TreeConfig{input_token_budget,output_token_budget,summary_fanout,flush_age_secs}, @@ -86,7 +91,7 @@ retrieval: RetrievalConfig{default_profile}, sync_budget: SyncBudgetConfig}` + ### G2 — Graph relation-edge persistence (E2GraphRAG accumulation) — **HARD** -- **Host:** `memory_store/unified/graph.rs`, `unified/query.rs` persist and query co-occurrence +- **Host:** `memory_store/namespace_store/{graph,query}.rs` persist and query co-occurrence / LLM-triple relations; `NamespaceMemoryHit.supporting_relations: Vec` is populated at retrieval. - **Crate:** persists the **entity occurrence index** at persist time diff --git a/docs/tinycortex-drift-ledger.md b/docs/tinycortex-drift-ledger.md index 0f15010fe..66420eb77 100644 --- a/docs/tinycortex-drift-ledger.md +++ b/docs/tinycortex-drift-ledger.md @@ -25,7 +25,9 @@ touched an engine-mapping memory module after the port line, and classifies each | Host audit SHA | `7850cf363559bcbb7ba688cbc4fccdb6bd9ce754` (`main`, 2026-07-04) | | TinyCortex submodule | `vendor/tinycortex` → `tinyhumansai/tinycortex` | | TinyCortex audit SHA | `d1a8c7be2babc8fff7a72ed93861f459f3d6fa58` | -| **TinyCortex pinned SHA (current)** | `a8e10f7dd8ebdb9b0905e1380fefcc6bf5a65207` — audit SHA **+ #59** (native-dep alignment, §0.4) **+ #63/#64** (D2/D1 drift ports) merged; this is the live gitlink | +| TinyCortex historical cutover SHA | `a8e10f7dd8ebdb9b0905e1380fefcc6bf5a65207` — audit SHA **+ #59** (native-dep alignment, §0.4) **+ #63/#64** (D2/D1 drift ports) merged | +| **TinyCortex reviewed gitlink (2026-07-22)** | `daaaf6ba5f02635c08deae2b2b2ed7fcc8c06b6a` — includes OpenHuman #4794/#4820/#4863-era crate work; upstream currently has no tags | +| **TinyCortex migration branch (current)** | `7b4b115` — TinyAgents 2.1 dependency alignment, standalone patch configuration, and corrected sync ownership docs | | TinyCortex crate version | `0.1.1` | | **Port line (derived)** | **after 2026-06-25, before 2026-06-28** (see below) | @@ -38,6 +40,11 @@ touched an engine-mapping memory module after the port line, and classifies each > (conversations) are now unblocked per the gate rule. D4 (memory_sync corpus, > W-SYNC) remains its own separate track. +> **Consolidation status (2026-07-22): D1–D4 CLOSED.** D1–D3 remain closed. +> D4 was re-audited against the crate sync implementation and the live host +> dispatch path. The host schedulers, credentials, RPC, source-scope/redaction +> policy, product task normalizers, and event-bus adapters remain host-owned. + ### How the port line was located The port commits in `vendor/tinycortex` are all dated **2026-06-29**, but that is the date the @@ -132,26 +139,23 @@ its `tools/` stay host (agent tools), its `vector`/`scoring` are engine (W5) — ## D4 — memory_sync corpus (2026-07-09 reclassification, W-SYNC) -The plan's §8 amendment moves the sync engine into the crate, so the blanket **HOST-OWNED -"live sync"** classification above is superseded for engine-mapping sync code. Unlike D1–D3 there -is **no existing crate port to diff against** — the entire `memory_sync/` engine (plus the -`memory_sources` dispatcher/reconcile parts) is port scope. The port baseline for W-SYNC is taken -**fresh from host `main` at W-SYNC.1 branch time**, so ordinary drift cannot accumulate; this entry -exists to (a) record the reclassification and (b) pin the post-port-line commits that already -touched the corpus, so the W-SYNC.1 port provably includes them: +The plan's §8 amendment moved the generic sync engine into the crate, so the +blanket **HOST-OWNED "live sync"** classification above is superseded for +engine-mapping sync code. The crate port landed in `0333d10` and is the path +called by `memory_sync::composio::run_connection_sync` and the default +`ComposioProvider::sync` implementation. | # | Host commit | Files in corpus | Note | | --- | --- | --- | --- | -| D4.1 | `c43f79641` (07-03) | `composio/providers/{sync_state,traits}.rs` | TinyAgents-cutover import churn | -| D4.2 | `27b00b539` (07-05) | `sources/rebuild.rs` | test-parity cleanup (1 line) | -| D4.3 | `653e6e143` (07-06) | `memory_sources/sync.rs` (+312) | self-heal drifted content-sha tokens + prune vanished folder items | -| D4.4 | `e456b7799` (07-07) | `memory_sources/{rpc,sync}.rs`, `canonicalize/email.rs`, `composio/providers/{gmail/post_process,notion/source,orchestrator}.rs`, `sources/github.rs` | orchestration-fixes wave | +| D4.1 | `c43f79641` (07-03) | `composio/providers/{sync_state,traits}.rs` | ✅ **CLOSED.** Import churn only; persistence and provider dispatch are implemented by the `SyncStateStore` seam and crate dispatcher. | +| D4.2 | `27b00b539` (07-05) | `sources/rebuild.rs` | ✅ **CLOSED.** Test-only parity cleanup; no engine semantic delta. | +| D4.3 | `653e6e143` (07-06) | `memory_sources/sync.rs` (+312) | ✅ **CLOSED.** Crate `sync/workspace.rs` updates changed items and prunes vanished items through `LocalDocumentSink::delete`; covered by workspace sync tests. | +| D4.4 | `e456b7799` (07-07) | `memory_sources/{rpc,sync}.rs`, `canonicalize/email.rs`, `composio/providers/{gmail/post_process,notion/source,orchestrator}.rs`, `sources/github.rs` | ✅ **CLOSED.** Generic provider fetch/pagination/canonical memory records are in the crate port; orchestration, product task normalization, action-result presentation, and RPC fixes are host policy by design. | -**Gate:** W-SYNC.1 (crate scaffolding PR) requires this enumeration current as of its branch point -(re-run the scan: `git log --since=2026-06-25 --oneline -- src/openhuman/memory_sync -src/openhuman/memory_sources`); the W-SYNC.3 host flip requires D4 **CLOSED** — every listed commit -(and any accrued since) verifiably contained in the crate port or explicitly waived. Host-retained -parts (schedulers, bus subscribers, RPC wrappers, keychain/OAuth) remain HOST-OWNED as before. +**Gate result:** D4 is **CLOSED**. Gmail's Composio 413 mitigation is additionally +present in crate commit `ba9e12e` (25-message fetch pages). Host-retained parts +(schedulers, bus subscribers, RPC wrappers, keychain/OAuth, action tools and +product task/profile projections) remain HOST-OWNED. ## Closing the ledger (procedure) diff --git a/docs/tinycortex-memory-migration-plan.md b/docs/tinycortex-memory-migration-plan.md index c6f673aac..3fd78a86f 100644 --- a/docs/tinycortex-memory-migration-plan.md +++ b/docs/tinycortex-memory-migration-plan.md @@ -1,6 +1,9 @@ # TinyCortex Memory Migration — Plan & Audit -**Status:** W1–W2 landed; W3 partial (chunks sub-store flipped). Amended 2026-07-09 (see §8) to a **sync-inclusive** scope. +**Status:** Historical execution plan. W1–W8 and the engine test port are landed +(OpenHuman #4794/#4820); persona/coding-session ingest followed in #4863. The +remaining consolidation work is tracked in +[`tinycortex-migration-plan-2026-07-22.md`](tinycortex-migration-plan-2026-07-22.md). **Anchor precedent:** the TinyAgents harness migration (#4249 / #4399 / #4473 and follow-ups). **Target:** migrate large portions of the OpenHuman memory subsystem onto the `tinycortex` crate, vendored as a git submodule at **`vendor/tinycortex`** (`https://github.com/tinyhumansai/tinycortex`). @@ -83,8 +86,8 @@ New sibling module mirroring `src/openhuman/tinyagents/`, holding every impl of - `chat.rs` — `impl ChatProvider` + `impl Summariser` over `memory::chat::build_chat_provider` / `inference::provider`. - `queue_driver.rs` — tokio worker loop calling `queue::run_once`/`drain_until_idle`, owned by `memory/global.rs`, with event-bus progress emission and Sentry hooks (host-side, since TinyCortex dropped its scheduler). - `config.rs` — `Config` → `MemoryConfig` (workspace roots, `EmbeddingConfig`, `TreeConfig`, `WeightProfile`, `SyncBudgetConfig`) with `tree_policy.rs` flavour overlays. -- `sinks.rs` — `TreeJobSink`, `SnapshotItemSource` impls bridging to host state. -- `bus.rs` — translate engine outcomes into `core::event_bus` `DomainEvent`s (host-side only; the crate stays bus-free). +- `ingest.rs` / `seal.rs` — `TreeJobSink`, seal observer, and host state bridges. +- `sync.rs` — sync persistence/policy plus translation into host `DomainEvent`s (the crate stays bus-free). - `mod.rs` — facade re-exports (`pub use tinycortex::memory::{…}`) so the rest of the host imports through one seam, plus module-doc explaining the boundary (the tinyagents seam's `mod.rs` header is the template). --- @@ -103,7 +106,7 @@ This phase produces documents and upstream issues only; it is the gate for every **0.5 Type-unification decision.** Host `memory::traits` types and crate types are wire-compatible twins. Decide: host re-exports crate types (preferred — one source of truth, 30+ consumer sites unchanged via `pub use`), vs. keeping host types + `From` conversions (fallback if serde/API divergence is found in 0.3). Special care: `MemoryTaint` is **security-critical provenance** (fails closed to `ExternalSync`, drives external-effect-tool gating) — its semantics, serde representation, and fail-closed defaults must be proven identical before re-exporting. -**0.6 Spec doc.** Write `docs/tinycortex-migration-spec.md` version-anchored to exact reviewed SHAs (host + tinycortex), with the ownership lists above, the drift/gap/parity ledgers, and a **deletion ledger** skeleton (every legacy file, with preconditions for deletion) — directly modeled on `docs/tinyagents-migration-spec.md` + `99-deletion-ledger.md`. +**0.6 Spec doc.** Write `docs/tinycortex-migration-spec.md` version-anchored to exact reviewed SHAs (host + tinycortex), with the ownership lists above, the drift/gap/parity ledgers, and a **deletion ledger** skeleton (every legacy file, with preconditions for deletion) — coordinated with `docs/tinyagents-migration-plan-2026-07-22.md`; the TinyCortex deletion ledger lives in §2 of the spec. --- @@ -146,7 +149,7 @@ W3–W5 are the risky core (user data on disk). W1–W2 can land quickly; W6–W Ordering rule for this migration: **within each workstream, the implementation (adapter + cutover + deletion) lands first; the test work follows as the second slice.** Tests are not skipped — they are sequenced after the implementation is proven to compile and pass the *existing* suites. Concretely: -1. **Slice A (impl):** adapters + cutover + legacy deletion. Gate: `cargo check` both worlds, existing crate-level integration tests (`tests/memory_roundtrip_e2e.rs`, `memory_tree_sync_deep_raw_coverage_e2e.rs`, etc.) still green — these exercise the public `openhuman::memory::` surface and act as the built-in parity harness for every flip. +1. **Slice A (impl):** adapters + cutover + legacy deletion. Gate: `cargo check` both worlds, existing crate-level integration tests (`tests/memory_roundtrip_e2e.rs`, `tests/raw_coverage/memory_tree_sync_deep_raw_coverage_e2e.rs`, etc.) still green — these exercise the public `openhuman::memory::` surface and act as the built-in parity harness for every flip. 2. **Slice B (tests):** port and extend the test estate for the new boundary: - **Engine-internal tests move upstream.** The host's `#[path]`-included sibling tests (`ops_tests.rs` pattern) that poke engine internals migrate into `vendor/tinycortex` as crate tests (via tinycortex PRs), following the crate's own `*_tests.rs` sibling convention. - **Host keeps boundary tests:** RPC schema/handler tests, tool gating tests, seam adapter tests (config mapping, embedding signature, taint fail-closed, source-scope enforcement), and the E2E files under `tests/`. diff --git a/docs/tinycortex-migration-plan-2026-07-22.md b/docs/tinycortex-migration-plan-2026-07-22.md new file mode 100644 index 000000000..8cdb9912d --- /dev/null +++ b/docs/tinycortex-migration-plan-2026-07-22.md @@ -0,0 +1,417 @@ +# TinyCortex Migration — Current-State Audit & Consolidated Plan (2026-07-22) + +**Status:** **DONE** on `feat/tinycortex-migration-2026-07-22`. +[CI Full run 29925645209](https://github.com/senamakel/openhuman/actions/runs/29925645209) +is green, including the final gate and every Linux, macOS, and Windows desktop +shard +(companion to `tinyagents-migration-plan-2026-07-22.md`; +supersedes the *status/phasing* in `tinycortex-memory-migration-plan.md`, +`tinycortex-migration-spec.md`, `tinycortex-api-gap-audit.md`, and +`tinycortex-parity-checklist.md` — those docs' ownership splits, gap lists, and +parity dimensions remain the reference detail. `tinycortex-drift-ledger.md` +stays the row-level ledger but its anchors are stale — see §2). +**Scope:** finish moving the remaining generic memory-engine code from +`src/openhuman/` into the vendored `tinycortex` crate (`vendor/tinycortex`), +delete the in-tree duplicates and staging code, migrate/retire the affected +tests, and clean up every dangling doc/code reference. +**Method:** fresh four-way audit — (1) the crate surface, (2) the core memory +domains, (3) the periphery + the `src/openhuman/tinycortex/` seam, (4) the +docs/tests/git history — against the working tree at `main` (`5b8a9f269`, +2026-07-22). + +--- + +## 1. Executive summary + +**The engine migration is essentially done and the docs don't know it.** All +five `docs/tinycortex-*` docs self-describe a "W1–W2 landed; W3 partial" state, +but git shows the full W-track landed: W1 seam (#4526), W2 type re-export +(#4529), the W3 chunk-store cutover chain (#4532–#4559), W4 queue flip +(#4781), W5 vector+scoring (#4790), the W7 long-tail shims (#4785–#4789), +**engine-migration completion (#4794)**, **engine test port into the crate +(#4820)**, and post-completion feature work (persona/coding-session ingest +#4863) on top. The drift ledger's D1–D3 rows are all CLOSED. + +What remains is not "migrate the engine" but five bounded packages (§5): + +1. **Housekeeping** — the pin/tag story is broken (crate has *no git tags* + despite the Cargo.toml comment demanding tag-lockstep), the host CI vendor + suite omits the `persona` feature the host enables, and there are ~8 + broken/stale references including a seam module doc naming files that were + never created. +2. **Delete `memory_store/unified/`** — the explicitly "staging for removal" + tier (~5k LOC + ~3.7k test LOC), gated by the G1 `sqlite_conn()` escape + hatch. +3. **W-SYNC completion** — `memory_sync/` (17.7k LOC, the largest remaining + host mass) still runs the host provider pipelines while the crate now ships + a full `memory::sync` Composio engine; the D4 ledger rows gate the flip. +4. **Embedding-provider dedupe** — host `embeddings/` adapters and + `memory_tree/score/embed/` providers now duplicate what + `tinyagents::harness::embeddings` ships since tinyagents #58 (this package + lands in **tinyagents**, not tinycortex — the crates split the job). +5. **Shim retirement + funnel decision** — the compatibility shims left behind + by W3–W7, and a policy call on the fact that domains import + `tinycortex::memory::*` directly while the seam's re-export facade has + **zero** consumers. + +--- + +## 2. Ground truth: pins, versions, and discrepancies (fix first) + +| Fact | Value | Where | +| --- | --- | --- | +| Host requirement | `tinycortex = { version = "0.1", features = ["git-diff", "persona", "sync"] }` | root `Cargo.toml:116` | +| Path override | `[patch.crates-io] tinycortex = { path = "vendor/tinycortex" }` | `Cargo.toml:682` | +| Vendored crate | `0.1.1`, edition 2021, **MIT** (not GPL — unlike tinyagents), repo `tinyhumansai/tinycortex` | `vendor/tinycortex/Cargo.toml` | +| Submodule HEAD | `daaaf6ba` (`heads/main`, dependabot-era, ~50+ commits past every SHA the docs cite) | `git submodule status` | +| Tags | **none exist** — `git describe --tags` fails; yet host `Cargo.toml:115` says "Keep the version pin in lockstep with the submodule tag" | submodule | +| Docs' claimed pin | gitlink `a8e10f7` (exists in history, long since passed) | drift ledger / spec anchors | +| Nested dep | tinycortex requires `tinyagents = { version = "2", path = "vendor/tinyagents" }`, **its own nested submodule pinned at v2.0.0** | `vendor/tinycortex/Cargo.toml:69` | +| Host tinyagents req | `2.1` | root `Cargo.toml:107` | +| Fork patch | `[patch."https://github.com/senamakel/tinyagents"]` exists *for tinycortex's sake* ("temporarily pins the embedding API port while tinyagents #58 is pending" — #58 is now merged at the tinyagents submodule HEAD) | `Cargo.toml:685-691` | +| Host CI vendor suite | `cargo test --manifest-path vendor/tinycortex/Cargo.toml --features git-diff,sync` — **omits `persona`**, which the host enables | `.github/workflows/ci-lite.yml:859` | + +Version skew summary: three different tinyagents expectations (host `2.1`, +tinycortex `2`, nested submodule `2.0.0`) resolve only via path patches. The +tinyagents plan's WP-0 (bump `vendor/tinyagents` to v2.1.0) should be executed +together with bumping tinycortex's *nested* `vendor/tinyagents` so both crates +share one trait identity at one version, after which the `senamakel` patch +block and its "temporarily pins" comment can likely be retired. + +--- + +## 3. What the crate already ships + +`tinycortex 0.1.1` exposes a single `memory` module — the engine "ported from +OpenHuman" — with features `tokio`, `git-diff` (git2-backed `memory::diff`), +`providers-http` (reqwest embedding/LLM providers, host does **not** enable), +`sync` (live Composio + workspace-scan engine), `persona` (doc-06 persona +distillation). Modules: `store` (content/vectors/kv/entity_index/safety+PII), +`chunks`, `sources` (+readers), `score` (embed/extract/signals), `tree` +(+runtime, bucket/document seal, flavoured trees), `queue` (SQLite job queue + +tokio runtime), `retrieval` (hybrid search, MMR, rerank), `entities`, `graph`, +`goals`, `tool_memory`, `conversations`, `archivist`, `ingest` +(canonicalize→chunk→score→tree), `diff`, `sync` (composio +clickup/github/linear/notion/slack/gmail), `providers`, `persona` +(readers for claude_code/codex/git history), `fsutil`. + +Host-facing extension points (~20 public traits) with the seam's current +implementations in parentheses: `Memory`, `MemoryStore`, `EmbeddingBackend` + +`Embedder` (`SeamEmbedder`), `ChatProvider` (`SeamChatProvider`), `Summariser` +(`HostSummariser`), `SealObserver` + seal-time `Embedder` bridge (`seal.rs`), +`TreeJobSink` (`HostTreeJobSink`), `QueueDelegates` (`HostQueueDelegates`), +`SourceReader`, `SelfIdentity`, `EntityOccurrenceIndex`, `SnapshotItemSource`, +`GoalsGenerator`, `ConversationEventBus`/`ChannelEventHandler`, +`PersonaStateStore`, and the sync seams `SyncStateStore` / `SyncEventSink` / +`SkillDocSink` / `LocalDocumentSink` / `ExternalSourceReader` / `SyncPipeline` +/ `ActionExecutor` / `IncrementalSource` (`sync.rs`, `HostSyncAdapter`). + +The crate consumes `tinyagents::harness::embeddings::{EmbeddingModel, +format_embedding_signature}` in `store/vectors/embedding.rs` — that is the +whole tinyagents dependency (the W-EMB decision made real by tinyagents #58). + +Crate CI: fmt, clippy `-D warnings`, build+test `--all-features`, doc build +with warnings denied, plus a per-feature matrix (`core`, `tokio`, `git-diff`, +`sync`, all). Offline by convention; Composio/live tests are wiremock-mocked or +env-key-gated. + +--- + +## 4. Audit: what remains host-side + +### 4.1 Per-domain state (crate-backed file counts via `use tinycortex::`) + +| Domain | Files / LOC | Crate-backed | State | +| --- | --- | --- | --- | +| `memory/` | 69 / 18.0k | 3 | Orchestration + policy + RPC layer (by design "no SQLite here"). Predominantly host: `read_rpc/`, `schema*/`, `rpc_models`, agent tools, sync orchestration, tree *policy*. Stays, minus shim cleanup. | +| `memory_store/` | 54 / 17.7k | 15 | Sub-stores flipped to crate (chunks, content, vectors, kv, entities, trees, safety/PII per W3/W7 markers). **`unified/` (~5k + ~3.7k tests) is README-marked "Staging for removal"** — documents/query/segments/events/profile await per-kind flips. `factories.rs` (config/provider selection) stays host. | +| `memory_tree/` | 47 / 10.3k | 16 | Largest crate-backed surface; engine parts delegate. Host keeps RPC (`tree/rpc.rs` 1,387, `retrieval/rpc.rs`), CLI, bus subscriber, `health/` doctor (G5, host-retained by decision), and `score/embed/` **providers** (~1.9k — see WP-3). | +| `memory_queue/` | 7 / 1.4k | 2 | W4 flipped: `worker.rs` delegates claim→dispatch→settle to `tinycortex::memory::queue::run_once` via `HostQueueDelegates`; per-kind handlers deleted. Host keeps worker pool, scheduler tick, `scheduler_gate` throttle, SQLite glue. Done, minus the `memory::jobs` re-export shim. | +| `memory_search/` | 8 / 0.8k | 2 | W5 flipped: `scoring.rs`/`vector/mmr.rs` are near-empty shims over crate `WeightProfile`/`retrieval::mmr`. Host keeps the three agent tools. Done. | +| `memory_tools/` | 9 / 1.4k | 3 | W7 flipped: types/store/prompt are shims over `tool_memory::{types,store,render}`. Host keeps `capture.rs` (`PostTurnHook`) + put/list tools. Done. | +| `memory_archivist/` | 1 / 44 | 1 | Pure W7 shim over `memory::archivist`. Done. | +| `memory_conversations/` | 2 / 0.9k | 2 | W7 shim over `memory::conversations`; host keeps `bus.rs` event-bus glue (+ a `tinychannels` legacy-message conversion). Done. | +| `memory_diff/` | 7 / 2.0k | 11 uses | W7 flipped: engine crate-owned; host keeps RPC/tools + `DomainEvent::MemoryDiff*` publishing. Done. | +| `memory_goals/` | 6 / 0.9k | 4 uses | W7 flipped: types/store crate-owned (byte-identical `MEMORY_GOALS.md`); host keeps agent wiring. Done. | +| `memory_sources/` | 16 / 5.0k | 12 uses | Product registry (config-persisted connectors) with crate readers underneath. Host by design. | +| `memory_sync/` | **73 / 17.7k** | 5 uses | **The largest remaining mass.** Host Composio provider pipelines (gmail/slack/github/notion/linear/clickup) still run host-side while the crate's `memory::sync` ships the same provider engine. W-SYNC seam (`sync.rs`, 679) is landed; the host flip (W-SYNC.3) has not happened. See WP-2. | +| `embeddings/` | 13 / 3.1k | **0** | Bridges **tinyagents** (9 files), not tinycortex. Post-#58, the host per-provider adapters duplicate crate `harness::embeddings` providers. See WP-4. | +| `agent_memory/`, `subconscious/` (memory parts), `learning/` (memory parts) | — | 0 | Host product consumers of the memory stack; no crate duplication. Stay. | + +Consumer counts (host memory tiers are deeply embedded and *stay* host, so +these are context, not deletion blockers): `memory::` 166 files, +`memory_store::` 132, `memory_tree::` 67, `memory_queue::` 18, +`memory_tools::` 11, `memory_search::` 3. + +### 4.2 The seam — `src/openhuman/tinycortex/` (11 files, ~3.2k LOC) + +`config.rs`, `embeddings.rs`, `chat.rs`, `summariser.rs`, `ingest.rs`, +`seal.rs`, `sync.rs` (679), `queue_driver.rs` (1,007), `persona.rs` (446), +`parity.rs` (`#[cfg(test)]` format pins), `mod.rs`. Healthy and intentionally +host-owned. Two findings: + +- **The seam is not the funnel.** `mod.rs` re-exports crate types + (`MemoryEntry`, `MemoryTaint`, `RecallOpts`, …) as the intended + type-unification funnel, but **zero** files import `crate::…::tinycortex::` + re-exports — every memory domain imports `tinycortex::memory::*` directly. + Either convention works; pick one and record it (WP-5). Direct crate imports + are simpler and match how `tinyagents` is consumed; if so, shrink the facade + instead of promoting it. +- **Stale internal doc:** `mod.rs:28-30` names sibling adapters `sinks.rs` and + `bus.rs` that were never created (their roles landed as + `ingest.rs`/`seal.rs`/`sync.rs`). `queue_driver.rs:21-26` still says "This + brick is additive: nothing is flipped yet" — the flip landed in #4781. + +Coupling is bidirectional by design (seam reaches back into +`memory_store`/`memory_tree`/`memory_sync` to do host work inside crate +callbacks); not a defect, but `queue_driver.rs`'s 11 host imports will thin as +WP-1/WP-3 shims retire. + +### 4.3 Broken/dangling references (resolved in WP-0) + +| Resolved issue | Resolution | +| --- | --- | +| Missing TinyAgents companion-spec link | Replaced with the existing July 22 migration plan. | +| Missing standalone deletion-ledger link | Replaced with §2 of the TinyCortex migration spec. | +| Wrong agent-harness documentation path | Pointed at the architecture subdirectory. | +| Never-landed golden-workspace generator | Removed after the post-cutover parity decision descoped synthetic pre-cutover fixtures. | +| Wrong raw-coverage test path | Corrected to the `tests/raw_coverage/` location. | +| Stale line anchors `Cargo.toml:82`, `src/openhuman/mod.rs:130` | `tinycortex-migration-spec.md` §0.4 (actual: `Cargo.toml:116`, `mod.rs:140`) | +| Never-created seam sibling names | Replaced with the adapters that actually landed. | +| Developer-specific absolute path in crate docs | Replaced with repository-relative paths. | + +Also stale-in-place: all five docs' status headers (pre-#4794), the drift +ledger's pin anchors, and the crate migration doc's claim that "TinyCortex +will not own memory sync" (superseded by the plan's §8 sync-inclusive +amendment and the crate's shipped `sync` module — the *scheduler/credentials/ +RPC* stay host, the *engine* is crate; restate the boundary in one place). + +### 4.4 Test inventory + +- **In-crate (done):** engine coverage was ported by #4820; the crate runs its + own suite offline (wiremock Composio mock; live tests env-gated). W8's CI + wiring exists: `ci-lite.yml:859` runs the vendor suite on submodule-pointer + changes — **but with `--features git-diff,sync` only; `persona` is untested + in host CI while enabled in the host build.** +- **Dies with WP-1:** `memory_store/unified/*_tests.rs` (~3.7k LOC: + `documents_tests` 1,275, `query_tests` 1,000, `profile_tests` 763, + `segments_tests` 393, `events_tests` 287) go with the unified tier; port any + still-unique assertions to the per-kind crate backends first. +- **Retargets with WP-2:** the memory_sync suites (389 inline tests across 35 + files + provider `*_tests.rs`: github 655, gmail 301+354, slack 180, linear + 172, clickup 155, notion 119) — post-flip, provider *engine* behavior is + crate-tested; host keeps orchestration/bus/config tests. Integration: + `memory_sync_pipeline_e2e.rs` (570), `memory_golden_parity_e2e.rs` (275), + `raw_coverage/memory_sync_*` retarget to the crate-backed path. +- **Golden parity harness:** comparators 1 & 5 green; 2/3/4 TODO; + `frontmatter_parity` Layer-1 asserter pending; fixture generator script + missing. Decide finish-vs-descope in WP-0 (post-cutover, parity pins matter + mainly for `MemoryTaint`/on-disk stability — keep those, consider dropping + the rest). +- **Stays host:** `json_rpc_e2e.rs` memory sections, `memory_sources_e2e` + (837), `memory_tree_summarizer_e2e` (583), `memory_roundtrip_e2e`, + `memory_graph_sync_e2e`, `memory_fast_retrieve_e2e`, `autocomplete_memory_e2e`, + `memory_artifacts_e2e`, the 15 `raw_coverage/memory_*` modules, and all + subconscious/learning suites. + +--- + +## 5. The plan — five work packages + +Same conventions as the tinyagents companion plan: WP-0 first (hygiene, +unblocks honest review), then largely independent packages; drift-ledger rows +before deletions; failing-before/passing-after regression tests; small +validated commits on feature branches per submodule. + +### WP-0 — Version/tag + CI + docs housekeeping (no engine behavior change) + +1. **Establish the tag discipline the Cargo comment promises:** tag + `vendor/tinycortex` (e.g. `v0.1.1` at the current release point, or cut + `v0.2.0` at HEAD), and record the bump workflow the plan prescribed + (`chore(vendor): bump tinycortex …` — zero such commits exist in host + history). Alternatively amend the comment to describe reality + (main-tracking submodule). Either is fine; the current state (comment + demands tags, none exist) is the worst option. +2. **Align the nested tinyagents:** when the tinyagents plan's WP-0 bumps + `vendor/tinyagents` to v2.1.0, bump `vendor/tinycortex/vendor/tinyagents` + in lockstep and update tinycortex's `tinyagents = "2"` if needed; then + evaluate retiring the `[patch."…senamakel/tinyagents"]` block and its + obsolete "while tinyagents #58 is pending" comment (#58 is merged). +3. **Close the persona CI hole:** add `persona` to the vendor-suite features + in `.github/workflows/ci-lite.yml:859` (and confirm the crate's own + feature-matrix covers `persona`, which today it does not list either). +4. **Docs refresh:** update the five docs' status headers to post-#4794/#4820 + reality; update drift-ledger anchors (pin `daaaf6ba`+tag, note #4794/#4820/ + #4863); document the `persona` feature + seam (currently invisible in all + five docs); restate the sync ownership boundary once (engine=crate, + scheduler/credentials/RPC/bus=host) and fix the crate-side migration doc's + contradictory "will not own sync" line + its absolute local path. +5. **Fix the §4.3 broken references**, including the two code-adjacent ones: + correct the seam module inventory and the queue-driver's stale pre-flip + description. +6. **Golden-parity decision:** finish comparators 2/3/4 + `frontmatter_parity` + + the fixture script, **or** descope to the security-relevant pins + (`MemoryTaint` byte-identity, chunk/vector on-disk format) with the + decision recorded in the parity checklist. + +**Exit:** pins/tags/CI/docs agree with the tree; `grep` for the phantom paths +comes back clean. + +### WP-1 — Retire `memory_store/unified/` (the "staging for removal" tier) + +1. Inventory the five remaining unified facets (documents, query, segments, + events, profile — ~5k LOC) against their per-kind crate backends; flip + callers facet-by-facet (the W3 sub-store pattern, already proven for + chunks/content/vectors/kv/entities/trees/safety). +2. The blocker is **G1** (`sqlite_conn()` escape hatch, ~312 refs at audit + time): re-audit the current count, then either land the crate-side access + the gap audit sketched or explicitly re-scope G1 to the host-retained + namespace-document tables and record it. +3. Decide the fate of the 10-table host-retained `UnifiedMemory` + namespace-document tier: keep host (spec's standing decision) but move it + out of `unified/` into a clearly-named home so "staging for removal" can + actually be removed. +4. Tests: port unique assertions from the ~3.7k LOC of `unified/*_tests.rs` + to per-kind backends (crate-side where engine behavior, host-side where + glue), then delete with the code. + +**Exit:** `memory_store/unified/` gone or reduced to the renamed host tier; +`memory_store/` ≈ 9–10k LOC of glue + host tier; `memory_roundtrip_e2e` + +`memory_golden_parity_e2e` + full mock-suite green. + +### WP-2 — W-SYNC.3: flip `memory_sync/` onto the crate sync engine + +The seam (`sync.rs`: `ExternalSourceReader`, `SkillDocSink`, +`LocalDocumentSink`, `SyncStateStore`, `SyncEventSink`) is landed; the crate +ships the Composio engine incl. per-provider modules. Remaining: + +1. Close the **D4.1–D4.4** drift rows (ledger requires this before the flip). +2. Flip provider-by-provider (gmail first — the crate already carries the + Gmail 413-page fix #73), keeping host-side: scheduler/periodic trigger, + credentials, `config.toml` source registry (`memory_sources/`), event-bus + publishing (`ChannelMessageReceived` et al.), RPC/status surface, and + redaction/`source_scope` gating. +3. Dedupe post-processing: host `providers/*/post_process*` vs crate provider + modules — upstream generic normalization, keep host product policy. +4. Tests per §4.4: engine behavior moves to crate suites (wiremock Composio), + host keeps orchestration/bus/e2e; `memory_sync_pipeline_e2e` retargets. + +**Exit:** `memory_sync/` shrinks from 17.7k toward ~6–8k LOC of +orchestration/product glue; D4 rows CLOSED; no duplicated provider parsing. + +### WP-3 — Embedding-provider dedupe (lands in *tinyagents*, coordinated here) + +Two host clusters now duplicate `tinyagents::harness::embeddings` (post-#58: +openai/cohere/voyage/ollama/cloud + rate-limit/retry-after): + +1. `src/openhuman/embeddings/` adapters (`openai/voyage/cohere/ollama/ + cloud_adapter.rs`, `provider_trait.rs`) → construct crate + `EmbeddingModel`s directly; host keeps `rpc.rs` (1,479), `schemas.rs`, + `catalog.rs`, `factory.rs` (config/BYOK selection, #4056 dimension + gating). +2. `memory_tree/score/embed/{openai_compat,ollama,cloud,factory}.rs` (~1.9k) + → same crate models via the seam's `SeamEmbedder`/`EmbedderBridge`. + +Because tinycortex consumes the same `EmbeddingModel` trait, this single +change serves both crates — it is the concrete payoff of the W-EMB decision. +Sequence after WP-0's tinyagents version alignment. + +**Exit:** one embedding-provider implementation per provider across the +workspace; `embeddings_rpc_e2e` (822) + `ollama_embeddings_fallback_e2e` (234) +green against the crate-backed path. + +### WP-4 — Shim retirement + funnel decision + +1. **Funnel policy:** adopt direct `tinycortex::memory::*` imports as the + convention (matches reality and the tinyagents precedent), shrink the seam + `mod.rs` facade to the types host code genuinely re-brands + (`MemoryTaint` stays pinned by parity tests) — or the reverse; either way, + record it in the spec and stop carrying both. +2. Retire compatibility shims whose consumers can move: `memory_tree/tools.rs` + (points at `memory`), `memory_store/trees` legacy paths, the + `memory_queue`→`memory::jobs` re-export, `memory_archivist` (44-line shim — + move its 2 call sites and delete the domain), and the W5/W7 near-empty + shims once import counts hit zero (`memory_search/scoring.rs`, + `vector/mmr.rs`, `memory_tools/{types,store}.rs`). +3. Thin `queue_driver.rs` (1,007) as WP-1 removes its `memory_store` + reach-backs. + +**Exit:** shim count measurably down (each deletion = one commit with its +import-migration); seam ≤ ~2.5k LOC; no module whose entire body is a +re-export except deliberate facades recorded in the spec. + +### WP-5 — Exit gate + +- Full suite: `scripts/test-rust-with-mock.sh` (incl. the 15 + `raw_coverage/memory_*` modules), vendor suite + `cargo test --manifest-path vendor/tinycortex/Cargo.toml --features + git-diff,sync,persona`, slim disabled build (`--no-default-features + --features tokenjuice-treesitter` — memory domains are ungated but the + standing repo rule applies), `pnpm rust:check`. +- Drift ledger: D4 CLOSED; new rows for every WP-1/WP-2 deletion; the spec's + §2 deletion-ledger skeleton filled in with actuals. +- Docs: the five tinycortex docs + `src/openhuman/memory*/README.md` files + + `gitbooks/developing/architecture.md` reflect the post-flip reality; this + plan stamped done. + +### Execution record (2026-07-22) + +| Package | Result | +| --- | --- | +| WP-0 | TinyAgents unified at 2.1 across OpenHuman, TinyCortex, and TinyFlows; obsolete fork patch removed; TinyCortex CI covers `persona`; docs and phantom references repaired. TinyCortex intentionally tracks reviewed upstream commits rather than nonexistent tags. | +| WP-1 | G1 count is zero. The ten-table namespace/document product tier is host-owned, so the complete `unified/` directory was renamed to `namespace_store/` instead of deleted as engine duplication. Its 153 targeted tests pass. | +| WP-2 | The live default path was already TinyCortex-backed. D4 is CLOSED; dead Gmail duplicate removed; remaining provider task/profile projections explicitly classified as product policy. Provider tests pass (301). | +| WP-3 | Concrete provider transports deduplicated into TinyAgents. The memory tree now uses a thin `ProviderEmbedder`; Ollama's 8k context/batch and missing-model guidance moved upstream before the host client was deleted. TinyAgents embedding tests pass (38); host library check passes. | +| WP-4 | Archivist, search scoring/MMR, tool-memory type/store, tree-tool, jobs-alias, and unused seam type facades retired. Direct crate imports are canonical. Seam production code is 2,229 LOC (below the 2.5k exit target). | +| WP-5 | Local focused validation is recorded above; the slim `--no-default-features --features tokenjuice-treesitter` build also passes. [CI Full run 29925645209](https://github.com/senamakel/openhuman/actions/runs/29925645209) is green: core quality and full tests, TinyCortex, Tauri, frontend, mock-backend Rust E2E, Playwright, three desktop builds, every launched desktop shard, and the final gate. Two first-attempt Linux jobs ended without uploaded logs; rerunning only failed jobs passed the Linux build, Rust integration suite, and all eight Linux shards. | + +--- + +## 6. Risks and standing gotchas + +- **`MemoryTaint` is security-critical.** The type is proven byte-identical + host↔crate and fails closed to `ExternalSync`; the parity pins in + `tinycortex/parity.rs` are the regression guard — whatever WP-0 decides + about the golden harness, **keep these**. +- **Redaction and `source_scope` gating stay host.** WP-2 must not let crate + sync code become a path around host redaction or the source allowlist — + review the `SyncEventSink`/`LocalDocumentSink` implementations for every + provider flipped. +- **On-disk compat:** chunk/vector formats, `MEMORY_GOALS.md` byte-identity, + and the P5 fresh-DB divergence (3 legacy inline-embedding columns in + `mem_tree_chunks`) — migrations of user data must be forward-only and + tested against a pre-migration workspace fixture. +- **MIT vs GPL:** tinycortex is MIT (tinyagents is GPL-3.0) — moving host code + *down* into tinycortex relicenses it; fine for our own code, but keep the + "no product policy / no keys / nothing openhuman-branded as API" rule, and + don't move anything derived from GPL-only sources into the MIT crate. +- **Two Cargo worlds + nested submodule:** vendor bumps now touch up to three + lockfiles (root, `app/src-tauri`, and tinycortex's own) and two submodule + pointers (`vendor/tinycortex`, `vendor/tinycortex/vendor/tinyagents`). CI + clones with `submodules: recursive` for the vendor suite — verify after any + bump. +- **Event-bus surface:** `memory_sync` publishes 7 distinct `DomainEvent` + variants; the WP-2 flip must keep publishing them from host callbacks (the + crate has no event bus by design). + +--- + +## 7. Quick reference — disposition of every audited module + +| Module | Verdict | +| --- | --- | +| `memory_store/unified/` | DELETE after per-kind flips; host tier re-homed (WP-1) | +| `memory_store/` sub-stores, `factories.rs` | DONE (crate-backed) / STAYS (host glue) | +| `memory_sync/` provider engines | FLIP to crate `memory::sync` (WP-2) | +| `memory_sync/` scheduler, bus, RPC, credentials | STAYS | +| `embeddings/` per-provider adapters | REPLACE with `tinyagents::harness::embeddings` models (WP-3) | +| `embeddings/` rpc/schemas/catalog/factory | STAYS | +| `memory_tree/score/embed/` providers | REPLACE via seam + crate models (WP-3) | +| `memory_tree/` engine files | DONE (crate-backed); RPC/CLI/bus/health STAY | +| `memory_queue/`, `memory_search/`, `memory_tools/`, `memory_diff/`, `memory_goals/`, `memory_conversations/` | DONE — retire residual shims when import counts reach zero (WP-4) | +| `memory_archivist/` | DELETE the 44-line shim after moving 2 call sites (WP-4) | +| `memory/` (orchestration/RPC/tools/tree-policy) | STAYS | +| `memory_sources/` | STAYS (product registry over crate readers) | +| `agent_memory/`, `subconscious/`, `learning/` | STAYS (consumers, no duplication) | +| Seam `src/openhuman/tinycortex/` | STAYS, shrinks; fix stale module docs (WP-0), thin `queue_driver.rs` (WP-4) | +| Golden-parity harness | FINISH or DESCOPE by decision (WP-0); `MemoryTaint`/format pins KEEP | diff --git a/docs/tinycortex-migration-spec.md b/docs/tinycortex-migration-spec.md index 0b8711438..7bb8c8298 100644 --- a/docs/tinycortex-migration-spec.md +++ b/docs/tinycortex-migration-spec.md @@ -1,11 +1,10 @@ # TinyCortex Memory Migration — Spec (Phase 0.5 / 0.6) -**Status:** Phase 0 baseline **+ execution underway** — #59 (native-dep alignment) merged and the dep -is active (§0.4); W1 seam + W2 type re-export + W3-chunks partial landed; **drift closure COMPLETE** -(**D3** via #59, **D2** via tinycortex#63, **D1** via tinycortex#64; gitlink `a8e10f7`) — so **W4 -(queue) and W7 (conversations) are unblocked**. Anchors the migration to exact reviewed SHAs and ties -together the drift, gap, and parity ledgers. Modeled on `docs/tinyagents-migration-spec.md` + its -deletion ledger. +**Status:** Post-engine-cutover reference. W1–W8 and the crate-owned engine test +port landed in OpenHuman #4794/#4820, with persona/coding-session ingest in +#4863. The remaining host consolidation is tracked by +[`tinycortex-migration-plan-2026-07-22.md`](tinycortex-migration-plan-2026-07-22.md). +This document retains the ownership contract and deletion ledger detail. **Companion plan:** [`tinycortex-memory-migration-plan.md`](tinycortex-memory-migration-plan.md) **Ledgers:** [`tinycortex-drift-ledger.md`](tinycortex-drift-ledger.md) · @@ -19,7 +18,10 @@ deletion ledger. | `tinyhumansai/openhuman` | `7850cf363559bcbb7ba688cbc4fccdb6bd9ce754` | host audit base (`main`, 2026-07-04) | | `tinyhumansai/tinycortex` | `d1a8c7be2babc8fff7a72ed93861f459f3d6fa58` | crate audit base (v0.1.1) | | `tinyhumansai/tinycortex` | `33dda943053e61ef585fc39647cf1854344b6323` | audit base **+ #59** (native-dep alignment, §0.4) merged | -| `tinyhumansai/tinycortex` | `a8e10f7dd8ebdb9b0905e1380fefcc6bf5a65207` | **current pinned gitlink** — **+ #63/#64** (D2/D1 drift closure) merged; all drift rows CLOSED | +| `tinyhumansai/tinycortex` | `a8e10f7dd8ebdb9b0905e1380fefcc6bf5a65207` | historical cutover gitlink — **+ #63/#64** (D2/D1 drift closure) merged | +| `tinyhumansai/openhuman` | `5b8a9f269` | 2026-07-22 consolidation audit base | +| `tinyhumansai/tinycortex` | `daaaf6ba5f02635c08deae2b2b2ed7fcc8c06b6a` | 2026-07-22 reviewed gitlink; no upstream tags exist, so the host tracks reviewed main commits | +| `tinyhumansai/tinycortex` | `7b4b115` | current migration branch: TinyAgents 2.1 alignment + corrected ownership docs | Port line (derived by content, §0.1): **after 2026-06-25, before 2026-06-28** for engine features. @@ -72,14 +74,14 @@ ledger. Re-export covers the data types (`MemoryEntry`, `MemoryCategory`, `Memor | **rusqlite alignment** | ✅ **resolved & merged (#59).** Crate was pinned `0.32` (bundled), host pins `=0.40.0` (bundled). Two `links = "sqlite3"` = hard Cargo error. Fixed in #59 (bump to `0.40` + `usize`→`i64`/`try_from` — the same sweep that closed drift **D3**). | | **git2 alignment** | ✅ **resolved & merged (#59).** Crate was pinned `0.19`, host `0.21` (vendored-libgit2). Two `links = "git2"`. Fixed in #59 (bump to `0.21` + API deltas: `Tag::message`, `StringArray::Iter`, `Buf::as_str`). | | Crate compiles with aligned deps | ✅ `cargo check --all-targets` clean; 38 diff/checkpoint tests pass. | -| **Host root world compiles with dep active** | ✅ `cargo check --manifest-path Cargo.toml --lib` **exit 0** with `tinycortex = "0.1"` active (`Cargo.toml:82`) + submodule at `33dda94`. **No `multiple packages link to native library` error** — one bundled SQLite + one libgit2 confirmed. **Now landed** (post-#59-merge): the `[dependencies]` line and gitlink are committed on the working branch, not reverted. Re-verified 2026-07-09. | +| **Host root world compiles with dep active** | ✅ `cargo check --manifest-path Cargo.toml --lib` **exit 0** with `tinycortex = "0.1"` active (`Cargo.toml:116`) + submodule at `33dda94`. **No `multiple packages link to native library` error** — one bundled SQLite + one libgit2 confirmed. **Now landed** (post-#59-merge): the `[dependencies]` line and gitlink are committed on the working branch, not reverted. Re-verified 2026-07-09. | | Host `app/src-tauri` world | to verify in W1 (separate Cargo world / lockfile). | | `GGML_NATIVE=OFF` macOS ARM | to verify on a macOS runner in W1 (no macOS host here). | **Activation landed (post-#59).** The native-dep alignment merged upstream as **#59** and the host gitlink was bumped to `33dda94`, so — per the submodule rule (bump only to a **merged** SHA) — W1's activation is now in place: `[dependencies] tinycortex = "0.1"` is active in the root world -(`Cargo.toml:82`), the seam (`src/openhuman/tinycortex/`) is wired (`src/openhuman/mod.rs:130`), and +(`Cargo.toml:116`), the seam (`src/openhuman/tinycortex/`) is wired (`src/openhuman/mod.rs:140`), and `cargo check --lib` is **exit 0**. Remaining §0.4 follow-ups: the `app/src-tauri` world and the `GGML_NATIVE=OFF` macOS-runner check still verify in W1 (see rows above). @@ -117,7 +119,7 @@ activation is now in place: `[dependencies] tinycortex = "0.1"` is active in the coexist in the shared DB but **do not move**: `memory_docs`, `graph_global`, `graph_namespace`, `episodic_log` (+ `episodic_fts` + triggers), `event_log` (+ `event_fts`, `event_embeddings`, triggers), `conversation_segments`, `segment_embeddings`, `vector_chunks`, `user_profile`. - These live in `memory_store/unified/{init,fts5,events,segments,profile}.rs` and remain host — the + These live in `memory_store/namespace_store/{init,fts5,events,segments,profile}.rs` and remain host — the crate is the **primitive substrate**, not a drop-in for the whole DB. - **Content-store host surfaces the crate explicitly excludes:** `content::wiki_git`, `content::obsidian`, `content::obsidian_registry`. @@ -128,8 +130,11 @@ activation is now in place: `[dependencies] tinycortex = "0.1"` is active in the `embeddings.rs` (`EmbeddingBackend`/`Embedder`), `chat.rs` (`ChatProvider`/`Summariser`×2/ `EntityExtractor`/`GoalsGenerator`), `queue_driver.rs` (`QueueDelegates` + tokio worker loop + Sentry/bus), `config.rs` (`Config`→`MemoryConfig`), `sinks.rs` (`TreeJobSink`/`TreeLeafSink`/ -`SnapshotItemSource`/`EntityOccurrenceIndex`), `bus.rs` (engine outcomes → `DomainEvent`), -`mod.rs` (facade re-exports + boundary doc). All 17 W1 seam traits confirmed present (§0.2). +`SnapshotItemSource`/`EntityOccurrenceIndex`), `sync.rs` (sync outcomes → `DomainEvent`), +`mod.rs` (adapter namespace + compatibility re-exports + boundary doc). New +engine consumers import `tinycortex::memory::*` directly; the seam owns +implementations rather than serving as a second type funnel. All 17 W1 seam +traits confirmed present (§0.2). **Later seam additions (amended 2026-07-09):** @@ -144,7 +149,7 @@ Sentry/bus), `config.rs` (`Config`→`MemoryConfig`), `sinks.rs` (`TreeJobSink`/ --- -## 2. Deletion ledger (skeleton) +## 2. Deletion ledger Every legacy engine file is deleted only when its module's **drift row is closed**, its **gaps are resolved**, and the **golden-workspace parity harness is green** for its flip. Counts from the host @@ -152,7 +157,7 @@ audit SHA. | Legacy module | Files (test files) | Deletes in | Preconditions | | --- | --- | --- | --- | -| `memory_store/` | 66 (11) | W3 | drift **D3** closed; gap **G1** (escape hatch) migrated to `with_connection`; parity P3/P5/P11/P12 green; `unified/` tier re-homed as host-retained (kept, not deleted) | +| `memory_store/` | 66 (11) | W3/WP-1 | drift **D3** closed; gap **G1** migrated completely (zero `sqlite_conn()` call sites at the 2026-07-22 audit); parity P3/P5/P11/P12 green; namespace tier re-homed to `namespace_store/` as host-retained | | `memory_tree/` | 65 (7) | W5 | gaps **G3** (seal-embed), **G6** (2× Summariser) resolved; `source_scope` allowlist re-verified; parity P7/P11 green; `health/` + `tree_policy.rs` kept host (G5) | | `memory_queue/` | 10 (1) | W4 | drift **D2** closed (predicate upstreamed); job payload_json parity (P4/P9); host worker loop + Sentry/degraded wiring kept host | | `memory_conversations/` | 7 (1) | W7 | drift **D1** closed; `bus.rs` kept host | @@ -168,10 +173,19 @@ audit SHA. | `memory_search/` | 8 (0) | W5 | `vector`/`scoring` → crate `retrieval`/`score`; `tools/` kept host | | `memory/ingest_pipeline.rs` internals | (thin entry points kept) | W6 | `ingest_chat`/`ingest_document_with_scope` signatures unchanged; 11 call sites untouched | +### 2026-07-22 consolidation actuals + +| Package | Actual result | +| --- | --- | +| WP-1 namespace tier | `memory_store/unified/` removed and re-homed byte-for-byte as `memory_store/namespace_store/`; G1 re-audit found zero `sqlite_conn()` call sites. The retained ten-table product store explains why total `memory_store` remains 17.7k LOC rather than the plan's speculative 9–10k target. | +| WP-2 sync | The default provider `sync()` and `run_connection_sync` already call the TinyCortex engine. Deleted the dead host Gmail sync parser; renamed GitHub/Notion/Linear/ClickUp product projections from `sync.rs` to `normalization.rs`. D4.1-D4.4 are CLOSED. The retained 17.2k LOC is schedulers, bus/RPC, action tools/catalogs, credentials, profiles, post-processing, and product task projections; 6.9k LOC is provider catalogs/tools/normalization/profile/RPC alone. | +| WP-3 embeddings | Deleted host OpenAI, Cohere, Voyage, general Ollama, memory-tree cloud, and memory-tree Ollama provider implementations (746 production LOC) plus their obsolete 828-line raw-coverage suite. Provider transport now has one implementation in TinyAgents; OpenHuman retains selection and credential/privacy adapters. | +| WP-4 shims | Deleted `memory_archivist`, `memory_search::{scoring,vector}`, `memory_tools::{types,store}`, `memory_tree::tools`, and the `memory::jobs` alias. Removed the unused TinyCortex type facade; direct `tinycortex::memory::*` imports are the convention. The seam is 2,229 pre-test LOC. | + **Kept host (never deleted):** `memory/{ops,schemas,schema,read_rpc,tools,query,tree_source, ingestion,util}`, `memory/{global,source_scope,chat,sync,preferences,remember,tree_policy,rpc_models, traits(→re-exports)}.rs`, `memory_sync/`'s host-retained shell only (schedulers `periodic.rs`, -`bus.rs` subscribers, RPC registration — the engine moves in W-SYNC, plan §8), `memory_store/unified/*` (the namespace-document +`bus.rs` subscribers, RPC registration — the engine moves in W-SYNC, plan §8), `memory_store/namespace_store/*` (the namespace-document tier), `memory_store/content/{wiki_git,obsidian,obsidian_registry}`, `memory_tree/health/`, and the new `src/openhuman/tinycortex/` seam. diff --git a/docs/tinycortex-parity-checklist.md b/docs/tinycortex-parity-checklist.md index bd91c6568..e1af5f08d 100644 --- a/docs/tinycortex-parity-checklist.md +++ b/docs/tinycortex-parity-checklist.md @@ -1,5 +1,13 @@ # TinyCortex Data-Format Parity Checklist (Phase 0.3) +**Status (2026-07-22):** The engine cutover and crate test port are complete. +The proposed pre-migration golden-fixture generator and differential +comparators 2–4 are deliberately **descoped**: there is no trustworthy +pre-cutover binary/fixture left to generate them from after cutover. Retained +gates are the security-critical `MemoryTaint` byte/serde pins, deterministic +chunk IDs, vector encoding/signature, content paths, schema composition, +idempotent reopen, and the host public-surface E2E suites. + **Purpose.** Existing user workspaces must open **unchanged** after every cutover flip. This checklist enumerates every on-disk format shared between the host engine and TinyCortex, records the audit result, and specifies the **golden-workspace parity harness** that gates W3, W5, W6. @@ -7,7 +15,8 @@ records the audit result, and specifies the **golden-workspace parity harness** **Hard rule (plan §0.3/§6):** any mismatch is fixed **upstream in tinycortex**, never papered over with a host shim. -Anchors: host `7850cf363` · tinycortex `d1a8c7be` (v0.1.1). +Anchors: historical audit host `7850cf363` · crate `d1a8c7be`; consolidation +base host `5b8a9f269` · reviewed crate `daaaf6ba` · migration branch `7b4b115`. ## Ownership tiers in the shared workspace (key finding) @@ -28,10 +37,10 @@ A user workspace's `chunks.db` (and content vault) holds **two tiers**: `event_ai/ad/au` triggers), `conversation_segments`, `segment_embeddings`, `vector_chunks`, `user_profile` **(10 tables + FTS + triggers)**. - These live in `src/openhuman/memory_store/unified/{init,fts5,events,segments,profile}.rs`. - **W3 must keep the host creating/reading these in the same DB the crate now manages** — + These live in `src/openhuman/memory_store/namespace_store/{init,fts5,events,segments,profile}.rs`. + **The host continues creating/reading these in the same DB the crate now manages** — the crate's `chunks::with_connection` opens the shared handle; host `UnifiedMemory` schema - init runs alongside the crate's. Parity requirement: crate schema init and host `unified` + init runs alongside the crate's. Parity requirement: crate schema init and host namespace-store init must **compose without collision** on both fresh and existing DBs. ## Parity results by format dimension @@ -93,17 +102,15 @@ The core mechanism from plan §0.3: **one on-disk workspace, opened by both engi > tables) and the host `UnifiedMemory` tier (10 tables) **coexist without collision** (P3/P5/P11/P12), > and that re-running the flow adds/drops no tables (comparator 5). Still TODO: `vectors`/`store_meta`/ > `kv_*` (created by the chunk/embed pipeline, need a widened ingest flow), the seeded golden fixture + -> `scripts/gen-golden-workspace.sh`, and comparators **2** (recall/retrieval snapshot), **3** (tree -> read), **4** (byte-compare vault) — these require a populated, sealed fixture and the W5 retrieval -> surface, so they land alongside the W3/W5 flips. +> Comparators **2** (recall/retrieval snapshot), **3** (tree read), and **4** +> (byte-compare vault) were never landed before cutover and are now descoped by +> the decision above. Comparator 1 (schema composition) and comparator 5 +> (idempotent reopen) remain active. -**Fixture.** Check in a small, deterministic `tests/fixtures/golden-workspace/` produced by the -*pre-migration* build: a real `chunks.db` + content vault + diff `.git`, seeded via a fixed script -(`scripts/gen-golden-workspace.sh`) with: a handful of chat + document + email sources across ≥2 -namespaces, ingested + scored + sealed to ≥2 tree levels, some entities/edges, a few queue jobs in -mixed states, and both tiers populated (episodic/event/segment rows present). Store the **generator -script** and a **manifest** (expected chunk-ids, summary paths, recall snapshots) so the fixture is -regenerable and reviewable, not an opaque blob. +**Fixture decision.** Do not synthesize a fixture and label it “pre-migration” +after the fact. Compatibility is guarded by the retained format pins, schema +composition/idempotency test, and real host E2E fixtures. A future format +migration may add a versioned fixture captured before that migration starts. **Comparators (read-only, both engines open the SAME copied workspace):** @@ -124,7 +131,8 @@ regenerable and reviewable, not an opaque blob. **Wiring.** Runs under `pnpm test:rust` (host-side, counts toward coverage) and as a dedicated `tests/memory_golden_parity_e2e.rs`. The existing crate-level integration tests -(`tests/memory_roundtrip_e2e.rs`, `memory_tree_sync_deep_raw_coverage_e2e.rs`) act as the +(`tests/memory_roundtrip_e2e.rs`, +`tests/raw_coverage/memory_tree_sync_deep_raw_coverage_e2e.rs`) act as the "public-surface still green" guard (plan §5.1); this harness adds the **differential** guard that a flip preserves *existing* data, not just that the API still functions. @@ -154,13 +162,13 @@ Engine tests now run at their ownership boundary rather than through OpenHuman r surfaces, agent-tool response post-processing, source registry side effects, and the security-critical `MemoryTaint` seam. - OpenHuman CI now runs `cargo test --manifest-path vendor/tinycortex/Cargo.toml --features - git-diff,sync` when the submodule pointer changes and in the reusable full Rust suite. This is + git-diff,sync,persona` when the submodule pointer changes and in the reusable full Rust suite. This is required because Cargo does not run dependency test targets while testing `openhuman`. The focused local verification commands are: ```bash -cargo test --manifest-path vendor/tinycortex/Cargo.toml --features git-diff,sync +cargo test --manifest-path vendor/tinycortex/Cargo.toml --features git-diff,sync,persona cargo test --test raw_coverage_all memory_sync -- --test-threads=1 cargo test --test memory_sync_pipeline_e2e --test memory_artifacts_e2e \ --test memory_golden_parity_e2e --test memory_roundtrip_e2e --test memory_sources_e2e diff --git a/gitbooks/developing/architecture.md b/gitbooks/developing/architecture.md index e7f48a02f..cb9cea6dc 100644 --- a/gitbooks/developing/architecture.md +++ b/gitbooks/developing/architecture.md @@ -291,10 +291,10 @@ Every layer is async and non-blocking. The Rust core processes thousands of conc ## Vendored crate family & recent shifts -Core subsystems are being re-platformed onto published `tiny*` crates, vendored as git submodules under `vendor/` (`tinyagents`, `tinyflows`, `tinycortex`, `tinychannels`, `tinyjuice`, `tinyplace`) so crate changes can be tested in-tree before publishing. The major ongoing shifts: +Core subsystems run on published `tiny*` crates, vendored as git submodules under `vendor/` (`tinyagents`, `tinyflows`, `tinycortex`, `tinychannels`, `tinyjuice`, `tinyplace`) so crate changes can be tested in-tree before publishing. The major ownership boundaries are: - **Agent engine on tinyagents** — every agent turn runs through the `tinyagents` crate harness via the seam in `src/openhuman/tinyagents/`; see [Agent Harness](architecture/agent-harness.md). -- **Memory on tinycortex** — memory modules (`memory_diff`, `memory_conversations`, `memory_queue`, …) are being deleted or shimmed onto the `tinycortex` crate engine (W7 migration; #4785–#4788). +- **Memory on tinycortex** — the generic store/tree/queue/retrieval/sync engine is crate-owned. OpenHuman keeps RPC, tools, scheduling, credentials, security/event policy, worker orchestration, and the host namespace-document store; `src/openhuman/tinycortex/` implements those seams. Concrete embedding transports are shared through `tinyagents::harness::embeddings`. - **Inference on the crate ModelRouter** — host workload-tier model routing and cloud provider slugs now use the crate-native `ModelRouter`/`OpenAiModel` (#4782, #4783). - **Hosted-only brain** — the client-local orchestration graph engine (`src/openhuman/orchestration/graph/`) was retired (#4738); the client is a thin hosted-brain participant (pushers, effect/tool executors, wire allowlist — #4725) surfaced in the `/orchestration` and `/brain/tinyplace-orchestration` routes. diff --git a/src/core/observability.rs b/src/core/observability.rs index 1f3ff24d3..d3f862a93 100644 --- a/src/core/observability.rs +++ b/src/core/observability.rs @@ -674,7 +674,7 @@ pub fn expected_error_kind(message: &str) -> Option { /// disk-full condition during its own page bookkeeping (journal/WAL extension) /// before the next syscall surfaces an errno, rusqlite renders the `SQLITE_FULL` /// result code as `"database or disk is full"` (Sentry TAURI-RUST-B6N, hit at -/// `memory_store::unified::documents::tx.commit()` during +/// `memory_store::namespace_store::documents::tx.commit()` during /// `openhuman.memory_doc_ingest`). `SQLITE_FULL` has only two causes: /// genuine ENOSPC/ERROR_DISK_FULL (always the case in practice — the same /// burst always produces an os-error-28/112 sibling event) or a @@ -683,7 +683,7 @@ pub fn expected_error_kind(message: &str) -> Option { /// rusqlite renders `SQLITE_FULL` in one of two shapes. The **bare** shape is /// the five words `"database or disk is full"` — Our local memory-store write /// call-sites wrap it with `format!(": {e}")` (e.g. `"commit tx: ..."` / -/// `"clear_namespace commit tx: ..."` in `memory_store::unified::documents`), +/// `"clear_namespace commit tx: ..."` in `memory_store::namespace_store::documents`), /// so the phrase lands as the **suffix** of the local emit. The **extended** /// shape carries the full error-code envelope, `"database or disk is full: /// Error code 13: Insertion failed because database is full"` (Sentry @@ -1190,9 +1190,8 @@ fn is_loopback_unavailable(lower: &str) -> bool { /// the local Ollama daemon — pure user-state errors the UI already surfaces /// (toast / settings page warning) where Sentry has no remediation path. /// -/// Several canonical wire shapes are covered, all emitted by -/// `openhuman::embeddings::ollama::OllamaEmbedding::embed` and the embed -/// service fallback path: +/// Several canonical wire shapes are covered, all emitted by the TinyAgents +/// Ollama embedder and the host embed service fallback path: /// /// - **TAURI-RUST-XS** (~376 events on self-hosted Sentry): user pointed the /// embedder at a chat / vision model id with a temperature suffix (e.g. @@ -4250,7 +4249,7 @@ mod tests { // SQLITE_FULL rendering from rusqlite — engine-level disk-full // detection during page-bookkeeping (journal/WAL extension) that // beats the next syscall to the errno. Production hit at - // `memory_store::unified::documents::tx.commit()` during + // `memory_store::namespace_store::documents::tx.commit()` during // `openhuman.memory_doc_ingest`, in the same burst that emits // os-error-112 siblings (Sentry TAURI-RUST-B6N). "commit tx: database or disk is full", diff --git a/src/openhuman/agent/harness/archivist/hook_impl.rs b/src/openhuman/agent/harness/archivist/hook_impl.rs index ab4db9a55..cea1ea003 100644 --- a/src/openhuman/agent/harness/archivist/hook_impl.rs +++ b/src/openhuman/agent/harness/archivist/hook_impl.rs @@ -80,15 +80,17 @@ impl PostTurnHook for ArchivistHook { tracing::debug!("[archivist] episodic rows written: session={session_id}"); - // Dual-write into memory_archivist::store (md-backed) so we can + // Dual-write into the crate-owned archivist store (md-backed) so we can // validate the FTS5 → md migration before flipping the read side. // Best-effort: a write failure here must not break the turn. The // user turn's assigned seq is captured into `current_seq` so the // segment ops can store it alongside the FTS5 episodic id. let mut current_seq: Option = None; if let Some(cfg) = self.config.as_ref() { + let engine_config = + crate::openhuman::tinycortex::memory_config_from(cfg, cfg.workspace_dir.clone()); let ts_ms = (timestamp * 1000.0) as i64; - let user_turn = crate::openhuman::memory_archivist::ArchivedTurn { + let user_turn = tinycortex::memory::archivist::types::ArchivedTurn { session_id: session_id.to_string(), seq: 0, // assigned by record_turn timestamp_ms: ts_ms, @@ -98,7 +100,7 @@ impl PostTurnHook for ArchivistHook { tool_calls_json: None, cost_microdollars: 0, }; - match crate::openhuman::memory_archivist::store::record_turn(cfg, user_turn) { + match tinycortex::memory::archivist::store::record_turn(&engine_config, user_turn) { Ok(stored) => current_seq = Some(stored.seq), Err(e) => { tracing::warn!("[archivist] memory_archivist user dual-write failed: {e}"); @@ -113,7 +115,7 @@ impl PostTurnHook for ArchivistHook { } else { Some(serde_json::to_string(&ctx.tool_calls).unwrap_or_default()) }; - let assistant_turn = crate::openhuman::memory_archivist::ArchivedTurn { + let assistant_turn = tinycortex::memory::archivist::types::ArchivedTurn { session_id: session_id.to_string(), seq: 0, timestamp_ms: ts_ms + 1, @@ -124,7 +126,7 @@ impl PostTurnHook for ArchivistHook { cost_microdollars: 0, }; if let Err(e) = - crate::openhuman::memory_archivist::store::record_turn(cfg, assistant_turn) + tinycortex::memory::archivist::store::record_turn(&engine_config, assistant_turn) { tracing::warn!("[archivist] memory_archivist assistant dual-write failed: {e}"); } diff --git a/src/openhuman/agent/harness/archivist/recap.rs b/src/openhuman/agent/harness/archivist/recap.rs index ce670be3b..07a1fbad6 100644 --- a/src/openhuman/agent/harness/archivist/recap.rs +++ b/src/openhuman/agent/harness/archivist/recap.rs @@ -11,7 +11,7 @@ use std::sync::Arc; impl ArchivistHook { /// Read every entry recorded for `session_id`, preferring the - /// md-backed `memory_archivist::store` when `self.config` is set and + /// crate-owned md-backed archivist store when `self.config` is set and /// falling back to the legacy FTS5 episodic table otherwise. /// /// Returns `EpisodicEntry` so the existing call sites (segment @@ -23,7 +23,10 @@ impl ArchivistHook { session_id: &str, ) -> Vec { if let Some(cfg) = self.config.as_ref() { - match crate::openhuman::memory_archivist::store::session_entries(cfg, session_id) { + let engine_config = + crate::openhuman::tinycortex::memory_config_from(cfg, cfg.workspace_dir.clone()); + match tinycortex::memory::archivist::store::session_entries(&engine_config, session_id) + { Ok(turns) => { return turns .into_iter() diff --git a/src/openhuman/embeddings/README.md b/src/openhuman/embeddings/README.md index a3501a4ed..d23f99310 100644 --- a/src/openhuman/embeddings/README.md +++ b/src/openhuman/embeddings/README.md @@ -1,106 +1,31 @@ # embeddings -Embedding providers for the OpenHuman memory system. Converts text into numerical vectors for semantic search, abstracting over multiple backends behind a single `EmbeddingProvider` trait. Owns provider construction (factory + slug catalog), per-endpoint client-side rate limiting, 429/503 retry-with-backoff logic, and the JSON-RPC surface that the Settings UI uses to pick a provider, store API keys, test connectivity, and embed text. The default provider is the OpenHuman backend ("managed" / "cloud", Voyage-backed); other paths include direct Voyage, OpenAI, Cohere, local Ollama, any OpenAI-compatible `custom:` endpoint, and a no-op (keyword-only) fallback. +Host policy and RPC surface for vector embeddings. Concrete OpenAI-compatible, +Cohere, Voyage, Ollama, cloud-transport, retry, and rate-limit implementations +live in `tinyagents::harness::embeddings`; this domain selects and adapts those +models for OpenHuman. -## Responsibilities +## Host-owned responsibilities -- Define `EmbeddingProvider` (async trait) and the canonical embedding-space signature format (`provider=…;model=…;dims=…`) — the single source of truth so config-derived and live-provider signatures stay byte-identical (#1574). -- Implement each provider: cloud/managed, Voyage, OpenAI(-compatible), Cohere, Ollama, noop. -- Construct providers from a slug + model + dims (with or without credentials) via the factory. -- Maintain the static provider/model catalog (slugs, labels, API-key/endpoint requirements, dimension presets) consumed by the frontend picker. -- Throttle outbound cloud-embedding HTTP requests with a process-global, per-endpoint token bucket (loopback exempt). -- Parse `Retry-After` and apply 429/503 exponential backoff in HTTP-based providers. -- Expose RPC handlers for settings, API-key management, embed, and connection testing; trigger memory wipe / re-embed backfill when the embedding signature changes. +- `factory.rs`: provider slug/model/dimension selection and construction of + TinyAgents models. +- `provider_trait.rs`: compatibility contract used by existing OpenHuman + consumers plus `TinyAgentsEmbeddingProvider`, the one trait adapter. +- `cloud_adapter.rs`: OpenHuman session-token resolution, egress disclosure, + and local-only privacy enforcement around TinyAgents `CloudEmbeddingModel`. +- `catalog.rs`, `rpc.rs`, `schemas.rs`: Settings catalog, credentials, JSON-RPC, + connection tests, and re-embed/wipe policy. +- `noop.rs`: unit-struct compatibility adapter over TinyAgents' no-op model. -## Key files +Provider HTTP behavior and its tests belong in `vendor/tinyagents`. Do not add +new per-provider clients here. -| File | Role | -| --- | --- | -| `src/openhuman/embeddings/mod.rs` | Module docstring + module decls + embedding provider `pub use` re-exports; exposes `all_embeddings_controller_schemas` / `all_embeddings_registered_controllers`. | -| `src/openhuman/embeddings/provider_trait.rs` | `EmbeddingProvider` trait (`name`/`model_id`/`dimensions`/`signature`/`embed`/`embed_one`) and `format_embedding_signature`. | -| `src/openhuman/embeddings/factory.rs` | `create_embedding_provider`, `create_embedding_provider_with_credentials`, `default_embedding_provider` (cloud), `default_local_embedding_provider` (Ollama). Maps provider slugs to concrete impls; unknown slugs error. | -| `src/openhuman/embeddings/catalog.rs` | Static catalog of `EmbeddingProviderEntry` + `EmbeddingModelPreset`; slug constants; `all_providers` / `find_provider` / `find_model` / `default_model_for`. | -| `src/openhuman/embeddings/cloud.rs` | `OpenHumanCloudEmbedding` — default provider; resolves session JWT + API URL per call, delegates HTTP to `OpenAiEmbedding` against `/openai/v1`. `DEFAULT_CLOUD_EMBEDDING_MODEL` = `embedding-v1`, dims 1024. | -| `src/openhuman/embeddings/openai.rs` | `OpenAiEmbedding` — OpenAI-compatible `POST /v1/embeddings`; URL inference, 429/503 retry, rate-limit gating, dimension/count validation. Used directly by cloud, voyage, and custom paths. | -| `src/openhuman/embeddings/voyage.rs` | `VoyageEmbedding` — thin wrapper delegating to `OpenAiEmbedding` against `api.voyageai.com`. | -| `src/openhuman/embeddings/cohere.rs` | `CohereEmbedding` — Cohere-native `POST /v2/embed` wire format (`texts`, `embedding_types`, nested `embeddings.float`) with its own 429/503 retry loop. | -| `src/openhuman/embeddings/ollama.rs` | `OllamaEmbedding` — local Ollama `POST /api/embed`; base-url/model normalization, blank-input zero-vector preservation, NaN-encoding 500 per-text recovery (TAURI-RUST-AZ). Defaults `bge-m3`/1024. | -| `src/openhuman/embeddings/noop.rs` | `NoopEmbedding` — returns empty vectors; keyword-only fallback (`name`/`model_id` = "none", dims 0). | -| `src/openhuman/embeddings/rate_limit.rs` | Process-global, per-endpoint token-bucket request limiter; `set_embedding_rate_limit`, `embedding_rate_limit`, `acquire_embedding_slot`. Default 60/min; loopback exempt; `0` disables. | -| `src/openhuman/embeddings/retry_after.rs` | `parse_retry_after_ms`, `backoff_ms_for_attempt`, and the `MAX_429_RETRIES` / backoff constants. | -| `src/openhuman/embeddings/rpc.rs` | RPC business logic: `get_settings`, `update_settings`, `set_api_key`, `clear_api_key`, `embed`, `test_connection`, plus `provider_from_config` and `resolve_api_key`. | -| `src/openhuman/embeddings/schemas.rs` | Controller schemas + `handle_*` param-deserializing wrappers delegating to `rpc.rs`. | -| `src/openhuman/embeddings/mod_tests.rs` | Module-level tests (via `#[path]`). | -| `src/openhuman/embeddings/openai_tests.rs` / `ollama_tests.rs` | Sibling test suites for the OpenAI and Ollama providers (via `#[path]`). | +The canonical embedding-space signature is +`provider={name};model={model};dims={dims}`. Configuration-derived and live +provider signatures must remain byte-identical or stored vectors split into +incompatible spaces. -## Public surface - -From `mod.rs` re-exports: - -- Trait + helper: `EmbeddingProvider`, `format_embedding_signature`. -- Providers: `OpenHumanCloudEmbedding`, `OpenAiEmbedding`, `OllamaEmbedding`, `NoopEmbedding` (Voyage/Cohere are public via their submodules through the factory). -- Factory fns: `create_embedding_provider`, `default_embedding_provider`, `default_local_embedding_provider`. -- Config-driven builder: `provider_from_config` (re-exported from `rpc`) — same construction `embed` uses, for callers like `codegraph` that need a provider without a JSON-RPC round-trip. -- Defaults/consts: `DEFAULT_CLOUD_EMBEDDING_MODEL`, `DEFAULT_CLOUD_EMBEDDING_DIMENSIONS`, `DEFAULT_OLLAMA_MODEL`, `DEFAULT_OLLAMA_DIMENSIONS`. -- RPC registry: `all_embeddings_controller_schemas`, `all_embeddings_registered_controllers`. - -## RPC / controllers - -Namespace `embeddings` (6 controllers, registered through `src/core/all.rs`): - -| Method | Description | -| --- | --- | -| `embeddings.get_settings` | Current provider/model/dims/rate-limit + full provider catalog with `has_api_key` flags and `vector_search_enabled`. | -| `embeddings.update_settings` | Update provider/model/dimensions/custom_endpoint/rate_limit. Requires `confirm_wipe=true` when **dimensions** change (returns `EMBEDDINGS_DIMENSION_CHANGE_REQUIRES_WIPE` otherwise); wipes memory on dim change and queues a re-embed backfill on any signature change. Also syncs `config.embeddings_provider` workload routing. | -| `embeddings.set_api_key` | Store an API key under credential provider `embeddings:`. | -| `embeddings.clear_api_key` | Remove the stored key for `embeddings:`. | -| `embeddings.embed` | Embed input texts using the configured provider; returns vectors, dimensions, count. | -| `embeddings.test_connection` | Run a single test embed against the configured or specified provider/model/dims. | - -All handlers return `RpcOutcome` via `into_cli_compatible_json`. - -## Agent tools - -None. This module owns no `tools.rs`; agent-facing memory tools live in `memory/tools/` and consume embeddings indirectly through the memory store. - -## Events - -None. No `bus.rs` / `EventHandler` impls. Cross-module side effects in `update_settings` are direct calls (`memory::read_rpc::wipe_all_rpc`, `memory_queue::ensure_reembed_backfill`), not domain events. - -## Persistence - -No `store.rs`. Settings live in the global `Config` (`config.memory.embedding_provider` / `embedding_model` / `embedding_dimensions` / `embedding_rate_limit_per_min`, plus `config.embeddings_provider`). API keys are persisted via the credentials domain (`AuthService`) under provider key `embeddings:`. Vectors themselves are stored by `memory_store::vectors`. - -## Dependencies - -- `crate::openhuman::config` — `Config`, `config::ops::load_config_with_timeout`, `build_runtime_proxy_client` (proxy-aware reqwest), config save. -- `crate::openhuman::credentials` — `AuthService`, `APP_SESSION_PROVIDER` for resolving the session JWT (cloud) and storing/loading provider API keys. -- `crate::openhuman::memory_store::vectors` — canonical vector store owner used by memory storage and retrieval. -- `crate::openhuman::memory::read_rpc` — `wipe_all_rpc` on a dimension change. -- `crate::openhuman::memory_queue` — `ensure_reembed_backfill` on a signature change. -- `crate::openhuman::inference::local` — `ollama_base_url()` when constructing the Ollama provider. -- `crate::api::config` — `effective_api_url` for the cloud backend base URL. -- `crate::core::all` — `ControllerFuture`, `RegisteredController` for the RPC registry. -- `crate::core::{ControllerSchema, FieldSchema, TypeSchema}` — schema definitions. -- `crate::core::observability` — `report_error_or_expected` so transient upstream HTTP failures demote to warning breadcrumbs instead of Sentry errors. -- `crate::rpc::RpcOutcome` — handler return contract. - -## Used by - -- `memory_store/*` — factories, vectors store, chunks store, unified store, retrieval, client (primary consumer; constructs providers and uses signatures to partition the embedding space). -- `memory/*` — ingestion queue, preferences, tools (recall/store/forget), `read_rpc`. -- `memory_tree/score/embed/cloud.rs` — scoring path. -- `codegraph/*` — index/search/tools obtain a provider via `provider_from_config` for `signature()` + direct embedding. -- `voice/factory.rs`, `screen_intelligence`, `channels`, `agent` tools — indirect consumers. -- `config/schema/load.rs` — wires `memory.embedding_rate_limit_per_min` into `rate_limit::set_embedding_rate_limit`. -- `core/all.rs` — registers the RPC controllers; `core/observability.rs` — classifies the canonical embedding error strings. - -## Notes / gotchas - -- **Signature drift is the core hazard.** `format_embedding_signature` is the single source of truth; both config-derived and live-provider signatures route through it. A mismatch silently splits one embedding space into two (#1574). `update_settings` keys memory wipe off **dimension** change (vectors stay comparable across provider/model swaps at the same dimensionality) but queues a re-embed backfill on **any** signature change. -- **Rate limiting is account-wide and process-global**, keyed by resolved base URL, with capacity = 1 token (no burst) to stay strictly under a hard per-minute cap. Ephemeral provider instances share one budget per endpoint. Loopback hosts are exempt; `limit==0` disables. The acquire chokepoint sits **inside** the retry loop so retried attempts consume tokens. -- **Cloud provider resolves auth lazily per call** — it can be constructed before login; the first `embed()` errors clearly if unauthenticated. It honors `OPENHUMAN_WORKSPACE` for the auth-profiles directory so non-default workspaces (tests/multi-instance) don't silently lose their session. -- **Ollama NaN recovery (TAURI-RUST-AZ):** a single bad input can poison a whole batch with a 500 `unsupported value: NaN`; the provider re-issues per-text and substitutes empty embeddings for the offending entries. Blank/whitespace inputs are preserved as zero-vectors so result length always matches input length. `local-*` model IDs are rejected (they're virtual routing aliases). -- **Custom endpoints** are encoded in the provider slug as `custom:`; `resolve_api_key` and the RPC handlers normalize this back to the `custom` credential slug. -- **Voyage and cloud reuse `OpenAiEmbedding`** for HTTP plumbing (Voyage's API is an OpenAI superset). **Cohere** speaks a distinct wire format and has its own retry loop. -- Error strings from OpenAI/Cohere providers intentionally preserve the `(429 ` / status substring so `core::observability`'s `TransientUpstreamHttp` classifier downgrades them. +The memory tree uses the same factory through +`memory_tree/score/embed::ProviderEmbedder`; its fixed on-disk dimension remains +1024. Ollama requests use TinyAgents' +`RECOMMENDED_OLLAMA_CONTEXT_TOKENS` for both `num_ctx` and `num_batch`. diff --git a/src/openhuman/embeddings/cohere_adapter.rs b/src/openhuman/embeddings/cohere_adapter.rs deleted file mode 100644 index 1ff51e7a9..000000000 --- a/src/openhuman/embeddings/cohere_adapter.rs +++ /dev/null @@ -1,51 +0,0 @@ -//! Compatibility wrapper for tinyagents' Cohere model. - -use async_trait::async_trait; -use tinyagents::harness::embeddings::{CohereEmbeddingModel, EmbeddingModel}; - -use super::EmbeddingProvider; - -pub struct CohereEmbedding { - inner: CohereEmbeddingModel, -} - -impl CohereEmbedding { - pub fn new(api_key: &str, model: &str, dimensions: usize) -> Self { - Self { - inner: CohereEmbeddingModel::new(api_key) - .with_model(model) - .with_dimensions(dimensions), - } - } - - pub fn with_base_url(mut self, base_url: impl Into) -> Self { - self.inner = self.inner.with_base_url(base_url); - self - } -} - -#[async_trait] -impl EmbeddingProvider for CohereEmbedding { - fn name(&self) -> &str { - self.inner.name() - } - fn model_id(&self) -> &str { - self.inner.model_id() - } - fn dimensions(&self) -> usize { - self.inner.dimensions() - } - fn signature(&self) -> String { - self.inner.signature() - } - async fn embed(&self, texts: &[&str]) -> anyhow::Result>> { - let texts = texts - .iter() - .map(|text| (*text).to_owned()) - .collect::>(); - self.inner - .embed(&texts) - .await - .map_err(|error| anyhow::anyhow!(error)) - } -} diff --git a/src/openhuman/embeddings/mod.rs b/src/openhuman/embeddings/mod.rs index 4854da55e..29b636633 100644 --- a/src/openhuman/embeddings/mod.rs +++ b/src/openhuman/embeddings/mod.rs @@ -15,14 +15,8 @@ pub mod catalog; #[path = "cloud_adapter.rs"] pub mod cloud; -#[path = "cohere_adapter.rs"] -pub mod cohere; mod factory; pub mod noop; -#[path = "ollama_adapter.rs"] -pub mod ollama; -#[path = "openai_adapter.rs"] -pub mod openai; mod provider_trait; pub mod rate_limit { pub use tinyagents::harness::embeddings::{ @@ -42,8 +36,6 @@ pub mod retry_after { } mod rpc; mod schemas; -#[path = "voyage_adapter.rs"] -pub mod voyage; pub use catalog::non_embedding_model_reason; pub use cloud::{ @@ -60,8 +52,6 @@ pub(crate) use factory::model_supports_dimensions; // #002 FR-015: the memory-tree OpenAI-compat embedder reuses the same key // resolution the embeddings RPC uses, so there is one source of truth. pub use noop::NoopEmbedding; -pub use ollama::OllamaEmbedding; -pub use openai::OpenAiEmbedding; pub use provider_trait::{ format_embedding_signature, EmbeddingProvider, TinyAgentsEmbeddingProvider, }; @@ -71,7 +61,9 @@ pub use schemas::{ all_controller_schemas as all_embeddings_controller_schemas, all_registered_controllers as all_embeddings_registered_controllers, }; -pub use tinyagents::harness::embeddings::{DEFAULT_OLLAMA_DIMENSIONS, DEFAULT_OLLAMA_MODEL}; +pub use tinyagents::harness::embeddings::{ + DEFAULT_OLLAMA_DIMENSIONS, DEFAULT_OLLAMA_MODEL, DEFAULT_OLLAMA_URL, +}; #[cfg(test)] #[path = "mod_tests.rs"] diff --git a/src/openhuman/embeddings/mod_tests.rs b/src/openhuman/embeddings/mod_tests.rs index a1151227d..ce8a0d5f7 100644 --- a/src/openhuman/embeddings/mod_tests.rs +++ b/src/openhuman/embeddings/mod_tests.rs @@ -21,15 +21,13 @@ fn noop_name_and_dims() { async fn noop_embed_returns_empty() { let p = NoopEmbedding; let result = p.embed(&["hello"]).await.unwrap(); - assert!(result.is_empty()); + assert_eq!(result, vec![Vec::::new()]); } #[tokio::test] -async fn noop_embed_one_returns_error() { - // embed returns empty vec → pop() returns None → error from default impl +async fn noop_embed_one_returns_empty_vector() { let p = NoopEmbedding; - let err = p.embed_one("hello").await.unwrap_err(); - assert!(err.to_string().contains("Empty embedding result")); + assert!(p.embed_one("hello").await.unwrap().is_empty()); } #[tokio::test] diff --git a/src/openhuman/embeddings/noop.rs b/src/openhuman/embeddings/noop.rs index 18949a532..948e53860 100644 --- a/src/openhuman/embeddings/noop.rs +++ b/src/openhuman/embeddings/noop.rs @@ -1,6 +1,7 @@ //! No-op embedding provider for keyword-only search fallback. use async_trait::async_trait; +use tinyagents::harness::embeddings::{EmbeddingModel, NoopEmbeddingModel}; use super::EmbeddingProvider; @@ -22,7 +23,14 @@ impl EmbeddingProvider for NoopEmbedding { 0 } - async fn embed(&self, _texts: &[&str]) -> anyhow::Result>> { - Ok(Vec::new()) + async fn embed(&self, texts: &[&str]) -> anyhow::Result>> { + let texts = texts + .iter() + .map(|text| (*text).to_owned()) + .collect::>(); + NoopEmbeddingModel + .embed(&texts) + .await + .map_err(|error| anyhow::anyhow!(error)) } } diff --git a/src/openhuman/embeddings/ollama_adapter.rs b/src/openhuman/embeddings/ollama_adapter.rs deleted file mode 100644 index 0b8162fe1..000000000 --- a/src/openhuman/embeddings/ollama_adapter.rs +++ /dev/null @@ -1,61 +0,0 @@ -//! Compatibility wrapper for tinyagents' Ollama model. - -use async_trait::async_trait; -use tinyagents::harness::embeddings::{EmbeddingModel, OllamaEmbeddingModel}; - -use super::EmbeddingProvider; - -pub use tinyagents::harness::embeddings::{ - DEFAULT_OLLAMA_DIMENSIONS, DEFAULT_OLLAMA_MODEL, DEFAULT_OLLAMA_URL, -}; - -#[derive(Default)] -pub struct OllamaEmbedding { - inner: OllamaEmbeddingModel, -} - -impl OllamaEmbedding { - pub fn try_new(base_url: &str, model: &str, dimensions: usize) -> anyhow::Result { - Ok(Self { - inner: OllamaEmbeddingModel::try_new(base_url, model, dimensions)?, - }) - } - - pub fn new(base_url: &str, model: &str, dimensions: usize) -> Self { - Self::try_new(base_url, model, dimensions).expect("invalid Ollama embedding configuration") - } - - pub fn base_url(&self) -> &str { - self.inner.base_url() - } - - pub fn model(&self) -> &str { - self.inner.model() - } -} - -#[async_trait] -impl EmbeddingProvider for OllamaEmbedding { - fn name(&self) -> &str { - self.inner.name() - } - fn model_id(&self) -> &str { - self.inner.model_id() - } - fn dimensions(&self) -> usize { - self.inner.dimensions() - } - fn signature(&self) -> String { - self.inner.signature() - } - async fn embed(&self, texts: &[&str]) -> anyhow::Result>> { - let texts = texts - .iter() - .map(|text| (*text).to_owned()) - .collect::>(); - self.inner - .embed(&texts) - .await - .map_err(|error| anyhow::anyhow!(error)) - } -} diff --git a/src/openhuman/embeddings/openai_adapter.rs b/src/openhuman/embeddings/openai_adapter.rs deleted file mode 100644 index 003cabe78..000000000 --- a/src/openhuman/embeddings/openai_adapter.rs +++ /dev/null @@ -1,71 +0,0 @@ -//! Compatibility wrapper for tinyagents' OpenAI-compatible model. - -use async_trait::async_trait; -use tinyagents::harness::embeddings::{EmbeddingModel, OpenAiEmbeddingModel}; - -use super::EmbeddingProvider; - -pub struct OpenAiEmbedding { - inner: OpenAiEmbeddingModel, -} - -impl OpenAiEmbedding { - pub fn new(base_url: &str, api_key: &str, model: &str, dimensions: usize) -> Self { - Self { - inner: OpenAiEmbeddingModel::new(api_key) - .with_base_url(base_url) - .with_model(model) - .with_dimensions(dimensions) - .with_send_dimensions(false) - .with_required_api_key(false), - } - } - - pub fn with_send_dimensions(mut self, send: bool) -> Self { - self.inner = self.inner.with_send_dimensions(send); - self - } - - pub fn with_required_api_key(mut self, required: bool) -> Self { - self.inner = self.inner.with_required_api_key(required); - self - } - - pub fn base_url(&self) -> &str { - self.inner.base_url() - } - - pub fn model(&self) -> &str { - self.inner.model() - } - - pub fn embeddings_url(&self) -> String { - self.inner.embeddings_url() - } -} - -#[async_trait] -impl EmbeddingProvider for OpenAiEmbedding { - fn name(&self) -> &str { - self.inner.name() - } - fn model_id(&self) -> &str { - self.inner.model_id() - } - fn dimensions(&self) -> usize { - self.inner.dimensions() - } - fn signature(&self) -> String { - self.inner.signature() - } - async fn embed(&self, texts: &[&str]) -> anyhow::Result>> { - let texts = texts - .iter() - .map(|text| (*text).to_owned()) - .collect::>(); - self.inner - .embed(&texts) - .await - .map_err(|error| anyhow::anyhow!(error)) - } -} diff --git a/src/openhuman/embeddings/voyage_adapter.rs b/src/openhuman/embeddings/voyage_adapter.rs deleted file mode 100644 index 9da55c8f4..000000000 --- a/src/openhuman/embeddings/voyage_adapter.rs +++ /dev/null @@ -1,69 +0,0 @@ -//! Compatibility wrapper for tinyagents' Voyage model. - -use async_trait::async_trait; -use tinyagents::harness::embeddings::{ - EmbeddingModel, VoyageEmbeddingModel, VOYAGE_API_BASE, VOYAGE_DEFAULT_DIMENSIONS, - VOYAGE_DEFAULT_MODEL, -}; - -use super::EmbeddingProvider; - -pub struct VoyageEmbedding { - inner: VoyageEmbeddingModel, -} - -impl VoyageEmbedding { - pub fn new(api_key: &str, model: &str, dimensions: usize) -> Self { - Self::new_with_base_url(api_key, model, dimensions, VOYAGE_API_BASE) - } - - pub fn new_with_base_url( - api_key: &str, - model: &str, - dimensions: usize, - base_url: &str, - ) -> Self { - Self { - inner: VoyageEmbeddingModel::with_options( - api_key, - if model.is_empty() { - VOYAGE_DEFAULT_MODEL - } else { - model - }, - if dimensions == 0 { - VOYAGE_DEFAULT_DIMENSIONS - } else { - dimensions - }, - base_url, - ), - } - } -} - -#[async_trait] -impl EmbeddingProvider for VoyageEmbedding { - fn name(&self) -> &str { - self.inner.name() - } - fn model_id(&self) -> &str { - self.inner.model_id() - } - fn dimensions(&self) -> usize { - self.inner.dimensions() - } - fn signature(&self) -> String { - self.inner.signature() - } - async fn embed(&self, texts: &[&str]) -> anyhow::Result>> { - let texts = texts - .iter() - .map(|text| (*text).to_owned()) - .collect::>(); - self.inner - .embed(&texts) - .await - .map_err(|error| anyhow::anyhow!(error)) - } -} diff --git a/src/openhuman/inference/local/model_requirements.rs b/src/openhuman/inference/local/model_requirements.rs index a23eb482e..4fc94b65a 100644 --- a/src/openhuman/inference/local/model_requirements.rs +++ b/src/openhuman/inference/local/model_requirements.rs @@ -2,7 +2,7 @@ //! //! The memory tree's embedder (`bge-m3`) is requested with //! `num_ctx = 8192` (see -//! [`crate::openhuman::memory_tree::score::embed::ollama::EMBED_NUM_CTX`]) +//! [`tinyagents::harness::embeddings::RECOMMENDED_OLLAMA_CONTEXT_TOKENS`]) //! and the summariser hard-caps its output to fit that 8192-token embed //! ceiling. A local model whose native context window is below this floor //! silently truncates chunks/summaries and corrupts recall, so we refuse @@ -16,12 +16,12 @@ use serde::Serialize; /// Minimum native context window (tokens) a local model must advertise to /// be accepted by the memory layer. /// -/// Re-exported from the embedder's own `EMBED_NUM_CTX` so this gate can +/// Re-exported from TinyAgents' canonical Ollama context setting so this gate can /// never drift from what the memory pipeline actually requests at embed /// time. Changing the embedder's context request automatically moves the /// acceptance floor with it. pub const MIN_CONTEXT_TOKENS: u64 = - crate::openhuman::memory_tree::score::embed::ollama::EMBED_NUM_CTX as u64; + tinyagents::harness::embeddings::RECOMMENDED_OLLAMA_CONTEXT_TOKENS as u64; /// Verdict for a single model's context window against /// [`MIN_CONTEXT_TOKENS`]. Serialized into the diagnostics payload so the @@ -79,7 +79,7 @@ mod tests { // requests; this guards against the two drifting apart. assert_eq!( MIN_CONTEXT_TOKENS, - crate::openhuman::memory_tree::score::embed::ollama::EMBED_NUM_CTX as u64 + tinyagents::harness::embeddings::RECOMMENDED_OLLAMA_CONTEXT_TOKENS as u64 ); assert_eq!(MIN_CONTEXT_TOKENS, 8_192); } diff --git a/src/openhuman/learning/README.md b/src/openhuman/learning/README.md index 17d482c86..dd8b8ee00 100644 --- a/src/openhuman/learning/README.md +++ b/src/openhuman/learning/README.md @@ -111,7 +111,7 @@ These are subscriber registrations rather than a single `bus.rs`; subscriptions - `src/core/all.rs` — registers the `learning.*` controllers + schemas. - `src/openhuman/agent/harness/session/{builder,turn}.rs` and `agent_memory/memory_loader.rs` — wire the post-turn hooks, prompt sections, and learned-context loading into the agent loop. - `src/openhuman/channels/runtime/startup.rs` — likely registers schedulers/subscribers at startup. -- `src/openhuman/memory_store/unified/profile.rs`, `memory_sync/composio/providers/profile.rs`, `memory_tools/{capture,mod}.rs`, `tools/impl/system/tool_stats.rs`, `tools/schemas.rs` — consume facet/learning types. +- `src/openhuman/memory_store/namespace_store/profile.rs`, `memory_sync/composio/providers/profile.rs`, `memory_tools/{capture,mod}.rs`, `tools/impl/system/tool_stats.rs`, `tools/schemas.rs` — consume facet/learning types. ## Notes / gotchas diff --git a/src/openhuman/learning/cache.rs b/src/openhuman/learning/cache.rs index 2321fad11..1c801dbd2 100644 --- a/src/openhuman/learning/cache.rs +++ b/src/openhuman/learning/cache.rs @@ -14,7 +14,7 @@ use crate::openhuman::memory_store::profile::{self, ProfileFacet, UserState}; /// Thin wrapper around the `user_profile` table. /// /// All methods delegate to the standalone helpers in -/// `memory::store::unified::profile`. This type exists so callers +/// `memory_store::namespace_store::profile`. This type exists so callers /// (stability detector, prompt sections, RPCs) share a single typed /// entry-point that can be constructed from any `Arc>`. pub struct FacetCache { diff --git a/src/openhuman/memory/README.md b/src/openhuman/memory/README.md index af91b7764..e45430cdb 100644 --- a/src/openhuman/memory/README.md +++ b/src/openhuman/memory/README.md @@ -26,7 +26,7 @@ one job. memory orchestrates and routes between them. | ------------------------------------------ | --------------------------------------------------------------------------------------------------- | | [`memory_store`](../memory_store/) | Storage primitives: raw / chunks / entities / trees / vectors / kv / contacts. SQLite + on-disk md. | | [`memory_tree`](../memory_tree/) | Generic tree mechanics: bucket-seal, flush, summarise, and retrieval/traversal backends. | -| [`memory_archivist`](../memory_archivist/) | Chat conversation → clip tool-calls → push to tree (shim over `tinycortex::memory::archivist`). | +| `tinycortex::memory::archivist` | Chat conversation → clip tool-calls → push to tree; called directly by the host harness. | | [`memory_tools`](../memory_tools/) | Tool-scoped rules + agent read/write tools. | | [`memory_sync`](../memory_sync/) | Composio + workspace + MCP sync pipelines. | diff --git a/src/openhuman/memory/mod.rs b/src/openhuman/memory/mod.rs index e1b56dce8..c16a5980d 100644 --- a/src/openhuman/memory/mod.rs +++ b/src/openhuman/memory/mod.rs @@ -57,6 +57,5 @@ pub use traits::{Memory, MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSumm // Re-export types that external tests and consumers historically imported // from `memory::*`. The definitions moved to sibling crates during the // memory refactor; these aliases keep the public surface stable. -pub use crate::openhuman::memory_queue as jobs; pub use crate::openhuman::memory_store::types::NamespaceDocumentInput; pub use crate::openhuman::memory_store::{MemoryClient, UnifiedMemory}; diff --git a/src/openhuman/memory/read_rpc_tests.rs b/src/openhuman/memory/read_rpc_tests.rs index 0f36905e6..430242390 100644 --- a/src/openhuman/memory/read_rpc_tests.rs +++ b/src/openhuman/memory/read_rpc_tests.rs @@ -4,7 +4,7 @@ use crate::openhuman::embeddings::NoopEmbedding; use crate::openhuman::memory::ingest_pipeline::ingest_chat; use crate::openhuman::memory_queue::drain_until_idle; use crate::openhuman::memory_store::content::raw::{write_raw_items, RawItem, RawKind}; -use crate::openhuman::memory_store::unified::UnifiedMemory; +use crate::openhuman::memory_store::namespace_store::UnifiedMemory; use crate::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage}; use chrono::{TimeZone, Utc}; use rusqlite::params; diff --git a/src/openhuman/memory_archivist/mod.rs b/src/openhuman/memory_archivist/mod.rs deleted file mode 100644 index 979946d26..000000000 --- a/src/openhuman/memory_archivist/mod.rs +++ /dev/null @@ -1,44 +0,0 @@ -//! Memory archivist — thin host shim over `tinycortex::memory::archivist` (W7). -//! -//! The archivist engine (clip / compose / tree-writer / episodic store + its -//! types) lives in the crate now. This module keeps the host's -//! `memory_archivist::…` import paths stable and adapts the host [`Config`] to -//! the crate's `MemoryConfig` for the two episodic-store functions host callers -//! use (`store::record_turn` / `store::session_entries`, from the agent -//! archivist hook + recap). -//! -//! On-disk layout is unchanged — the crate writes to the same path the host -//! did: `/memory_tree/content/episodic//.md`. -//! `Config::workspace_dir` maps to the crate `MemoryConfig.workspace`, so -//! `memory_tree_content_root()` (`/memory_tree/content`) resolves -//! identically on both sides. - -pub use tinycortex::memory::archivist::types::{ArchivedTurn, Turn}; - -/// Episodic conversation archive — thin adapters over the crate store that -/// convert the host [`Config`](crate::openhuman::config::Config) into the -/// crate's `MemoryConfig`. -pub mod store { - use anyhow::Result; - - use super::ArchivedTurn; - use crate::openhuman::config::Config; - use crate::openhuman::tinycortex::memory_config_from; - - fn engine_config(config: &Config) -> tinycortex::memory::MemoryConfig { - memory_config_from(config, config.workspace_dir.clone()) - } - - /// Append one turn to its session's episodic archive, assigning the next - /// per-session sequence number. - pub fn record_turn(config: &Config, turn: ArchivedTurn) -> Result { - tinycortex::memory::archivist::store::record_turn(&engine_config(config), turn) - } - - /// All turns recorded for a session, ordered by sequence. - pub fn session_entries(config: &Config, session_id: &str) -> Result> { - tinycortex::memory::archivist::store::session_entries(&engine_config(config), session_id) - } -} - -pub use store::{record_turn, session_entries}; diff --git a/src/openhuman/memory_queue/mod.rs b/src/openhuman/memory_queue/mod.rs index f0414ddfe..b2cc6b5b8 100644 --- a/src/openhuman/memory_queue/mod.rs +++ b/src/openhuman/memory_queue/mod.rs @@ -28,7 +28,6 @@ //! This queue used to live under `openhuman::memory::jobs`; it now has a //! dedicated top-level home (`openhuman::memory_queue`) because it is an //! execution/runtime concern rather than a leaf of the memory policy API. -//! `openhuman::memory` re-exports it as `memory::jobs` during the migration. mod ops; pub mod scheduler; diff --git a/src/openhuman/memory_search/mod.rs b/src/openhuman/memory_search/mod.rs index 7a310f203..ac7dc1321 100644 --- a/src/openhuman/memory_search/mod.rs +++ b/src/openhuman/memory_search/mod.rs @@ -5,9 +5,7 @@ //! `memory_tree`) provide persistence and tree traversal; this module composes //! them into tools the agent can invoke. -pub mod scoring; pub mod tools; -pub mod vector; // ── Public re-exports ─────────────────────────────────────────────────────── diff --git a/src/openhuman/memory_search/scoring.rs b/src/openhuman/memory_search/scoring.rs deleted file mode 100644 index b64cefd63..000000000 --- a/src/openhuman/memory_search/scoring.rs +++ /dev/null @@ -1,40 +0,0 @@ -//! Scoring weight profiles for hybrid retrieval — thin host shim over -//! `tinycortex::memory::WeightProfile` (W5). -//! -//! The weight profile (graph/vector/keyword/freshness weights + the -//! `BALANCED`/`SEMANTIC`/`LEXICAL`/`GRAPH_FIRST` presets + `by_name`) is the -//! crate's, a byte-identical port. The host keeps only [`compose_score`] — the -//! trivial weighted combination the crate expresses via -//! `retrieval::scoring::hybrid_score` at its own call sites; exposed here as a -//! free function so `memory_search::tools::hybrid_search` keeps its call shape. - -pub use tinycortex::memory::WeightProfile; - -/// Weighted composite of the four retrieval signals under `profile`. -/// -/// `graph·graph_relevance + vector·vector_similarity + keyword·keyword_relevance -/// + freshness·freshness`. -pub fn compose_score( - profile: &WeightProfile, - graph_relevance: f64, - vector_similarity: f64, - keyword_relevance: f64, - freshness: f64, -) -> f64 { - (profile.graph * graph_relevance) - + (profile.vector * vector_similarity) - + (profile.keyword * keyword_relevance) - + (profile.freshness * freshness) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn compose_score_is_weighted_sum() { - let p = WeightProfile::BALANCED; - let s = compose_score(&p, 1.0, 1.0, 1.0, 1.0); - assert!((s - (p.graph + p.vector + p.keyword + p.freshness)).abs() < 1e-9); - } -} diff --git a/src/openhuman/memory_search/tools/hybrid_search.rs b/src/openhuman/memory_search/tools/hybrid_search.rs index 544aa7678..aeb2c38db 100644 --- a/src/openhuman/memory_search/tools/hybrid_search.rs +++ b/src/openhuman/memory_search/tools/hybrid_search.rs @@ -12,10 +12,10 @@ use std::sync::Arc; use crate::openhuman::config::rpc as config_rpc; use crate::openhuman::embeddings::{provider_from_config, EmbeddingProvider}; -use crate::openhuman::memory_search::scoring::WeightProfile; use crate::openhuman::memory_store::types::MemoryItemKind; use crate::openhuman::memory_store::UnifiedMemory; use crate::openhuman::tools::traits::{Tool, ToolResult}; +use tinycortex::memory::WeightProfile; pub struct MemoryHybridSearchTool; @@ -178,13 +178,14 @@ impl Tool for MemoryHybridSearchTool { .enumerate() .map(|(i, hit)| { let bd = &hit.score_breakdown; - let score = crate::openhuman::memory_search::scoring::compose_score( + let score = tinycortex::memory::retrieval::scoring::hybrid_score( &profile, bd.graph_relevance, bd.vector_similarity, bd.keyword_relevance, bd.freshness, - ); + ) + .final_score; (i, score) }) .filter(|(_, score)| *score > 0.0) diff --git a/src/openhuman/memory_search/tools/vector_search.rs b/src/openhuman/memory_search/tools/vector_search.rs index 4fa106551..15b59a109 100644 --- a/src/openhuman/memory_search/tools/vector_search.rs +++ b/src/openhuman/memory_search/tools/vector_search.rs @@ -11,13 +11,13 @@ use std::fmt::Write; use crate::openhuman::config::rpc as config_rpc; use crate::openhuman::embeddings::provider_from_config; -use crate::openhuman::memory_search::vector::mmr::{mmr_select, MmrCandidate}; use crate::openhuman::memory_store::chunks::store::{ get_chunk_embeddings_for_signature_batch, list_chunks, ListChunksQuery, }; use crate::openhuman::memory_store::chunks::types::SourceKind; use crate::openhuman::memory_store::vectors::cosine_similarity; use crate::openhuman::tools::traits::{Tool, ToolResult}; +use tinycortex::memory::retrieval::mmr::{mmr_select, MmrCandidate}; pub struct MemoryVectorSearchTool; diff --git a/src/openhuman/memory_search/vector/mmr.rs b/src/openhuman/memory_search/vector/mmr.rs deleted file mode 100644 index 6f48a0013..000000000 --- a/src/openhuman/memory_search/vector/mmr.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! Maximal Marginal Relevance selection — thin host re-export of -//! `tinycortex::memory::retrieval::mmr` (W5). -//! -//! The MMR algorithm (relevance–diversity tradeoff over embeddings) is the -//! crate's, a byte-identical port. Host consumers (`memory_search::tools`) keep -//! their `memory_search::vector::mmr::{MmrCandidate, MmrResult, mmr_select}` -//! import paths unchanged. - -pub use tinycortex::memory::retrieval::mmr::{mmr_select, MmrCandidate, MmrResult}; diff --git a/src/openhuman/memory_search/vector/mod.rs b/src/openhuman/memory_search/vector/mod.rs deleted file mode 100644 index 9ce45fd7d..000000000 --- a/src/openhuman/memory_search/vector/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -//! Vector diversity algorithms. - -pub mod mmr; diff --git a/src/openhuman/memory_store/README.md b/src/openhuman/memory_store/README.md index 1216184ad..fe224cdf8 100644 --- a/src/openhuman/memory_store/README.md +++ b/src/openhuman/memory_store/README.md @@ -12,9 +12,8 @@ trees/ summary tree persistence (one table, kind-parameterized) vectors/ local vector DB (cosine, brute-force) kv/ global + namespace key-value (kv_global, kv_namespace) contacts/ [removed] facade over people::store (Person/Handle/Interaction) -unified/ [staging for removal] UnifiedMemory's remaining SQLite surface - (documents/query/segments/events/profile) — replaced as the - Memory trait callers migrate to per-kind backends +namespace_store/ host-retained namespace documents, graph, episodic/event/ + segment/profile tables, and retrieval policy ``` ## Cross-cutting modules @@ -43,7 +42,7 @@ unified/ [staging for removal] UnifiedMemory's remaining SQLite surface | [`vectors/`](vectors/) | Standalone vector store. `VectorStore` over SQLite, byte-codec for f32 vectors, cosine similarity. | | [`kv.rs`](kv.rs) | Global + namespace key-value (`kv_global`, `kv_namespace` tables). | | `contacts/` | Removed. Contact access now lives outside `memory_store` via `people::store`. | -| [`unified/`](unified/) | **Staging for removal.** Shrinking SQLite surface that still backs the `Memory` trait while callers migrate to per-kind modules. Active pieces today: `documents`, `query`, `segments`, `events`, `profile`. The `fts5` episodic surface is replaced by [`memory_archivist`](../memory_archivist/) and `graph` by the crate's `tinycortex::memory::graph` (the host `memory_graph` placeholder was deleted in the W7 migration). See [`unified/README.md`](unified/README.md). | +| [`namespace_store/`](namespace_store/) | Host-retained namespace/document tier over the shared SQLite database: documents, persisted product graph relations, episodic/events, segments, profile facets, and host retrieval policy. TinyCortex owns the generic chunk/vector/tree/queue substrate; this tier remains the stable `Memory` implementation. See [`namespace_store/README.md`](namespace_store/README.md). | ## Layer rules diff --git a/src/openhuman/memory_store/client.rs b/src/openhuman/memory_store/client.rs index 8fa0139f1..8a250b0f0 100644 --- a/src/openhuman/memory_store/client.rs +++ b/src/openhuman/memory_store/client.rs @@ -17,10 +17,10 @@ use crate::openhuman::memory::ingestion::{ IngestionJob, IngestionQueue, IngestionState, MemoryIngestionConfig, MemoryIngestionRequest, MemoryIngestionResult, }; +use crate::openhuman::memory_store::namespace_store::UnifiedMemory; use crate::openhuman::memory_store::types::{ NamespaceDocumentInput, NamespaceMemoryHit, NamespaceRetrievalContext, }; -use crate::openhuman::memory_store::unified::UnifiedMemory; /// Reference-counted handle to a `MemoryClient`. pub type MemoryClientRef = Arc; @@ -56,7 +56,7 @@ pub struct MemoryClient { impl MemoryClient { /// Returns a handle to the underlying SQLite connection for direct /// profile-facet writes via - /// [`crate::openhuman::memory_store::unified::profile::profile_upsert`]. + /// [`crate::openhuman::memory_store::namespace_store::profile::profile_upsert`]. /// /// Intentionally `pub(crate)` — external consumers should use the /// higher-level `MemoryClient` API; this escape hatch exists so diff --git a/src/openhuman/memory_store/factories.rs b/src/openhuman/memory_store/factories.rs index 594157733..c13a76bd3 100644 --- a/src/openhuman/memory_store/factories.rs +++ b/src/openhuman/memory_store/factories.rs @@ -21,7 +21,7 @@ use crate::openhuman::embeddings::{ DEFAULT_CLOUD_EMBEDDING_MODEL, DEFAULT_OLLAMA_DIMENSIONS, DEFAULT_OLLAMA_MODEL, }; use crate::openhuman::memory::traits::Memory; -use crate::openhuman::memory_store::unified::UnifiedMemory; +use crate::openhuman::memory_store::namespace_store::UnifiedMemory; /// One-shot guard so the Ollama health-gate fallback only reports to Sentry /// once per process lifetime. Memory is constructed many times per session diff --git a/src/openhuman/memory_store/kv.rs b/src/openhuman/memory_store/kv.rs index 6afd67c1d..d647de655 100644 --- a/src/openhuman/memory_store/kv.rs +++ b/src/openhuman/memory_store/kv.rs @@ -2,8 +2,8 @@ use tinycortex::memory::store::kv::KvStore; +use crate::openhuman::memory_store::namespace_store::UnifiedMemory; use crate::openhuman::memory_store::types::MemoryKvRecord; -use crate::openhuman::memory_store::unified::UnifiedMemory; impl UnifiedMemory { fn tinycortex_kv(&self) -> Result { diff --git a/src/openhuman/memory_store/memory_trait.rs b/src/openhuman/memory_store/memory_trait.rs index 2ae603745..c7810c546 100644 --- a/src/openhuman/memory_store/memory_trait.rs +++ b/src/openhuman/memory_store/memory_trait.rs @@ -17,11 +17,11 @@ use serde_json::json; use crate::openhuman::memory::traits::{ Memory, MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary, RecallOpts, }; +use crate::openhuman::memory_store::namespace_store::fts5; use crate::openhuman::memory_store::types::{NamespaceDocumentInput, GLOBAL_NAMESPACE}; -use crate::openhuman::memory_store::unified::fts5; use anyhow::Context; -use super::unified::UnifiedMemory; +use super::namespace_store::UnifiedMemory; /// Convert a UNIX timestamp (f64) to RFC3339 string. fn timestamp_to_rfc3339(ts: f64) -> String { diff --git a/src/openhuman/memory_store/mod.rs b/src/openhuman/memory_store/mod.rs index e3ab38f24..efa65c84f 100644 --- a/src/openhuman/memory_store/mod.rs +++ b/src/openhuman/memory_store/mod.rs @@ -11,7 +11,8 @@ //! ## Submodules //! //! - `types`: Common data structures and types used across the memory store. -//! - `unified`: The primary SQLite-based memory implementation. +//! - `namespace_store`: Host-retained SQLite namespace documents, graph, +//! episodic/event/segment/profile tables, and their query policy. //! - `client`: High-level client interface for interacting with the memory system. //! - `factories`: Factory functions for creating and initializing memory instances. //! - `memory_trait`: Defines the `Memory` trait that all implementations must satisfy. @@ -21,13 +22,13 @@ pub mod content; pub mod entities; pub mod kinds; pub mod kv; +pub mod namespace_store; pub mod retrieval; pub mod safety; pub mod tools; pub mod traits; pub mod trees; pub mod types; -pub mod unified; pub mod vectors; mod client; @@ -42,16 +43,16 @@ pub use factories::{ active_embedding_signature, create_memory, create_memory_for_migration, create_memory_with_local_ai, effective_embedding_settings, effective_memory_backend_name, }; +pub use namespace_store::events; +pub use namespace_store::fts5; +pub use namespace_store::profile; +pub use namespace_store::segments; +pub use namespace_store::UnifiedMemory; pub use types::{ GraphRelationRecord, MemoryItemKind, MemoryKvRecord, NamespaceDocumentInput, NamespaceMemoryHit, NamespaceQueryResult, NamespaceRetrievalContext, RetrievalScoreBreakdown, StoredMemoryDocument, }; -pub use unified::events; -pub use unified::fts5; -pub use unified::profile; -pub use unified::segments; -pub use unified::UnifiedMemory; #[cfg(test)] mod tests { diff --git a/src/openhuman/memory_store/unified/README.md b/src/openhuman/memory_store/namespace_store/README.md similarity index 86% rename from src/openhuman/memory_store/unified/README.md rename to src/openhuman/memory_store/namespace_store/README.md index da4f5cad5..30aca9e70 100644 --- a/src/openhuman/memory_store/unified/README.md +++ b/src/openhuman/memory_store/namespace_store/README.md @@ -1,6 +1,11 @@ -# Unified memory store +# Namespace memory store -SQLite-backed implementation of the memory store. One `UnifiedMemory` struct owns a WAL-mode connection plus the on-disk markdown sidecar tree and vector storage path; the rest of this directory adds capabilities to it via per-domain `impl` blocks. +Host-retained SQLite namespace/document tier. One `UnifiedMemory` struct owns +the shared connection plus the on-disk markdown sidecar and compatibility +embedding handle; the rest of this directory adds the product-owned document, +graph, episodic, event, segment, profile, and retrieval policy via `impl` +blocks. TinyCortex owns the generic chunk/vector/tree/queue engine beside this +tier; this directory is intentionally not migration staging. ## Files diff --git a/src/openhuman/memory_store/unified/documents.rs b/src/openhuman/memory_store/namespace_store/documents.rs similarity index 100% rename from src/openhuman/memory_store/unified/documents.rs rename to src/openhuman/memory_store/namespace_store/documents.rs diff --git a/src/openhuman/memory_store/unified/documents_tests.rs b/src/openhuman/memory_store/namespace_store/documents_tests.rs similarity index 100% rename from src/openhuman/memory_store/unified/documents_tests.rs rename to src/openhuman/memory_store/namespace_store/documents_tests.rs diff --git a/src/openhuman/memory_store/unified/events.rs b/src/openhuman/memory_store/namespace_store/events.rs similarity index 100% rename from src/openhuman/memory_store/unified/events.rs rename to src/openhuman/memory_store/namespace_store/events.rs diff --git a/src/openhuman/memory_store/unified/events_tests.rs b/src/openhuman/memory_store/namespace_store/events_tests.rs similarity index 100% rename from src/openhuman/memory_store/unified/events_tests.rs rename to src/openhuman/memory_store/namespace_store/events_tests.rs diff --git a/src/openhuman/memory_store/unified/fts5.rs b/src/openhuman/memory_store/namespace_store/fts5.rs similarity index 100% rename from src/openhuman/memory_store/unified/fts5.rs rename to src/openhuman/memory_store/namespace_store/fts5.rs diff --git a/src/openhuman/memory_store/unified/graph.rs b/src/openhuman/memory_store/namespace_store/graph.rs similarity index 100% rename from src/openhuman/memory_store/unified/graph.rs rename to src/openhuman/memory_store/namespace_store/graph.rs diff --git a/src/openhuman/memory_store/unified/helpers.rs b/src/openhuman/memory_store/namespace_store/helpers.rs similarity index 100% rename from src/openhuman/memory_store/unified/helpers.rs rename to src/openhuman/memory_store/namespace_store/helpers.rs diff --git a/src/openhuman/memory_store/unified/init.rs b/src/openhuman/memory_store/namespace_store/init.rs similarity index 100% rename from src/openhuman/memory_store/unified/init.rs rename to src/openhuman/memory_store/namespace_store/init.rs diff --git a/src/openhuman/memory_store/unified/mod.rs b/src/openhuman/memory_store/namespace_store/mod.rs similarity index 100% rename from src/openhuman/memory_store/unified/mod.rs rename to src/openhuman/memory_store/namespace_store/mod.rs diff --git a/src/openhuman/memory_store/unified/profile.rs b/src/openhuman/memory_store/namespace_store/profile.rs similarity index 100% rename from src/openhuman/memory_store/unified/profile.rs rename to src/openhuman/memory_store/namespace_store/profile.rs diff --git a/src/openhuman/memory_store/unified/profile_tests.rs b/src/openhuman/memory_store/namespace_store/profile_tests.rs similarity index 100% rename from src/openhuman/memory_store/unified/profile_tests.rs rename to src/openhuman/memory_store/namespace_store/profile_tests.rs diff --git a/src/openhuman/memory_store/unified/query.rs b/src/openhuman/memory_store/namespace_store/query.rs similarity index 100% rename from src/openhuman/memory_store/unified/query.rs rename to src/openhuman/memory_store/namespace_store/query.rs diff --git a/src/openhuman/memory_store/unified/query_tests.rs b/src/openhuman/memory_store/namespace_store/query_tests.rs similarity index 100% rename from src/openhuman/memory_store/unified/query_tests.rs rename to src/openhuman/memory_store/namespace_store/query_tests.rs diff --git a/src/openhuman/memory_store/unified/segments.rs b/src/openhuman/memory_store/namespace_store/segments.rs similarity index 99% rename from src/openhuman/memory_store/unified/segments.rs rename to src/openhuman/memory_store/namespace_store/segments.rs index 8ad43d2b8..b15adca35 100644 --- a/src/openhuman/memory_store/unified/segments.rs +++ b/src/openhuman/memory_store/namespace_store/segments.rs @@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS conversation_segments ( status TEXT NOT NULL DEFAULT 'open', created_at REAL NOT NULL, updated_at REAL NOT NULL, - -- Per-session sequence numbers from memory_archivist::store, populated + -- Per-session sequence numbers from tinycortex::memory::archivist::store, populated -- alongside start_episodic_id / end_episodic_id during the FTS5 -> md -- migration. Once STM recall switches its segment-span dedup to use -- (session_id, seq) the legacy episodic_id columns can be dropped. @@ -108,7 +108,7 @@ pub struct ConversationSegment { pub status: SegmentStatus, pub created_at: f64, pub updated_at: f64, - /// Per-session seq number assigned by `memory_archivist::store::record_turn` + /// Per-session seq number assigned by the TinyCortex archivist store. /// for the user turn that opened this segment. `None` on legacy rows /// written before the FTS5 -> md migration began. #[serde(default)] diff --git a/src/openhuman/memory_store/unified/segments_tests.rs b/src/openhuman/memory_store/namespace_store/segments_tests.rs similarity index 100% rename from src/openhuman/memory_store/unified/segments_tests.rs rename to src/openhuman/memory_store/namespace_store/segments_tests.rs diff --git a/src/openhuman/memory_sync/README.md b/src/openhuman/memory_sync/README.md index 8825b9842..4f82ddadb 100644 --- a/src/openhuman/memory_sync/README.md +++ b/src/openhuman/memory_sync/README.md @@ -1,82 +1,28 @@ # memory_sync -Every "pull data from upstream → land it in memory_store" pipeline in -one place, organised by the kind of upstream they talk to. +OpenHuman orchestration and product policy around the TinyCortex sync engine. -## Three pipeline kinds +TinyCortex owns generic Composio provider fetch/pagination, canonical memory +records, sync budgets/state, workspace reconciliation, and persistence traits. +The live host path calls it through `src/openhuman/tinycortex/sync.rs` from +`composio::run_connection_sync` and the default provider `sync()` method. -| Kind | Submodule | Owns | -| --- | --- | --- | -| **Composio** | [`composio/`](composio/) | Per-provider sync via the Composio Edge API: gmail, slack, github, notion, linear, clickup, … | -| **Workspace** | [`workspace/`](workspace/) | Vault file watch, harness turn capture, dictation transcripts — anything local. | -| **MCP** | [`mcp/`](mcp/) | Third-party MCP servers via `mcp_clients/` transport. | +OpenHuman retains: -## Trait +- periodic scheduling and connection selection; +- credentials and Composio action execution; +- source-scope and redaction policy; +- translation into host `DomainEvent`s; +- JSON-RPC/status/connect surfaces; +- agent-facing action tools and result post-processing; +- product task/profile projections for GitHub, Notion, Linear, and ClickUp; +- local workspace watching and MCP orchestration. -Every pipeline implements [`SyncPipeline`]: +The provider directories therefore are not alternate sync engines. Their +remaining `provider.rs`, `tools.rs`, `normalization.rs`, profile, catalog, and +post-processing files implement host product surfaces over the crate-backed +sync path. New generic parsing or persistence behavior belongs in +`vendor/tinycortex/src/memory/sync/`. -```rust -async fn init(&self, &Config) -> anyhow::Result<()>; -async fn tick(&self, &Config) -> anyhow::Result; -fn id(&self) -> &str; -fn kind(&self) -> SyncPipelineKind; -``` - -`SyncOutcome { records_ingested, more_pending, note }` is the -orchestrator-facing result; pipelines own their own pagination cursors -and retry policy behind that. - -## Layout - -| Path | Role | -| --- | --- | -| [`mod.rs`](mod.rs) | Module root + re-exports. | -| [`traits.rs`](traits.rs) | `SyncPipeline`, `SyncOutcome`, `SyncPipelineKind`. | -| [`composio/`](composio/) | Per-provider pipelines (gmail, slack, github, notion, linear, clickup). | -| [`workspace/`](workspace/) | Vault, harness, dictation pipelines. | -| [`mcp/`](mcp/) | MCP-server pipelines (one per connected server). | - -## Schedulers & self-healing - -Three background loops keep memory synced and compressed without manual -"Sync now" clicks: - -| Loop | Module | Cadence | Covers | -| --- | --- | --- | --- | -| Composio periodic | [`composio/periodic.rs`](composio/periodic.rs) | 20-min tick, per-connection interval | Composio connections (gmail, slack, …) | -| Workspace periodic | [`workspace/periodic.rs`](workspace/periodic.rs) | 20-min tick, `memory_sync_interval_secs` (24h default, `0` = manual only) | `github_repo`, `folder`, `rss_feed`, `web_page` sources | -| Queue scheduler | `memory_queue::scheduler` | 3 h | flush stale L0 buffers + requeue transient-failed jobs | - -**Raw-archive coverage** ([`sources/rebuild.rs`](sources/rebuild.rs)): every -summary batch records its raw files in `mem_tree_ingested_sources` -(`source_kind = "raw_file"`). After each sync, `check_and_rebuild_tree` -diffs disk vs gate and summarises only the uncovered remainder — so an -interrupted sync self-heals on the next pass instead of stranding raw -files forever. Inspect/trigger over RPC with -`openhuman.memory_sources_reconcile`. - -⚠️ A source's **tree scope** and **raw-archive id** slugify to different -directories for GitHub (`github:owner/repo` vs `github.com/owner/repo`) — -always carry both (`memory_sources::sync::SourceScope`). - -## Status - -**Pipelines: scaffold only.** Today's sync code still lives in: - -- `composio/providers//ingest.rs` + `bin/{slack_backfill,gmail_backfill_3d}.rs` -- `vault/sync.rs`, `agent_experience/`, `dictation_hotkeys/` -- `mcp_clients/` (transport only; no drain loop yet) - -Each migrates here as its own per-pipeline PR. The job-queue orchestration -in `memory::jobs` stays put — it just gains the ability to iterate over a -registered `Vec>`. - -## Layer rules - -- Sync writes go through `memory::ingest_pipeline` so every record - lands as raw md → chunks → tree leaves like any other ingest. -- No direct writes into trees or unified. No upstream-specific data - models leak past the pipeline boundary. -- One pipeline per upstream service. Composio's GitHub and MCP's GitHub - are distinct pipelines because they hit different surfaces with - different cadence and auth. +D4.1-D4.4 are closed in `docs/tinycortex-drift-ledger.md`. Gmail's bounded +25-message page is crate-owned and prevents Composio 413 responses. diff --git a/src/openhuman/memory_sync/composio/providers/clickup/mod.rs b/src/openhuman/memory_sync/composio/providers/clickup/mod.rs index bbfd3a69f..fb4680e93 100644 --- a/src/openhuman/memory_sync/composio/providers/clickup/mod.rs +++ b/src/openhuman/memory_sync/composio/providers/clickup/mod.rs @@ -6,15 +6,15 @@ //! re-learning a new shape: //! //! - `provider.rs` — `impl ComposioProvider for ClickUpProvider` -//! - `sync.rs` — payload-shape helpers (results extraction, title) +//! - `normalization.rs` — payload-shape helpers (results extraction, title) //! - `ingest.rs` — memory_tree document ingest (issue #2885) //! - `tools.rs` — `CLICKUP_CURATED` whitelist of Composio actions //! - `tests.rs` — unit tests for the helpers + trait metadata //! //! Issue: #2288 (introduction); #2885 (memory_tree migration). +mod normalization; mod provider; -mod sync; #[cfg(test)] mod tests; pub mod tools; diff --git a/src/openhuman/memory_sync/composio/providers/clickup/sync.rs b/src/openhuman/memory_sync/composio/providers/clickup/normalization.rs similarity index 98% rename from src/openhuman/memory_sync/composio/providers/clickup/sync.rs rename to src/openhuman/memory_sync/composio/providers/clickup/normalization.rs index c7000c93b..fb0629019 100644 --- a/src/openhuman/memory_sync/composio/providers/clickup/sync.rs +++ b/src/openhuman/memory_sync/composio/providers/clickup/normalization.rs @@ -1,4 +1,4 @@ -//! ClickUp sync helpers — result extraction, task-title extraction, +//! ClickUp host normalization helpers — result extraction, task-title extraction, //! and time utilities. //! //! ClickUp's REST API (and therefore Composio's wrapping of it) returns diff --git a/src/openhuman/memory_sync/composio/providers/clickup/provider.rs b/src/openhuman/memory_sync/composio/providers/clickup/provider.rs index b31536aa2..7637d0159 100644 --- a/src/openhuman/memory_sync/composio/providers/clickup/provider.rs +++ b/src/openhuman/memory_sync/composio/providers/clickup/provider.rs @@ -27,7 +27,7 @@ use async_trait::async_trait; use serde_json::json; -use super::sync; +use super::normalization; use crate::openhuman::memory_sync::composio::providers::{ first_array_str, merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider, CuratedTool, NormalizedTask, ProviderContext, ProviderUserProfile, TaskFetchFilter, TaskKind, @@ -105,7 +105,7 @@ impl ComposioProvider for ClickUpProvider { let data = &resp.data; let display_name = pick_str(data, &["user.username", "data.user.username", "username"]); let email = pick_str(data, &["user.email", "data.user.email", "email"]); - let username = sync::extract_user_id(data); + let username = normalization::extract_user_id(data); let avatar_url = pick_str( data, &[ @@ -168,7 +168,7 @@ impl ComposioProvider for ClickUpProvider { resp.error.unwrap_or_else(|| "provider failure".into()) )); } - sync::extract_workspace_ids(&resp.data) + normalization::extract_workspace_ids(&resp.data) } }; @@ -188,7 +188,7 @@ impl ComposioProvider for ClickUpProvider { resp.error.unwrap_or_else(|| "provider failure".into()) )); } - let id = sync::extract_user_id(&resp.data).ok_or_else(|| { + let id = normalization::extract_user_id(&resp.data).ok_or_else(|| { "[composio:clickup] CLICKUP_GET_AUTHORIZED_USER returned no user.id".to_string() })?; vec![id] @@ -227,7 +227,7 @@ impl ComposioProvider for ClickUpProvider { )); } - for task in sync::extract_tasks(&resp.data) { + for task in normalization::extract_tasks(&resp.data) { if out.len() >= max { break 'workspaces; } @@ -246,8 +246,8 @@ impl ComposioProvider for ClickUpProvider { /// `None` only when the task has no extractable id (unroutable). fn normalize_clickup_task(task: &serde_json::Value) -> Option { let external_id = pick_str(task, TASK_ID_PATHS)?; - let title = - sync::extract_task_name(task).unwrap_or_else(|| format!("ClickUp task {external_id}")); + let title = normalization::extract_task_name(task) + .unwrap_or_else(|| format!("ClickUp task {external_id}")); Some(NormalizedTask { external_id, source_id: String::new(), @@ -265,7 +265,7 @@ fn normalize_clickup_task(task: &serde_json::Value) -> Option { due: pick_str(task, &["due_date", "data.due_date"]), labels: Vec::new(), priority: pick_str(task, &["priority.priority", "data.priority.priority"]), - updated_at: sync::extract_task_updated(task), + updated_at: normalization::extract_task_updated(task), raw: task.clone(), }) } diff --git a/src/openhuman/memory_sync/composio/providers/clickup/tests.rs b/src/openhuman/memory_sync/composio/providers/clickup/tests.rs index 6552e7fae..e6cd24967 100644 --- a/src/openhuman/memory_sync/composio/providers/clickup/tests.rs +++ b/src/openhuman/memory_sync/composio/providers/clickup/tests.rs @@ -1,6 +1,6 @@ //! Unit tests for the ClickUp provider. -use super::sync::{ +use super::normalization::{ extract_task_name, extract_task_updated, extract_tasks, extract_user_id, extract_workspace_ids, }; use super::ClickUpProvider; diff --git a/src/openhuman/memory_sync/composio/providers/github/mod.rs b/src/openhuman/memory_sync/composio/providers/github/mod.rs index 94c62d12f..29755ce55 100644 --- a/src/openhuman/memory_sync/composio/providers/github/mod.rs +++ b/src/openhuman/memory_sync/composio/providers/github/mod.rs @@ -6,14 +6,14 @@ //! re-learning a new shape: //! //! - `provider.rs` — `impl ComposioProvider for GitHubProvider` -//! - `sync.rs` — payload-shape helpers (result extraction, title, cursor) +//! - `normalization.rs` — payload-shape helpers (result extraction, title, cursor) //! - `tools.rs` — `GITHUB_CURATED` whitelist of Composio actions //! - `tests.rs` — unit tests for the helpers + trait metadata //! //! Issue: #2408. +mod normalization; mod provider; -mod sync; #[cfg(test)] mod tests; pub mod tools; diff --git a/src/openhuman/memory_sync/composio/providers/github/sync.rs b/src/openhuman/memory_sync/composio/providers/github/normalization.rs similarity index 98% rename from src/openhuman/memory_sync/composio/providers/github/sync.rs rename to src/openhuman/memory_sync/composio/providers/github/normalization.rs index e17ad23a9..ccd52770f 100644 --- a/src/openhuman/memory_sync/composio/providers/github/sync.rs +++ b/src/openhuman/memory_sync/composio/providers/github/normalization.rs @@ -1,4 +1,4 @@ -//! GitHub sync helpers — result extraction, identity helpers, and time utilities. +//! GitHub host normalization helpers — result extraction, identity helpers, and time utilities. //! //! GitHub's REST API (proxied through Composio) returns search results and //! authenticated-user payloads in a small number of shapes. The functions here diff --git a/src/openhuman/memory_sync/composio/providers/github/provider.rs b/src/openhuman/memory_sync/composio/providers/github/provider.rs index caea4a951..6d1d1b912 100644 --- a/src/openhuman/memory_sync/composio/providers/github/provider.rs +++ b/src/openhuman/memory_sync/composio/providers/github/provider.rs @@ -23,7 +23,7 @@ use async_trait::async_trait; use serde_json::{json, Value}; use std::time::Duration; -use super::sync; +use super::normalization; use crate::openhuman::memory_sync::composio::providers::{ merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider, CuratedTool, GithubFetchMode, NormalizedTask, ProviderContext, ProviderUserProfile, TaskFetchFilter, @@ -94,7 +94,7 @@ impl ComposioProvider for GitHubProvider { } let data = &resp.data; - let login = sync::extract_user_login(data); + let login = normalization::extract_user_login(data); let display_name = pick_str(data, &["name", "data.name"]).or_else(|| login.clone()); let email = pick_str(data, &["email", "data.email"]); let avatar_url = pick_str(data, &["avatar_url", "data.avatar_url"]); @@ -157,7 +157,7 @@ impl ComposioProvider for GitHubProvider { }; let mut out: Vec = Vec::new(); - for issue in sync::extract_issues(&data) { + for issue in normalization::extract_issues(&data) { if out.len() >= max { break; } @@ -481,7 +481,7 @@ pub(super) fn normalize_github_repo_filter(raw: &str) -> String { /// not depend on the fetch query), so even if a `closed` item slips through /// the query bias it is dropped here. pub(super) fn normalize_github_issue(issue: &serde_json::Value) -> Option { - let external_id = sync::extract_issue_id(issue)?; + let external_id = normalization::extract_issue_id(issue)?; let status = pick_str(issue, &["state", "data.state"]); if status .as_deref() @@ -494,8 +494,8 @@ pub(super) fn normalize_github_issue(issue: &serde_json::Value) -> Option Option Vec { - let candidates = [ - data.pointer("/data/messages"), - data.pointer("/messages"), - data.pointer("/data/data/messages"), - data.pointer("/data/items"), - data.pointer("/items"), - ]; - for cand in candidates.into_iter().flatten() { - if let Some(arr) = cand.as_array() { - return arr.clone(); - } - } - Vec::new() -} - -/// Try to extract a pagination token from the API response. -pub(crate) fn extract_page_token(data: &Value) -> Option { - let candidates = [ - data.pointer("/data/nextPageToken"), - data.pointer("/nextPageToken"), - data.pointer("/data/data/nextPageToken"), - ]; - for cand in candidates.into_iter().flatten() { - if let Some(s) = cand.as_str() { - let trimmed = s.trim(); - if !trimmed.is_empty() { - return Some(trimmed.to_string()); - } - } - } - None -} - -/// Convert a cursor value (epoch millis or date string) into a Gmail -/// `after:YYYY/MM/DD` filter component. Day-level precision — kept as a -/// last-resort filter for non-numeric cursors and for back-compat with -/// the older day-cursor write path. New code should prefer -/// [`cursor_to_gmail_after_epoch_filter`] which produces a -/// second-precision `after:` filter and so avoids re-fetching -/// large same-day windows on every tick. -pub(crate) fn cursor_to_gmail_after_filter(cursor: &str) -> Option { - let cursor = cursor.trim(); - // Try parsing as epoch millis first (Gmail's internalDate). - if let Ok(millis) = cursor.parse::() { - let secs = millis / 1000; - if let Some(dt) = chrono::DateTime::from_timestamp(secs, 0) { - return Some(dt.format("%Y/%m/%d").to_string()); - } - } - // Try parsing as an ISO date/datetime. - if let Ok(dt) = chrono::NaiveDate::parse_from_str(cursor, "%Y-%m-%d") { - return Some(dt.format("%Y/%m/%d").to_string()); - } - if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(cursor) { - return Some(dt.format("%Y/%m/%d").to_string()); - } - None -} - -/// Convert a cursor value (epoch millis or date string) into a Gmail -/// `after:` filter component. Gmail's search syntax -/// accepts a bare unix-seconds value for `after:` / `before:`, so the -/// filter is second-precision rather than the day-level -/// `after:YYYY/MM/DD` form. Used by the incremental sync path so a -/// same-day re-tick does not re-fetch every message Gmail has filed -/// today. -/// -/// Returns `None` when the cursor cannot be parsed; callers should -/// fall back to the coarse day filter to avoid sending an unbounded -/// query. -pub(crate) fn cursor_to_gmail_after_epoch_filter(cursor: &str) -> Option { - let secs = parse_cursor_to_epoch_secs(cursor)?; - Some(secs.to_string()) -} - -/// Parse a cursor (epoch millis as string, `YYYY-MM-DD`, or RFC3339) -/// into unix-seconds. Shared by the epoch filter and by the adaptive -/// page-cap recency check. -pub(crate) fn parse_cursor_to_epoch_secs(cursor: &str) -> Option { - let cursor = cursor.trim(); - if let Ok(millis) = cursor.parse::() { - return Some(millis / 1000); - } - if let Ok(date) = chrono::NaiveDate::parse_from_str(cursor, "%Y-%m-%d") { - return date.and_hms_opt(0, 0, 0).map(|dt| dt.and_utc().timestamp()); - } - if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(cursor) { - return Some(dt.timestamp()); - } - None -} - -pub(crate) fn now_ms() -> u64 { - use std::time::{SystemTime, UNIX_EPOCH}; - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn extract_messages_from_data_messages() { - let data = json!({"data": {"messages": [{"id": "1"}, {"id": "2"}]}}); - let msgs = extract_messages(&data); - assert_eq!(msgs.len(), 2); - } - - #[test] - fn extract_messages_from_top_level() { - let data = json!({"messages": [{"id": "1"}]}); - let msgs = extract_messages(&data); - assert_eq!(msgs.len(), 1); - } - - #[test] - fn extract_messages_from_data_items() { - let data = json!({"data": {"items": [{"id": "a"}]}}); - let msgs = extract_messages(&data); - assert_eq!(msgs.len(), 1); - } - - #[test] - fn extract_messages_empty_when_no_match() { - let data = json!({"foo": "bar"}); - assert!(extract_messages(&data).is_empty()); - } - - #[test] - fn extract_page_token_from_data() { - let data = json!({"data": {"nextPageToken": "abc123"}}); - assert_eq!(extract_page_token(&data), Some("abc123".into())); - } - - #[test] - fn extract_page_token_from_top_level() { - let data = json!({"nextPageToken": "tok"}); - assert_eq!(extract_page_token(&data), Some("tok".into())); - } - - #[test] - fn extract_page_token_none_when_empty() { - let data = json!({"data": {"nextPageToken": " "}}); - assert_eq!(extract_page_token(&data), None); - } - - #[test] - fn extract_page_token_none_when_missing() { - let data = json!({"data": {}}); - assert_eq!(extract_page_token(&data), None); - } - - #[test] - fn cursor_to_filter_epoch_millis() { - let filter = cursor_to_gmail_after_filter("1700000000000").unwrap(); - assert!(filter.contains('/')); - assert_eq!(filter, "2023/11/14"); - } - - #[test] - fn cursor_to_filter_iso_date() { - let filter = cursor_to_gmail_after_filter("2024-01-15").unwrap(); - assert_eq!(filter, "2024/01/15"); - } - - #[test] - fn cursor_to_filter_rfc3339() { - let filter = cursor_to_gmail_after_filter("2024-06-01T12:00:00Z").unwrap(); - assert_eq!(filter, "2024/06/01"); - } - - #[test] - fn cursor_to_filter_invalid_returns_none() { - assert!(cursor_to_gmail_after_filter("not-a-date").is_none()); - } - - #[test] - fn cursor_to_filter_trims_whitespace() { - let filter = cursor_to_gmail_after_filter(" 2024-01-15 ").unwrap(); - assert_eq!(filter, "2024/01/15"); - } - - #[test] - fn now_ms_returns_nonzero() { - assert!(now_ms() > 0); - } - - // ── second-precision cursor ────────────────────────────────── - - #[test] - fn epoch_filter_emits_unix_seconds_for_internal_date_millis() { - let filter = cursor_to_gmail_after_epoch_filter("1700000000000").unwrap(); - assert_eq!(filter, "1700000000"); - } - - #[test] - fn epoch_filter_handles_iso_date() { - let filter = cursor_to_gmail_after_epoch_filter("2024-01-15").unwrap(); - // 2024-01-15 00:00:00 UTC == 1705276800. - assert_eq!(filter, "1705276800"); - } - - #[test] - fn epoch_filter_handles_rfc3339() { - let filter = cursor_to_gmail_after_epoch_filter("2024-06-01T12:00:00Z").unwrap(); - assert_eq!(filter, "1717243200"); - } - - #[test] - fn epoch_filter_returns_none_for_garbage() { - assert!(cursor_to_gmail_after_epoch_filter("not-a-date").is_none()); - } - - #[test] - fn epoch_filter_trims_whitespace() { - let filter = cursor_to_gmail_after_epoch_filter(" 2024-01-15 ").unwrap(); - assert_eq!(filter, "1705276800"); - } - - #[test] - fn parse_cursor_round_trip_matches_epoch_filter() { - // The adaptive page-cap relies on parse_cursor_to_epoch_secs - // agreeing with cursor_to_gmail_after_epoch_filter — both must - // emit the same seconds value for any given input. - for cursor in ["1700000000000", "2024-01-15", "2024-06-01T12:00:00Z"] { - let secs = parse_cursor_to_epoch_secs(cursor).unwrap(); - let filter = cursor_to_gmail_after_epoch_filter(cursor).unwrap(); - assert_eq!(filter, secs.to_string(), "cursor `{cursor}`"); - } - } -} diff --git a/src/openhuman/memory_sync/composio/providers/gmail/tests.rs b/src/openhuman/memory_sync/composio/providers/gmail/tests.rs index fe0ab84f3..b04045539 100644 --- a/src/openhuman/memory_sync/composio/providers/gmail/tests.rs +++ b/src/openhuman/memory_sync/composio/providers/gmail/tests.rs @@ -1,301 +1,46 @@ -//! Unit tests for the Gmail provider. +//! Host-owned Gmail provider surface tests. +//! +//! Pagination, cursor, envelope parsing, and ingest behavior are owned and +//! tested by `tinycortex::memory::sync::GmailSyncPipeline`. use super::provider::{BASE_QUERY, SENT_QUERIES}; -use super::sync::{ - cursor_to_gmail_after_epoch_filter, cursor_to_gmail_after_filter, extract_messages, - extract_page_token, now_ms, parse_cursor_to_epoch_secs, -}; use super::GmailProvider; use crate::openhuman::memory_sync::composio::providers::ComposioProvider; -use serde_json::json; - -#[test] -fn extract_messages_finds_data_messages() { - let v = json!({ - "data": { "messages": [{"id": "m1"}, {"id": "m2"}] }, - "successful": true, - }); - assert_eq!(extract_messages(&v).len(), 2); -} - -#[test] -fn extract_messages_finds_top_level_messages() { - let v = json!({ "messages": [{"id": "m1"}] }); - assert_eq!(extract_messages(&v).len(), 1); -} - -#[test] -fn extract_messages_returns_empty_when_missing() { - let v = json!({ "data": { "other": [] } }); - assert_eq!(extract_messages(&v).len(), 0); -} - -#[test] -fn extract_page_token_finds_nested() { - let v = json!({ "data": { "nextPageToken": "tok123" } }); - assert_eq!(extract_page_token(&v), Some("tok123".to_string())); -} - -#[test] -fn extract_page_token_none_when_missing() { - let v = json!({ "data": {} }); - assert_eq!(extract_page_token(&v), None); -} - -#[test] -fn cursor_to_filter_from_epoch_millis() { - // 1774915200000 ms = 2026-03-31 UTC - let millis = "1774915200000"; - assert_eq!( - cursor_to_gmail_after_filter(millis), - Some("2026/03/31".to_string()) - ); -} - -#[test] -fn cursor_to_filter_from_iso_date() { - assert_eq!( - cursor_to_gmail_after_filter("2026-03-15"), - Some("2026/03/15".to_string()) - ); -} - -#[test] -fn cursor_to_filter_from_rfc3339() { - let f = cursor_to_gmail_after_filter("2026-03-15T12:00:00Z"); - assert_eq!(f, Some("2026/03/15".to_string())); -} - -#[test] -fn cursor_to_filter_returns_none_for_garbage() { - assert_eq!(cursor_to_gmail_after_filter("not-a-date"), None); -} #[test] fn provider_metadata_is_stable() { - let p = GmailProvider::new(); - assert_eq!(p.toolkit_slug(), "gmail"); - assert_eq!(p.sync_interval_secs(), Some(15 * 60)); + let provider = GmailProvider::new(); + assert_eq!(provider.toolkit_slug(), "gmail"); + assert_eq!(provider.sync_interval_secs(), Some(15 * 60)); } #[test] fn default_impl_matches_new() { - let _a = GmailProvider::new(); - let _b = GmailProvider::default(); - // Both are unit structs — constructing via Default is the cover target. + let _new = GmailProvider::new(); + let _default = GmailProvider::default(); } -#[test] -fn epoch_filter_is_preferred_over_day_filter_for_typical_internal_date() { - // The provider tries `cursor_to_gmail_after_epoch_filter` first - // and only falls back to the day filter if the parse fails. Both - // helpers must accept the same internalDate (epoch millis) input - // so the fallback path is genuinely a fallback, not a divergence. - let internal_date = "1774915200000"; // 2026-03-31 UTC - let epoch = cursor_to_gmail_after_epoch_filter(internal_date).unwrap(); - let day = cursor_to_gmail_after_filter(internal_date).unwrap(); - assert_eq!(epoch, "1774915200"); - assert_eq!(day, "2026/03/31"); - // Sanity bound: the epoch filter must be after 2020 and before - // year 2100, otherwise we shipped a regression in the cursor - // converter that would silently let queries land on year 1970. - let secs: i64 = epoch.parse().unwrap(); - assert!( - secs > 1_577_836_800, - "epoch filter must be after 2020-01-01" - ); - assert!( - secs < 4_102_444_800, - "epoch filter must be before 2100-01-01" - ); -} - -// ── Adaptive page cap and early-stop helpers (issue#1404, pr#1474) ────────── -// -// The full `sync()` path needs a live ComposioClient + MemoryClient, so -// we test the helper functions that gate the adaptive cap and early-stop -// decisions: -// -// * `parse_cursor_to_epoch_secs` — used to decide whether `last_sync_at_ms` -// falls within `RECENT_SYNC_WINDOW_MS` (5 min) for the adaptive page cap. -// * `now_ms` — sanity check: must not return 0 and must be within a plausible -// range so the adaptive window comparison never produces pathological results. -// * early-stop guard: when `last_seen_id` matches the first page's head id -// the sync loop breaks with `stop_reason = "head_unchanged"`. We pin the -// helper logic that feeds this decision. - -#[test] -fn parse_cursor_to_epoch_secs_handles_epoch_millis() { - // Gmail internalDate is epoch milliseconds as a numeric string. - // 1774915200000 ms = 1774915200 s (2026-03-31 00:00:00 UTC). - assert_eq!( - parse_cursor_to_epoch_secs("1774915200000"), - Some(1774915200) - ); -} - -#[test] -fn parse_cursor_to_epoch_secs_handles_iso_date() { - // YYYY-MM-DD date cursor produced by the older day-cursor write path. - let secs = parse_cursor_to_epoch_secs("2024-01-15").unwrap(); - // 2024-01-15 00:00:00 UTC = 1705276800 - assert_eq!(secs, 1705276800); -} - -#[test] -fn parse_cursor_to_epoch_secs_handles_rfc3339() { - let secs = parse_cursor_to_epoch_secs("2024-01-15T00:00:00Z").unwrap(); - assert_eq!(secs, 1705276800); -} - -#[test] -fn parse_cursor_to_epoch_secs_returns_none_for_garbage() { - assert_eq!(parse_cursor_to_epoch_secs("not-a-timestamp"), None); - assert_eq!(parse_cursor_to_epoch_secs(""), None); - assert_eq!(parse_cursor_to_epoch_secs(" "), None); -} - -/// The adaptive page cap relies on `parse_cursor_to_epoch_secs` and `now_ms` -/// agreeing on a common epoch so the "less than 5 min ago" comparison works. -/// `now_ms()` must return epoch-milliseconds (not zero, not micros). If it -/// returned microseconds, every sync would appear "recent" (delta < 300_000 ms -/// vs delta actually being ~ 1e12 µs); if it returned seconds, every sync -/// would appear "old" (delta > 300_000 ms trivially). -#[test] -fn now_ms_is_in_epoch_milliseconds_range() { - let ms = now_ms(); - // Must be strictly positive. - assert!(ms > 0, "now_ms must not return zero"); - // Must be > 2024-01-01 00:00:00 UTC in milliseconds so it's clearly - // millisecond-epoch and not seconds-epoch (which would be ~1.7e9, much - // smaller than 1.7e12). - let jan_2024_ms: u64 = 1_704_067_200_000; - assert!( - ms > jan_2024_ms, - "now_ms ({ms}) must be above 2024-01-01 in epoch-millisecond scale" - ); - // Must be < year 2100 in milliseconds — rules out microseconds/nanoseconds. - let year_2100_ms: u64 = 4_102_444_800_000; - assert!( - ms < year_2100_ms, - "now_ms ({ms}) must be below year 2100 in epoch-millisecond scale" - ); -} - -/// The early-stop optimisation fires when `last_seen_id` equals the first -/// message id on the first page. We test the helper that extracts message ids -/// — `extract_messages` — to verify it correctly surfaces the `id` field so -/// the comparison in the sync loop gets the right value. -/// -/// The early-stop check uses `messages.first()` with the `MESSAGE_ID_PATHS` -/// extractor. We can't call the private extractor, but we can pin -/// `extract_messages` to return messages with their `id` intact so the -/// sync loop can compare them to `state.last_seen_id`. -#[test] -fn extract_messages_preserves_id_field_for_early_stop() { - // The early-stop check reads `m["id"]` via `extract_item_id`. Verify - // `extract_messages` doesn't strip or transform the field. - let v = json!({ - "data": { - "messages": [ - {"id": "msg_abc", "internalDate": "1774915200000"}, - {"id": "msg_def", "internalDate": "1774915100000"} - ] - }, - "successful": true - }); - let msgs = extract_messages(&v); - assert_eq!(msgs.len(), 2); - assert_eq!( - msgs[0]["id"], "msg_abc", - "first message id must be preserved" - ); - assert_eq!( - msgs[1]["id"], "msg_def", - "second message id must be preserved" - ); -} - -/// Variant: messages embedded in `data.data.messages` (deeper nesting -/// seen in some Composio provider responses) — the extractor must still -/// find them so the early-stop comparison has data to work with. -#[test] -fn extract_messages_handles_deep_nesting() { - let v = json!({ - "data": { - "data": { - "messages": [ - {"id": "deep_msg_1"} - ] - } - } - }); - let msgs = extract_messages(&v); - assert_eq!(msgs.len(), 1); - assert_eq!(msgs[0]["id"], "deep_msg_1"); -} - -// Note: full `sync` / `fetch_user_profile` / `on_trigger` paths require a -// live `ComposioClient` (HTTP) plus the global `MemoryClient` singleton. -// Those go through the integration test suite. Here we just lock in -// the provider's identity surface and helpers. - -// ── Regression tests for issue #1713: sent-mail retrieval ─────────────────── -// -// Before the fix the sync query was `in:inbox -in:spam -in:trash`, which meant -// sent emails (label:SENT) were never fetched or ingested into the memory tree. -// The fix removes `in:inbox` so both inbox and sent mail are fetched. - -/// Guard: the provider source must NOT contain the `in:inbox` restriction. -/// If this test fails, someone reintroduced the inbox-only query that caused -/// issue #1713. #[test] fn provider_source_does_not_restrict_to_inbox() { let source = include_str!("provider.rs"); - // The old restriction started with `"in:inbox`. The double-quote is part - // of the Rust string literal, so this catches the exact regression pattern. assert!( !source.contains("\"in:inbox"), - "provider.rs sync query must NOT start with 'in:inbox' — this \ - restriction prevents sent emails from being ingested (issue #1713). \ - Use '-in:spam -in:trash' to allow both inbox and sent mail." + "provider query must not exclude sent mail" ); } -/// The base sync query must exclude spam and trash but NOT restrict to inbox. -/// Asserts against the canonical `BASE_QUERY` constant from provider.rs so -/// any change to the production value is caught immediately. #[test] -fn sync_base_query_excludes_spam_and_trash_without_inbox_restriction() { - assert!( - BASE_QUERY.contains("-in:spam"), - "BASE_QUERY must exclude spam (got: {BASE_QUERY:?})" - ); - assert!( - BASE_QUERY.contains("-in:trash"), - "BASE_QUERY must exclude trash (got: {BASE_QUERY:?})" - ); - assert!( - !BASE_QUERY.contains("in:inbox"), - "BASE_QUERY must NOT restrict to inbox — omitting this allows sent \ - mail to be fetched and ingested (issue #1713). Got: {BASE_QUERY:?}" - ); +fn base_query_excludes_spam_and_trash_without_inbox_restriction() { + assert!(BASE_QUERY.contains("-in:spam")); + assert!(BASE_QUERY.contains("-in:trash")); + assert!(!BASE_QUERY.contains("in:inbox")); } -/// Sent-mail query strings must be non-empty and must not restrict to inbox. -/// Iterates `SENT_QUERIES` from provider.rs — the canonical list of query -/// strings a user or agent can pass to GMAIL_FETCH_EMAILS for sent mail. #[test] fn sent_mail_query_strings_are_well_formed() { - assert!(!SENT_QUERIES.is_empty(), "SENT_QUERIES must not be empty"); - for q in SENT_QUERIES { - assert!( - !q.is_empty(), - "sent-mail query must not be empty, got empty string in SENT_QUERIES" - ); - assert!( - !q.starts_with("in:inbox"), - "sent-mail query '{q}' must not restrict to inbox" - ); + assert!(!SENT_QUERIES.is_empty()); + for query in SENT_QUERIES { + assert!(!query.is_empty()); + assert!(!query.starts_with("in:inbox")); } } diff --git a/src/openhuman/memory_sync/composio/providers/linear/mod.rs b/src/openhuman/memory_sync/composio/providers/linear/mod.rs index d744dbe6a..971e9e4db 100644 --- a/src/openhuman/memory_sync/composio/providers/linear/mod.rs +++ b/src/openhuman/memory_sync/composio/providers/linear/mod.rs @@ -3,8 +3,8 @@ //! //! Issue: #2400. +mod normalization; mod provider; -mod sync; #[cfg(test)] mod tests; pub mod tools; diff --git a/src/openhuman/memory_sync/composio/providers/linear/sync.rs b/src/openhuman/memory_sync/composio/providers/linear/normalization.rs similarity index 99% rename from src/openhuman/memory_sync/composio/providers/linear/sync.rs rename to src/openhuman/memory_sync/composio/providers/linear/normalization.rs index 952943e3b..59100ff37 100644 --- a/src/openhuman/memory_sync/composio/providers/linear/sync.rs +++ b/src/openhuman/memory_sync/composio/providers/linear/normalization.rs @@ -1,4 +1,4 @@ -//! Linear sync helpers — result extraction, issue-title extraction, +//! Linear host normalization helpers — result extraction, issue-title extraction, //! viewer identity, cursor extraction, and time utilities. //! //! Linear's GraphQL API (and therefore Composio's wrapping of it) returns diff --git a/src/openhuman/memory_sync/composio/providers/linear/provider.rs b/src/openhuman/memory_sync/composio/providers/linear/provider.rs index 41547ad59..5f207e9c4 100644 --- a/src/openhuman/memory_sync/composio/providers/linear/provider.rs +++ b/src/openhuman/memory_sync/composio/providers/linear/provider.rs @@ -21,7 +21,7 @@ use async_trait::async_trait; use serde_json::json; -use super::sync; +use super::normalization; use crate::openhuman::memory_sync::composio::providers::{ merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider, CuratedTool, NormalizedTask, ProviderContext, ProviderUserProfile, TaskFetchFilter, TaskKind, @@ -86,7 +86,7 @@ impl ComposioProvider for LinearProvider { } let data = &resp.data; - let viewer = sync::extract_viewer(data); + let viewer = normalization::extract_viewer(data); let viewer_ref = viewer.as_ref().unwrap_or(data); let display_name = pick_str(viewer_ref, &["name", "data.name", "displayName"]); @@ -143,7 +143,7 @@ impl ComposioProvider for LinearProvider { resp.error.unwrap_or_else(|| "provider failure".into()) )); } - let viewer_id = sync::extract_viewer_id(&resp.data).ok_or_else(|| { + let viewer_id = normalization::extract_viewer_id(&resp.data).ok_or_else(|| { "[composio:linear] LINEAR_LIST_LINEAR_USERS returned no viewer id".to_string() })?; args["assigneeId"] = json!(viewer_id); @@ -176,7 +176,7 @@ impl ComposioProvider for LinearProvider { .filter(|s| !s.is_empty()); let mut out: Vec = Vec::new(); - for issue in sync::extract_issues(&resp.data) { + for issue in normalization::extract_issues(&resp.data) { if out.len() >= max { break; } @@ -203,8 +203,8 @@ impl ComposioProvider for LinearProvider { /// Map a raw Linear issue payload into a [`NormalizedTask`]. fn normalize_linear_issue(issue: &serde_json::Value) -> Option { let external_id = pick_str(issue, ISSUE_ID_PATHS)?; - let title = - sync::extract_issue_title(issue).unwrap_or_else(|| format!("Linear issue {external_id}")); + let title = normalization::extract_issue_title(issue) + .unwrap_or_else(|| format!("Linear issue {external_id}")); Some(NormalizedTask { external_id, source_id: String::new(), @@ -218,7 +218,7 @@ fn normalize_linear_issue(issue: &serde_json::Value) -> Option { due: pick_str(issue, &["dueDate", "data.dueDate"]), labels: extract_linear_labels(issue), priority: pick_str(issue, &["priorityLabel", "data.priorityLabel"]), - updated_at: sync::extract_issue_updated(issue), + updated_at: normalization::extract_issue_updated(issue), raw: issue.clone(), }) } diff --git a/src/openhuman/memory_sync/composio/providers/linear/tests.rs b/src/openhuman/memory_sync/composio/providers/linear/tests.rs index ed605c720..d8a805013 100644 --- a/src/openhuman/memory_sync/composio/providers/linear/tests.rs +++ b/src/openhuman/memory_sync/composio/providers/linear/tests.rs @@ -1,6 +1,6 @@ //! Unit tests for the Linear provider. -use super::sync::{ +use super::normalization::{ extract_issue_title, extract_issue_updated, extract_issues, extract_pagination_cursor, extract_viewer, extract_viewer_id, }; diff --git a/src/openhuman/memory_sync/composio/providers/notion/mod.rs b/src/openhuman/memory_sync/composio/providers/notion/mod.rs index 427b6fb3f..07d2a73e3 100644 --- a/src/openhuman/memory_sync/composio/providers/notion/mod.rs +++ b/src/openhuman/memory_sync/composio/providers/notion/mod.rs @@ -1,5 +1,5 @@ +mod normalization; mod provider; -mod sync; #[cfg(test)] mod tests; pub mod tools; diff --git a/src/openhuman/memory_sync/composio/providers/notion/sync.rs b/src/openhuman/memory_sync/composio/providers/notion/normalization.rs similarity index 99% rename from src/openhuman/memory_sync/composio/providers/notion/sync.rs rename to src/openhuman/memory_sync/composio/providers/notion/normalization.rs index 79a8729cd..a3ff5745e 100644 --- a/src/openhuman/memory_sync/composio/providers/notion/sync.rs +++ b/src/openhuman/memory_sync/composio/providers/notion/normalization.rs @@ -1,4 +1,4 @@ -//! Notion sync helpers — result extraction, pagination cursor, +//! Notion host normalization helpers — result extraction, pagination cursor, //! page title extraction, and time utilities. use serde_json::Value; diff --git a/src/openhuman/memory_sync/composio/providers/notion/provider.rs b/src/openhuman/memory_sync/composio/providers/notion/provider.rs index 1006a0d6a..fcbfb6dce 100644 --- a/src/openhuman/memory_sync/composio/providers/notion/provider.rs +++ b/src/openhuman/memory_sync/composio/providers/notion/provider.rs @@ -18,7 +18,7 @@ use async_trait::async_trait; use serde_json::{json, Value}; -use super::sync; +use super::normalization; use crate::openhuman::memory_sync::composio::providers::{ first_array_str, merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider, CuratedTool, NormalizedTask, ProviderContext, ProviderUserProfile, TaskContainer, @@ -194,7 +194,7 @@ impl ComposioProvider for NotionProvider { .filter(|s| !s.is_empty()); let mut out: Vec = Vec::new(); - for page in sync::extract_results(&resp.data) { + for page in normalization::extract_results(&resp.data) { if out.len() >= max { break; } @@ -302,8 +302,8 @@ impl ComposioProvider for NotionProvider { /// preserved for enrichment. fn normalize_notion_page(page: &serde_json::Value) -> Option { let external_id = pick_str(page, PAGE_ID_PATHS)?; - let title = - sync::extract_page_title(page).unwrap_or_else(|| format!("Notion page {external_id}")); + let title = normalization::extract_page_title(page) + .unwrap_or_else(|| format!("Notion page {external_id}")); Some(NormalizedTask { external_id, source_id: String::new(), @@ -358,7 +358,7 @@ fn normalize_notion_page(page: &serde_json::Value) -> Option { /// "keep only object==database" check silently dropped every database. /// Pure (no I/O) so it is unit-testable. pub(super) fn parse_database_results(data: &serde_json::Value) -> Vec { - let results = sync::extract_results(data); + let results = normalization::extract_results(data); let mut kinds: std::collections::BTreeMap = std::collections::BTreeMap::new(); let mut out: Vec = Vec::new(); for item in &results { diff --git a/src/openhuman/memory_sync/composio/providers/notion/tests.rs b/src/openhuman/memory_sync/composio/providers/notion/tests.rs index 880b08e41..8771301aa 100644 --- a/src/openhuman/memory_sync/composio/providers/notion/tests.rs +++ b/src/openhuman/memory_sync/composio/providers/notion/tests.rs @@ -1,6 +1,6 @@ //! Unit tests for the Notion provider. -use super::sync::{extract_notion_cursor, extract_page_title, extract_results}; +use super::normalization::{extract_notion_cursor, extract_page_title, extract_results}; use super::NotionProvider; use crate::openhuman::memory_sync::composio::providers::ComposioProvider; use serde_json::json; diff --git a/src/openhuman/memory_sync/workspace/mod.rs b/src/openhuman/memory_sync/workspace/mod.rs index ea73b4329..557083bae 100644 --- a/src/openhuman/memory_sync/workspace/mod.rs +++ b/src/openhuman/memory_sync/workspace/mod.rs @@ -6,7 +6,7 @@ //! | Submodule | Source | Notes | //! | --- | --- | --- | //! | `folder` | Files under a user-added folder memory source | Watch + diff | -//! | `harness` | Agent harness turns (memory_archivist's caller side) | Push-based | +//! | `harness` | Agent harness turns (TinyCortex archivist caller side) | Push-based | //! | `dictation` | Local audio capture transcripts | Push-based | //! //! ## Status diff --git a/src/openhuman/memory_tools/README.md b/src/openhuman/memory_tools/README.md index 5c7001853..cab74b568 100644 --- a/src/openhuman/memory_tools/README.md +++ b/src/openhuman/memory_tools/README.md @@ -5,16 +5,15 @@ from generic namespace memory and from `learning::tool_tracker` statistics. ## Namespace convention -Each tool gets its own namespace `tool-{tool_name}`. Build the string via -[`types::tool_memory_namespace`] — never hard-code it. +Each tool gets its own namespace `tool-{tool_name}`. Build the string via the +`tool_memory_namespace` re-export — never hard-code it. ## Layout | Path | Role | | --- | --- | | [`mod.rs`](mod.rs) | Module root + public re-exports. | -| [`types.rs`](types.rs) | **Shim** — re-exports `ToolMemoryRule` / `ToolMemoryPriority` / `ToolMemorySource` / `tool_memory_namespace` from `tinycortex::memory::tool_memory::types` (W7). | -| [`store.rs`](store.rs) | **Shim** — re-exports the crate `ToolMemoryStore` (`put_rule`, `get_rule`, `list_rules`, `delete_rule`, `rules_for_prompt`, `list_tool_names`, `record`, `list_rules_json`) + `tool_memory_store(Arc)`. The trait and engine are both crate-owned. | +| [`mod.rs`](mod.rs) | Re-exports crate types/store and defines `tool_memory_store(Arc)`. | | [`capture.rs`](capture.rs) | `ToolMemoryCaptureHook` — `PostTurnHook` impl that captures user edicts and repeated tool failures into the store (host-retained). | | [`prompt.rs`](prompt.rs) | **Shim** — re-exports the crate `ToolMemoryRulesSection` + `render_tool_memory_rules` + `TOOL_MEMORY_HEADING`, and keeps the host `PromptSection` impl that plugs the section into the system-prompt builder. | | [`tools/`](tools/) | Agent-facing read/write tools: `MemoryToolsListTool` (list rules for a tool), `MemoryToolsPutTool` (upsert a rule). | @@ -35,6 +34,5 @@ The agent harness: - No upward dependencies — only `memory::Memory` trait (via `Arc`) and project-wide primitives (`tools::traits::Tool`, `serde_json`). - `MockMemory` is `#[cfg(test)]`-only — never available outside test builds. -- Re-exports in `mod.rs` are the public surface; the underlying submodules - are `pub` so test code can reach in but consumers should go through the - re-exports. +- Re-exports in `mod.rs` are the public surface. Crate-owned type and store + forwarding files were removed; consumers should use the domain root. diff --git a/src/openhuman/memory_tools/capture.rs b/src/openhuman/memory_tools/capture.rs index b885f86ed..ce6cd500d 100644 --- a/src/openhuman/memory_tools/capture.rs +++ b/src/openhuman/memory_tools/capture.rs @@ -33,8 +33,7 @@ use std::sync::Arc; use async_trait::async_trait; -use super::store::{tool_memory_store, ToolMemoryStore}; -use super::types::{ToolMemoryPriority, ToolMemorySource}; +use super::{tool_memory_store, ToolMemoryPriority, ToolMemorySource, ToolMemoryStore}; use crate::openhuman::agent::hooks::{PostTurnHook, ToolCallRecord, TurnContext}; use crate::openhuman::memory::Memory; @@ -291,8 +290,8 @@ fn tool_aliases(tool_name: &str) -> Vec<&'static str> { mod tests { use super::*; use crate::openhuman::agent::hooks::ToolCallRecord; - use crate::openhuman::memory_tools::store::tool_memory_store; use crate::openhuman::memory_tools::test_helpers::MockMemory; + use crate::openhuman::memory_tools::tool_memory_store; fn ctx_with(message: &str, tool_calls: Vec) -> TurnContext { TurnContext { diff --git a/src/openhuman/memory_tools/prompt.rs b/src/openhuman/memory_tools/prompt.rs index df1c22b33..48fb3719a 100644 --- a/src/openhuman/memory_tools/prompt.rs +++ b/src/openhuman/memory_tools/prompt.rs @@ -46,9 +46,7 @@ mod tests { use crate::openhuman::agent::prompts::types::{ LearnedContextData, PromptContext, ToolCallFormat, }; - use crate::openhuman::memory_tools::types::{ - ToolMemoryPriority, ToolMemoryRule, ToolMemorySource, - }; + use crate::openhuman::memory_tools::{ToolMemoryPriority, ToolMemoryRule, ToolMemorySource}; fn rule(tool: &str, body: &str, priority: ToolMemoryPriority) -> ToolMemoryRule { ToolMemoryRule { diff --git a/src/openhuman/memory_tools/store.rs b/src/openhuman/memory_tools/store.rs index 1cee3a757..765bbf276 100644 --- a/src/openhuman/memory_tools/store.rs +++ b/src/openhuman/memory_tools/store.rs @@ -1,16 +1,10 @@ -//! Storage layer for tool-scoped rules — thin host shim over -//! `tinycortex::memory::tool_memory::store` (W7). -//! -//! The store engine (put / get / list / delete / prompt over an -//! `Arc`) and the `Memory` trait are both owned by tinycortex. - use std::sync::Arc; use crate::openhuman::memory::Memory; pub use tinycortex::memory::tool_memory::store::{ToolMemoryStore, TOOL_MEMORY_PROMPT_CAP}; -/// Build a crate [`ToolMemoryStore`] over the shared tinycortex memory trait. +/// Build the crate-owned store over OpenHuman's shared memory object. pub fn tool_memory_store(memory: Arc) -> ToolMemoryStore { ToolMemoryStore::new(memory) } diff --git a/src/openhuman/memory_tools/types.rs b/src/openhuman/memory_tools/types.rs index 2a173548c..84e6280c2 100644 --- a/src/openhuman/memory_tools/types.rs +++ b/src/openhuman/memory_tools/types.rs @@ -1,10 +1,8 @@ //! Domain types for the tool-scoped memory layer — thin host re-export of -//! `tinycortex::memory::tool_memory::types` (W7). +//! `tinycortex::memory::tool_memory::types`. //! -//! [`ToolMemoryRule`] / [`ToolMemoryPriority`] / [`ToolMemorySource`] and the -//! [`tool_memory_namespace`] helper are the crate's (a byte-identical port, -//! preserving the serde wire strings + `rule/{id}` storage keys). Host consumers -//! keep their `memory_tools::types::*` import paths unchanged. +//! This module preserves the public `memory_tools::types::*` import path while +//! TinyCortex remains the single implementation and wire-format authority. pub use tinycortex::memory::tool_memory::types::{ tool_memory_namespace, ToolMemoryPriority, ToolMemoryRule, ToolMemorySource, diff --git a/src/openhuman/memory_tree/README.md b/src/openhuman/memory_tree/README.md index 35d498dd5..9bc90b16c 100644 --- a/src/openhuman/memory_tree/README.md +++ b/src/openhuman/memory_tree/README.md @@ -31,8 +31,7 @@ memory_store::trees (persistence: one Tree table, one schema) | [`tree/`](tree/) | `bucket_seal` (append leaf + cascade seal), `flush` (time-based partial seal), `registry` (kind-parameterized `get_or_create_tree` with UNIQUE-race recovery), `mod.rs` (re-exports + `memory_store::trees` shims for legacy paths). | | [`summarise.rs`](summarise.rs) | One function: produce the next-level summary text for a bucket. Wraps the chat model with a fixed prompt and token budget. | | [`retrieval/`](retrieval/) | Agent-facing tools. Read: `walk` (agentic), `drill_down`, `fetch_leaves`, `query_{source,global,topic}`, `search_entities`. Write: `ingest_document` (orchestrator-facing). | -| [`score/`](score/) | Scoring signals, embedding (cloud/ollama/inert), entity extraction (regex/LLM), canonical resolver, entity index store. | -| [`tools.rs`](tools.rs) | Re-exports from `memory::query` for backward compatibility. | +| [`score/`](score/) | Product adapters over TinyCortex scoring and TinyAgents embedding models, plus entity extraction and the entity index store. | ## Layer rules diff --git a/src/openhuman/memory_tree/mod.rs b/src/openhuman/memory_tree/mod.rs index 5f6e579b3..78b9e16d4 100644 --- a/src/openhuman/memory_tree/mod.rs +++ b/src/openhuman/memory_tree/mod.rs @@ -13,7 +13,6 @@ pub mod nlp; pub mod retrieval; pub mod score; pub mod summarise; -pub mod tools; pub mod tree; pub mod tree_runtime; diff --git a/src/openhuman/memory_tree/score/embed/README.md b/src/openhuman/memory_tree/score/embed/README.md index f4d881715..43b71bd30 100644 --- a/src/openhuman/memory_tree/score/embed/README.md +++ b/src/openhuman/memory_tree/score/embed/README.md @@ -1,18 +1,17 @@ -# Memory tree — score embed (Phase 4 / #710) +# Memory tree embedding bridge -Vector embedder for chunks and summaries. Produces a fixed-dimension (`EMBEDDING_DIM = 768`) `Vec` per text so retrieval can rerank candidates by semantic similarity. Default backend is local Ollama running `nomic-embed-text`; tests use the deterministic `InertEmbedder` so no network is required. +The memory tree stores fixed 1024-dimensional vectors. Concrete provider +transports live in TinyAgents; this directory keeps only memory-tree policy and +compatibility: -## Public surface +- `factory.rs`: read/write provider resolution, cloud-session policy, and + degraded-state behavior; +- `openai_compat.rs`: OpenHuman config/credential and custom-slug resolution; +- `inert.rs`: deterministic 1024-element zero vectors for opt-out/tests; +- `mod.rs`: the legacy `Embedder` contract, `ProviderEmbedder` bridge, batch + fallback/dimension checks, cosine math, and SQLite f32 packing helpers. -- `pub trait Embedder` — `mod.rs` — `embed(text) -> Vec` contract; impls must return exactly `EMBEDDING_DIM` floats. -- `pub fn build_embedder_from_config` — `factory.rs` — returns `OllamaEmbedder` when configured, otherwise `InertEmbedder` (or bails when `embedding_strict = true`). -- `pub struct OllamaEmbedder` — `ollama.rs` — HTTP client posting to `{endpoint}/api/embeddings`. -- `pub struct InertEmbedder` — `inert.rs` — zero-vector embedder for tests. -- `pub fn cosine_similarity` / `pack_embedding` / `unpack_embedding` / `pack_checked` / `decode_optional_blob` — `mod.rs` — math + SQLite BLOB packing helpers. - -## Files - -- `mod.rs` — trait, `EMBEDDING_DIM`, math + pack/unpack helpers, write-time / read-time semantics. -- `factory.rs` — `Config::memory_tree`-driven embedder selection with `embedding_strict` opt-in. -- `ollama.rs` — Ollama `/api/embeddings` client; defaults at `http://localhost:11434` / `nomic-embed-text` / 10s timeout. -- `inert.rs` — zero-vector embedder; cosine similarity between any two inert vectors is 0.0 (zero-magnitude short-circuit), so retrieval tests that need real reranking should hand-stitch embeddings instead of relying on this path. +Ollama uses TinyAgents `OllamaEmbeddingModel` and `/api/embed`, with the shared +8192-token context and batch window. Managed cloud uses the host credential and +privacy wrapper around TinyAgents `CloudEmbeddingModel`. Do not add provider +HTTP clients in this directory. diff --git a/src/openhuman/memory_tree/score/embed/cloud.rs b/src/openhuman/memory_tree/score/embed/cloud.rs deleted file mode 100644 index c0787cb9b..000000000 --- a/src/openhuman/memory_tree/score/embed/cloud.rs +++ /dev/null @@ -1,109 +0,0 @@ -//! Cloud (Voyage-backed) embedder for the memory tree. -//! -//! Adapts the OpenHuman backend's `POST /openai/v1/embeddings` surface -//! (Voyage `voyage-3.5`, 1024 dims) to the memory_tree [`Embedder`] trait -//! so Phase 4 ingest / bucket-seal can vectorize chunks without a local -//! Ollama install. -//! -//! The 1024-dim output matches existing on-disk blobs (which were -//! produced by `bge-m3`, also 1024-dim), so this is a drop-in replacement -//! for the Ollama path — no migration of `mem_tree_chunks.embedding` -//! required. -//! -//! Auth: the cloud embedder resolves the session JWT per call via -//! [`OpenHumanCloudEmbedding`], so a session refresh between batches is -//! picked up transparently. When the user is unauthenticated the first -//! `embed()` returns an error; ingest treats that the same as any other -//! embedder failure (don't persist the row, let job retry). - -use anyhow::{Context, Result}; -use async_trait::async_trait; - -use super::{Embedder, EMBEDDING_DIM}; -use crate::openhuman::config::Config; -use crate::openhuman::embeddings::cloud::{ - OpenHumanCloudEmbedding, DEFAULT_CLOUD_EMBEDDING_DIMENSIONS, DEFAULT_CLOUD_EMBEDDING_MODEL, -}; -use crate::openhuman::embeddings::EmbeddingProvider; - -/// Cloud-backed memory_tree embedder. -/// -/// Wraps [`OpenHumanCloudEmbedding`] (which speaks the OpenAI-compatible -/// `/openai/v1/embeddings` shape backed by Voyage on the OpenHuman -/// backend) and adapts it to the memory_tree [`Embedder`] trait. -pub struct CloudEmbedder { - inner: OpenHumanCloudEmbedding, -} - -impl CloudEmbedder { - /// Build a cloud embedder using the same backend resolution as the - /// main embeddings path: `api_url` falls back to - /// [`effective_api_url`](crate::api::config::effective_api_url) and - /// the workspace dir comes from `config.workspace_dir` so the auth - /// service finds the user's session JWT. - pub fn new(config: &Config) -> Self { - let openhuman_dir = config.config_path.parent().map(std::path::PathBuf::from); - Self { - inner: OpenHumanCloudEmbedding::new( - None, - openhuman_dir, - config.secrets.encrypt, - DEFAULT_CLOUD_EMBEDDING_MODEL, - DEFAULT_CLOUD_EMBEDDING_DIMENSIONS, - ), - } - } -} - -#[async_trait] -impl Embedder for CloudEmbedder { - fn name(&self) -> &'static str { - "cloud" - } - - async fn embed(&self, text: &str) -> Result> { - let v = self - .inner - .embed_one(text) - .await - .context("cloud embeddings failed")?; - if v.len() != EMBEDDING_DIM { - anyhow::bail!( - "cloud embedder returned {} dims, expected {}", - v.len(), - EMBEDDING_DIM - ); - } - Ok(v) - } - - /// Collapse N per-text round-trips into a single batched Voyage request by - /// delegating to the inner provider's native batch `embed`. Falls back to - /// per-text embedding (preserving per-position error attribution) on a - /// whole-batch failure or a length mismatch. - async fn embed_batch(&self, texts: &[&str]) -> Vec>> { - super::embed_batch_via_provider(&self.inner, "cloud", texts).await - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::openhuman::config::Config; - use tempfile::TempDir; - - fn test_config() -> (TempDir, Config) { - let tmp = TempDir::new().unwrap(); - let mut cfg = Config::default(); - cfg.workspace_dir = tmp.path().to_path_buf(); - cfg.config_path = tmp.path().join("config.toml"); - (tmp, cfg) - } - - #[test] - fn name_is_cloud() { - let (_tmp, cfg) = test_config(); - let e = CloudEmbedder::new(&cfg); - assert_eq!(e.name(), "cloud"); - } -} diff --git a/src/openhuman/memory_tree/score/embed/factory.rs b/src/openhuman/memory_tree/score/embed/factory.rs index 7f8601cfb..00a4b50ed 100644 --- a/src/openhuman/memory_tree/score/embed/factory.rs +++ b/src/openhuman/memory_tree/score/embed/factory.rs @@ -29,11 +29,14 @@ //! - `OPENHUMAN_MEMORY_EMBED_MODEL` //! - `OPENHUMAN_MEMORY_EMBED_TIMEOUT_MS` -use anyhow::Result; +use anyhow::{Context, Result}; -use super::{CloudEmbedder, Embedder, InertEmbedder, OllamaEmbedder}; +use std::time::Duration; + +use super::{Embedder, InertEmbedder, ProviderEmbedder, EMBEDDING_DIM}; use crate::openhuman::config::Config; use crate::openhuman::inference::local::ollama_base_url; +use tinyagents::harness::embeddings::{OllamaEmbeddingModel, RECOMMENDED_OLLAMA_CONTEXT_TOKENS}; /// Cheap heuristic for "is a backend session reachable?" — the cloud /// embedder needs one and bails on first embed call without it. We use @@ -69,7 +72,7 @@ pub fn build_embedder_from_config(config: &Config) -> Result> log::debug!( "[memory_tree::embed::factory] read → Ollama endpoint={endpoint} model={model} timeout_ms={timeout_ms}" ); - Box::new(OllamaEmbedder::new(endpoint, model, timeout_ms)) + Box::new(build_ollama_embedder(&endpoint, &model, timeout_ms)?) } EmbedderChoice::OptOut => { log::info!( @@ -90,7 +93,7 @@ pub fn build_embedder_from_config(config: &Config) -> Result> "[memory_tree::embed::factory] read → cloud (Voyage) — flip \ 'Memory embeddings' in Local AI Settings to switch to local" ); - Box::new(CloudEmbedder::new(config)) + Box::new(build_cloud_embedder(config)) } EmbedderChoice::NoProvider => { log::warn!( @@ -215,7 +218,9 @@ pub fn build_write_embedder(config: &Config) -> Result> timeout_ms, } => { clear_semantic_recall_degraded(); - Some(Box::new(OllamaEmbedder::new(endpoint, model, timeout_ms))) + Some(Box::new(build_ollama_embedder( + &endpoint, &model, timeout_ms, + )?)) } EmbedderChoice::OptOut => { clear_semantic_recall_degraded(); @@ -231,7 +236,7 @@ pub fn build_write_embedder(config: &Config) -> Result> } EmbedderChoice::Cloud => { clear_semantic_recall_degraded(); - Some(Box::new(CloudEmbedder::new(config))) + Some(Box::new(build_cloud_embedder(config))) } EmbedderChoice::NoProvider => { log::warn!( @@ -245,6 +250,36 @@ pub fn build_write_embedder(config: &Config) -> Result> }) } +fn build_ollama_embedder(endpoint: &str, model: &str, timeout_ms: u64) -> Result { + let timeout = Duration::from_millis(if timeout_ms == 0 { 10_000 } else { timeout_ms }); + let client = reqwest::Client::builder() + .connect_timeout(timeout) + .build() + .context("build Ollama embeddings HTTP client")?; + let model = OllamaEmbeddingModel::try_new(endpoint, model, EMBEDDING_DIM)? + .with_context_options( + RECOMMENDED_OLLAMA_CONTEXT_TOKENS, + RECOMMENDED_OLLAMA_CONTEXT_TOKENS, + ) + .with_client(client); + Ok(ProviderEmbedder::new( + crate::openhuman::embeddings::TinyAgentsEmbeddingProvider::boxed(model), + "ollama", + )) +} + +fn build_cloud_embedder(config: &Config) -> ProviderEmbedder { + let openhuman_dir = config.config_path.parent().map(std::path::PathBuf::from); + let provider = crate::openhuman::embeddings::cloud::OpenHumanCloudEmbedding::new( + None, + openhuman_dir, + config.secrets.encrypt, + crate::openhuman::embeddings::cloud::DEFAULT_CLOUD_EMBEDDING_MODEL, + crate::openhuman::embeddings::cloud::DEFAULT_CLOUD_EMBEDDING_DIMENSIONS, + ); + ProviderEmbedder::new(Box::new(provider), "cloud") +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/openhuman/memory_tree/score/embed/mod.rs b/src/openhuman/memory_tree/score/embed/mod.rs index 5c2143d90..a8a745c69 100644 --- a/src/openhuman/memory_tree/score/embed/mod.rs +++ b/src/openhuman/memory_tree/score/embed/mod.rs @@ -29,16 +29,12 @@ use anyhow::{Context, Result}; use async_trait::async_trait; -pub mod cloud; pub mod factory; pub mod inert; -pub mod ollama; pub mod openai_compat; -pub use cloud::CloudEmbedder; pub use factory::{build_embedder_from_config, build_write_embedder}; pub use inert::InertEmbedder; -pub use ollama::OllamaEmbedder; pub use openai_compat::OpenAiCompatEmbedder; /// Embedding dimensionality used across the memory tree. @@ -89,6 +85,43 @@ pub trait Embedder: Send + Sync { } } +/// Adapts the canonical host embedding-provider contract to the legacy +/// memory-tree embedder shape. Concrete network implementations live in +/// `tinyagents::harness::embeddings`; this bridge owns only dimension checks +/// and the memory tree's per-position batch fallback contract. +pub struct ProviderEmbedder { + inner: Box, + label: &'static str, +} + +impl ProviderEmbedder { + pub fn new( + inner: Box, + label: &'static str, + ) -> Self { + Self { inner, label } + } +} + +#[async_trait] +impl Embedder for ProviderEmbedder { + fn name(&self) -> &'static str { + self.label + } + + async fn embed(&self, text: &str) -> Result> { + self.inner + .embed_one(text) + .await + .with_context(|| format!("{} embeddings failed", self.label)) + .and_then(|vector| check_embed_dim(vector, self.label)) + } + + async fn embed_batch(&self, texts: &[&str]) -> Vec>> { + embed_batch_via_provider(self.inner.as_ref(), self.label, texts).await + } +} + /// Validate that a freshly-produced embedding has exactly [`EMBEDDING_DIM`] /// floats, returning a labelled error otherwise. Shared by the per-text and /// batched provider adapters so the "wrong dims" diagnostic is identical diff --git a/src/openhuman/memory_tree/score/embed/ollama.rs b/src/openhuman/memory_tree/score/embed/ollama.rs deleted file mode 100644 index e9bef395e..000000000 --- a/src/openhuman/memory_tree/score/embed/ollama.rs +++ /dev/null @@ -1,384 +0,0 @@ -//! Ollama-backed embedder for Phase 4 (#710). -//! -//! Posts `{model, prompt, options: {num_ctx}}` to -//! `{endpoint}/api/embeddings` and expects -//! `{"embedding": [f32; EMBEDDING_DIM]}` back. Designed for a local -//! `ollama serve` hosting `bge-m3`. -//! -//! This is intentionally a tiny HTTP client — no retry, no pool caching, -//! no streaming. Phase 4 wants the simplest thing that works so we can -//! land embedding end-to-end and iterate once baseline retrieval quality -//! is measurable. Timeouts, parallelism, and caching are explicit -//! follow-ups. - -use std::time::Duration; - -use anyhow::{Context, Result}; -use async_trait::async_trait; -use serde::{Deserialize, Serialize}; - -use super::{Embedder, EMBEDDING_DIM}; - -/// Default Ollama endpoint. Matches the local-install default from the -/// `local_ai` subsystem and the Ollama defaults. -pub const DEFAULT_ENDPOINT: &str = "http://localhost:11434"; - -/// Default embedding model — must output exactly [`EMBEDDING_DIM`] -/// (1024) dims. `bge-m3` is a multilingual BERT-family encoder with -/// native 8192-token context and 1024-dim output. -pub const DEFAULT_MODEL: &str = "bge-m3"; - -/// Default request timeout. Ollama's first-use latency is a few hundred -/// ms on a warm model; 10s absorbs a cold-model load on commodity -/// hardware without stalling ingest on a broken backend. -pub const DEFAULT_TIMEOUT_MS: u64 = 10_000; - -/// HTTP client wrapping a single Ollama endpoint + model pair. -/// -/// Cloneable — `reqwest::Client` shares a connection pool under the hood -/// so cloning the wrapper stays cheap across seal / ingest call sites. -#[derive(Clone)] -pub struct OllamaEmbedder { - endpoint: String, - model: String, - timeout: Duration, - client: reqwest::Client, -} - -impl OllamaEmbedder { - /// Build a new embedder. `endpoint` is trimmed of trailing slashes - /// so callers don't have to worry about mixing `http://host` and - /// `http://host/`. Empty values fall back to the public defaults. - pub fn new(endpoint: String, model: String, timeout_ms: u64) -> Self { - let endpoint = if endpoint.trim().is_empty() { - DEFAULT_ENDPOINT.to_string() - } else { - endpoint.trim().trim_end_matches('/').to_string() - }; - let model = if model.trim().is_empty() { - DEFAULT_MODEL.to_string() - } else { - model.trim().to_string() - }; - let timeout_ms = if timeout_ms == 0 { - DEFAULT_TIMEOUT_MS - } else { - timeout_ms - }; - let timeout = Duration::from_millis(timeout_ms); - // No body-read timeout. Ollama is local — slow responses mean - // the model is genuinely processing, not that the network - // broke. A body-read timeout here would cancel mid-stream and - // force retries against the same slow model. `timeout` is - // repurposed as the TCP connect timeout (fast-fail when - // Ollama is actually down). - let client = reqwest::Client::builder() - .connect_timeout(timeout) - .build() - .unwrap_or_else(|e| { - log::warn!( - "[memory_tree::embed::ollama] failed to build client \ - with connect_timeout — using default: {e}" - ); - reqwest::Client::new() - }); - log::debug!( - "[memory_tree::embed::ollama] created endpoint={endpoint} \ - model={model} timeout_ms={timeout_ms}" - ); - Self { - endpoint, - model, - timeout, - client, - } - } - - /// Convenience constructor using all defaults ([`DEFAULT_ENDPOINT`], - /// [`DEFAULT_MODEL`], [`DEFAULT_TIMEOUT_MS`]). - pub fn default_new() -> Self { - Self::new(String::new(), String::new(), 0) - } - - fn embed_url(&self) -> String { - format!("{}/api/embeddings", self.endpoint) - } -} - -fn is_missing_model_response(status: reqwest::StatusCode, body: &str) -> bool { - if status != reqwest::StatusCode::NOT_FOUND { - return false; - } - - let normalized = body.to_ascii_lowercase(); - normalized.contains("model") && normalized.contains("not found") -} - -fn format_embedding_status_error( - status: reqwest::StatusCode, - body: &str, - endpoint: &str, - model: &str, -) -> String { - let trimmed_body = body.trim(); - - if is_missing_model_response(status, trimmed_body) { - return format!( - "Ollama embedding model `{model}` is not installed at {endpoint}. \ - Run `ollama pull {model}` or choose an installed embedding model in \ - Connections → API keys → Embeddings. status={status} body={trimmed_body}" - ); - } - - format!("ollama embeddings failed status={status} body={trimmed_body}") -} - -/// Override Ollama's per-model `num_ctx` default. Ollama loads -/// embedding models with `num_ctx = 4096` (or whatever default the -/// model's modelfile carries) unless the request explicitly asks for -/// more. `bge-m3` natively supports 8192 tokens, and chunker-token -/// counts undercount BERT-WordPiece tokens by ~1.5-2× for HTML-derived -/// markdown — so a 1500-chunker-token chunk routinely produces 2500+ -/// real tokens at embed time. Asking for 8192 unconditionally avoids -/// silent prompt truncation; on models that natively support less, -/// Ollama clamps `num_ctx` to the model's actual maximum, so this is -/// safe to over-request. -/// -/// This is also the **single source of truth for the memory layer's -/// minimum model context window**: a local model whose native context is -/// below this floor silently truncates chunks/summaries and corrupts -/// recall. `inference::local::model_requirements::MIN_CONTEXT_TOKENS` -/// re-exports this value so the model-acceptance gate can never drift -/// from what the embedder actually requests. -pub(crate) const EMBED_NUM_CTX: u32 = 8192; - -#[derive(Serialize)] -struct EmbedRequest<'a> { - model: &'a str, - prompt: &'a str, - options: EmbedOptions, -} - -#[derive(Serialize)] -struct EmbedOptions { - num_ctx: u32, - // Ollama/llama.cpp evaluates an embedding prompt in a SINGLE batch and - // rejects (HTTP 500 "input length exceeds the context length") any prompt - // longer than `n_batch` — whose default is 2048, far below `num_ctx`. - // Raising `num_ctx` alone does NOT lift this; `num_batch` must match, or - // chunks between ~2k and 8k real tokens fail to embed even though they fit - // the context window. Set equal to `num_ctx` so the batch limit never bites - // before the real context limit. - num_batch: u32, -} - -#[derive(Deserialize)] -struct EmbedResponse { - #[serde(default)] - embedding: Vec, -} - -#[async_trait] -impl Embedder for OllamaEmbedder { - fn name(&self) -> &'static str { - "ollama" - } - - async fn embed(&self, text: &str) -> Result> { - log::debug!( - "[memory_tree::embed::ollama] embed endpoint={} model={} bytes={}", - self.endpoint, - self.model, - text.len() - ); - let req = EmbedRequest { - model: &self.model, - prompt: text, - options: EmbedOptions { - num_ctx: EMBED_NUM_CTX, - num_batch: EMBED_NUM_CTX, - }, - }; - let resp = self - .client - // No per-request body-read timeout — see `OllamaEmbedder::new` - // for rationale. The Client's `connect_timeout` still applies - // and fails fast if Ollama isn't reachable. - .post(self.embed_url()) - .json(&req) - .send() - .await - .with_context(|| { - format!( - "ollama embeddings request failed (is Ollama running at {}?)", - self.endpoint - ) - })?; - - if !resp.status().is_success() { - let status = resp.status(); - let body = resp.text().await.unwrap_or_default(); - let msg = format_embedding_status_error(status, &body, &self.endpoint, &self.model); - // Log at WARN so missing-model failures surface in traces without - // requiring debug-level logging to be enabled. Missing-model 404s - // include the `ollama pull` remediation hint from - // `format_embedding_status_error`. - log::warn!("[embeddings] {msg}"); - anyhow::bail!(msg); - } - - let payload: EmbedResponse = resp - .json() - .await - .context("ollama embeddings response parse failed")?; - - if payload.embedding.len() != EMBEDDING_DIM { - anyhow::bail!( - "ollama embeddings returned {} dims, expected {EMBEDDING_DIM} \ - (check model={})", - payload.embedding.len(), - self.model - ); - } - - Ok(payload.embedding) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use axum::{extract::Json, http::StatusCode, routing::post, Router}; - use std::net::SocketAddr; - - /// Spin up a local axum server and return its base URL. - async fn start_mock(app: Router) -> String { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr: SocketAddr = listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve(listener, app).await.unwrap(); - }); - format!("http://127.0.0.1:{}", addr.port()) - } - - fn fixed_vec(val: f32) -> Vec { - vec![val; EMBEDDING_DIM] - } - - #[test] - fn defaults_applied_on_empty_input() { - let e = OllamaEmbedder::new(String::new(), String::new(), 0); - assert_eq!(e.endpoint, DEFAULT_ENDPOINT); - assert_eq!(e.model, DEFAULT_MODEL); - assert_eq!(e.timeout, Duration::from_millis(DEFAULT_TIMEOUT_MS)); - } - - #[test] - fn trailing_slash_trimmed() { - let e = OllamaEmbedder::new("http://host:1234/".into(), "m".into(), 1000); - assert_eq!(e.endpoint, "http://host:1234"); - } - - #[test] - fn embed_url_format() { - let e = OllamaEmbedder::default_new(); - assert_eq!(e.embed_url(), "http://localhost:11434/api/embeddings"); - } - - #[test] - fn name_is_ollama() { - assert_eq!(OllamaEmbedder::default_new().name(), "ollama"); - } - - #[tokio::test] - async fn happy_path_returns_embedding() { - let v = fixed_vec(0.25); - let v_clone = v.clone(); - let app = Router::new().route( - "/api/embeddings", - post(move |Json(body): Json| { - let v = v_clone.clone(); - async move { - assert_eq!(body["model"], "bge-m3"); - assert_eq!(body["prompt"], "hello world"); - assert_eq!(body["options"]["num_ctx"], 8192); - assert_eq!(body["options"]["num_batch"], 8192); - Json(serde_json::json!({ "embedding": v })) - } - }), - ); - let url = start_mock(app).await; - let e = OllamaEmbedder::new(url, String::new(), 0); - let out = e.embed("hello world").await.unwrap(); - assert_eq!(out.len(), EMBEDDING_DIM); - assert!((out[0] - 0.25).abs() < 1e-6); - } - - #[tokio::test] - async fn server_error_bubbles_up() { - let app = Router::new().route( - "/api/embeddings", - post(|| async { (StatusCode::INTERNAL_SERVER_ERROR, "model crashed") }), - ); - let url = start_mock(app).await; - let e = OllamaEmbedder::new(url, String::new(), 0); - let err = e.embed("hello").await.unwrap_err().to_string(); - assert!(err.contains("500"), "msg: {err}"); - assert!(err.contains("model crashed"), "msg: {err}"); - } - - #[tokio::test] - async fn missing_model_404_mentions_pull_command() { - let app = Router::new().route( - "/api/embeddings", - post(|| async { (StatusCode::NOT_FOUND, "{\"error\":\"model not found\"}") }), - ); - let url = start_mock(app).await; - let e = OllamaEmbedder::new(url, "custom-embed".into(), 0); - let err = e.embed("hello").await.unwrap_err().to_string(); - - assert!( - err.contains("embedding model `custom-embed` is not installed"), - "msg: {err}" - ); - assert!(err.contains("ollama pull custom-embed"), "msg: {err}"); - } - - #[tokio::test] - async fn dim_mismatch_rejected() { - let app = Router::new().route( - "/api/embeddings", - post(|| async { - // Return a 3-dim vector — must fail validation. - Json(serde_json::json!({ "embedding": [0.1, 0.2, 0.3] })) - }), - ); - let url = start_mock(app).await; - let e = OllamaEmbedder::new(url, String::new(), 0); - let err = e.embed("hi").await.unwrap_err().to_string(); - assert!(err.contains("3 dims"), "msg: {err}"); - assert!(err.contains("expected 1024"), "msg: {err}"); - } - - #[tokio::test] - async fn malformed_json_response_rejected() { - let app = Router::new().route( - "/api/embeddings", - post(|| async { (StatusCode::OK, "not even json") }), - ); - let url = start_mock(app).await; - let e = OllamaEmbedder::new(url, String::new(), 0); - let err = e.embed("hi").await.unwrap_err().to_string(); - assert!(err.contains("parse failed"), "msg: {err}"); - } - - #[tokio::test] - async fn request_failure_is_descriptive() { - let e = OllamaEmbedder::new("http://%".into(), String::new(), 500); - let err = e.embed("hi").await.unwrap_err().to_string(); - assert!( - err.contains("is Ollama running"), - "should mention Ollama: {err}" - ); - } -} diff --git a/src/openhuman/memory_tree/tools.rs b/src/openhuman/memory_tree/tools.rs deleted file mode 100644 index 7612da6ad..000000000 --- a/src/openhuman/memory_tree/tools.rs +++ /dev/null @@ -1,4 +0,0 @@ -//! Compatibility shim: high-level memory query tools now live in -//! `crate::openhuman::memory::query`. - -pub use crate::openhuman::memory::query::*; diff --git a/src/openhuman/mod.rs b/src/openhuman/mod.rs index 1e4c11734..57d3e5ae8 100644 --- a/src/openhuman/mod.rs +++ b/src/openhuman/mod.rs @@ -84,7 +84,6 @@ pub mod medulla_local; pub mod meet; pub mod meet_agent; pub mod memory; -pub mod memory_archivist; pub mod memory_conversations; pub mod memory_diff; pub mod memory_goals; diff --git a/src/openhuman/scheduler_gate/gate.rs b/src/openhuman/scheduler_gate/gate.rs index 728893d1e..e449e3d9f 100644 --- a/src/openhuman/scheduler_gate/gate.rs +++ b/src/openhuman/scheduler_gate/gate.rs @@ -24,7 +24,7 @@ use crate::openhuman::scheduler_gate::signals::Signals; /// simultaneous Ollama requests have crashed the user's laptop twice. /// /// Cloud-backend LLM calls bypass this semaphore at the worker layer -/// (see `memory::jobs::worker::run_once`) because they're +/// (see `memory_queue::worker::run_once`) because they're /// bandwidth-bound, not RAM-bound, and the worker pool itself bounds /// concurrency upstream. Keeping this at 1 preserves the laptop-RAM /// contract regardless of backend. diff --git a/src/openhuman/tinycortex/mod.rs b/src/openhuman/tinycortex/mod.rs index 99d5a36d8..72bb914bb 100644 --- a/src/openhuman/tinycortex/mod.rs +++ b/src/openhuman/tinycortex/mod.rs @@ -5,7 +5,7 @@ //! chunks / tree / retrieval / queue / ingest / score + the long tail). This //! module is the **adapter seam**, mirroring `src/openhuman/tinyagents/`: it //! implements the crate's engine traits over OpenHuman services and derives the -//! engine's [`MemoryConfig`] from the host [`Config`]. Nothing here contains +//! engine's [`tinycortex::memory::MemoryConfig`] from the host [`Config`]. Nothing here contains //! engine logic — that lives in the crate. //! //! ## Ownership boundary (the seam contract) @@ -16,19 +16,20 @@ //! archivist/tool-memory/conversations long tail. //! //! **Product (host, stays in OpenHuman):** JSON-RPC schemas/ops/read_rpc, agent -//! tools + `SecurityPolicy` gating, live sync (`memory_sync`), the event bus, -//! preferences, `source_scope` per-turn allowlist, redaction, the global -//! singleton + background queue worker, embedding/LLM **compute**, and the +//! tools + `SecurityPolicy` gating, sync scheduling/credentials/events, the +//! event bus, preferences, `source_scope` per-turn allowlist, redaction, the +//! global singleton + background queue worker, embedding/LLM **compute**, and the //! host-retained `UnifiedMemory` namespace-document tier (episodic/event/ //! segment/doc/graph/profile tables) plus the `wiki_git`/`obsidian` content //! surfaces the crate deliberately excludes. //! -//! The crate never makes a network call or owns a worker pool: LLM/embedding -//! compute is injected through `EmbeddingBackend`, `ChatProvider`, `Summariser`, -//! and `EntityExtractor`; the job queue is driven by the host worker loop via +//! Network-capable sync providers are feature-gated in the crate; their +//! credentials and product policy stay in the host. LLM/embedding compute is +//! injected through `EmbeddingBackend`, `ChatProvider`, `Summariser`, and +//! `EntityExtractor`; the job queue is driven by the host worker loop via //! `queue::run_once` / `drain_until_idle`. Those adapters live beside this file -//! (`embeddings.rs`, `chat.rs`, `queue_driver.rs`, `sinks.rs`, `bus.rs`) and are -//! added workstream by workstream. +//! (`embeddings.rs`, `chat.rs`, `queue_driver.rs`, `ingest.rs`, `seal.rs`, and +//! `sync.rs`). //! //! See `docs/tinycortex-migration-spec.md` for the full ownership split, //! drift/gap/parity ledgers, and the workstream order. @@ -66,12 +67,3 @@ pub use sync::{ run_gmail_backfill, run_slack_search_backfill, run_source_pipeline, sync_context, HostSyncAdapter, SourcePipelineFailure, HOST_SYNC_STATE_NAMESPACE, }; -// Facade re-exports — the rest of the host imports memory-engine types through -// this one seam so consumer import paths stay stable as the internals flip to -// the crate (the type-unification decision, spec §0.5). `MemoryTaint` is the -// security-critical provenance type; it is proven byte-identical to the host's -// (fail-closed to `ExternalSync`) before re-exporting. -pub use tinycortex::memory::{ - MemoryCategory, MemoryConfig, MemoryEntry, MemoryTaint, NamespaceSummary, RecallOpts, - WeightProfile, -}; diff --git a/src/openhuman/tinycortex/queue_driver.rs b/src/openhuman/tinycortex/queue_driver.rs index 0498122e4..2b9b67f7b 100644 --- a/src/openhuman/tinycortex/queue_driver.rs +++ b/src/openhuman/tinycortex/queue_driver.rs @@ -18,12 +18,10 @@ //! reproduces it exactly. It is a pure function so the policy is unit-tested //! without spinning a live loop. //! -//! Still to land in W4 (tracked in the migration spec): the real -//! `HostQueueDelegates` that bridges the 8 delegate methods to the host -//! `memory_tree` engine (paired with W5), the tokio worker-loop + scheduler that -//! applies this policy while driving `run_once`, re-pointing `memory/global.rs` -//! and the enqueue call sites onto `queue::store`, and deleting the legacy -//! `memory_queue` engine. This brick is additive: nothing is flipped yet. +//! The host worker is flipped to `tinycortex::memory::queue::run_once` through +//! `HostQueueDelegates`. The adapter preserves host-owned scheduling, health +//! reporting, event-bus publishing, and product-policy hooks while the crate +//! owns claim/dispatch/settle. use std::time::Duration; diff --git a/tests/agent_retrieval_e2e.rs b/tests/agent_retrieval_e2e.rs index 76eb06088..b1f557f10 100644 --- a/tests/agent_retrieval_e2e.rs +++ b/tests/agent_retrieval_e2e.rs @@ -22,7 +22,7 @@ use chrono::{TimeZone, Utc}; use openhuman_core::openhuman::config::Config; use openhuman_core::openhuman::memory::ingest_pipeline::{ingest_chat, ingest_email}; -use openhuman_core::openhuman::memory::jobs::drain_until_idle; +use openhuman_core::openhuman::memory_queue::drain_until_idle; use openhuman_core::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage}; use openhuman_core::openhuman::memory_sync::canonicalize::email::{EmailMessage, EmailThread}; use openhuman_core::openhuman::tools::{ diff --git a/tests/memory_golden_parity_e2e.rs b/tests/memory_golden_parity_e2e.rs index 64158170a..2ef033ba1 100644 --- a/tests/memory_golden_parity_e2e.rs +++ b/tests/memory_golden_parity_e2e.rs @@ -10,7 +10,7 @@ //! 1. the **crate-owned substrate** the `tinycortex` chunk DB creates //! (`init_db` → `chunks/schema.rs`), and //! 2. the **host-retained `UnifiedMemory` namespace-document tier** -//! (`memory_store/unified/*`), +//! (`memory_store/namespace_store/*`), //! //! coexisting without collision (parity checklist P3/P5/P11/P12 — the W3 gate). //! A store/tree cutover that reshaped, renamed, or dropped a table would strand @@ -105,7 +105,7 @@ const CRATE_CHUNK_SCHEMA_TABLES: &[&str] = &[ ]; /// The host-retained `UnifiedMemory` namespace-document tier -/// (`memory_store/unified/*`) — stays host, coexists in the shared workspace. +/// (`memory_store/namespace_store/*`) — stays host, coexists in the shared workspace. const HOST_UNIFIED_TABLES: &[&str] = &[ "memory_docs", "graph_global", diff --git a/tests/raw_coverage/composio_credentials_state_raw_coverage_e2e.rs b/tests/raw_coverage/composio_credentials_state_raw_coverage_e2e.rs index 5cade5a55..e33c05f3f 100644 --- a/tests/raw_coverage/composio_credentials_state_raw_coverage_e2e.rs +++ b/tests/raw_coverage/composio_credentials_state_raw_coverage_e2e.rs @@ -308,7 +308,7 @@ async fn round15_composio_agent_tools_backend_cache_and_trigger_history_edges() assert_eq!(action_tool.name(), "GMAIL_FETCH_EMAILS"); assert_eq!(action_tool.category().to_string(), "skill"); let contract_result = action_tool - .execute(json!({ "query": "from:me" })) + .execute(json!({ "invented_filter": "from:me" })) .await .expect("per-action contract gate"); assert!(contract_result.is_error); diff --git a/tests/raw_coverage/embeddings_ollama_raw_coverage_e2e.rs b/tests/raw_coverage/embeddings_ollama_raw_coverage_e2e.rs deleted file mode 100644 index e0e9840af..000000000 --- a/tests/raw_coverage/embeddings_ollama_raw_coverage_e2e.rs +++ /dev/null @@ -1,828 +0,0 @@ -//! Raw-line oriented E2E coverage for the Ollama embedding provider. -//! -//! These tests use the public embedding provider API against a local mock -//! Ollama HTTP server. They avoid a real daemon while exercising the same -//! request, validation, and NaN-recovery branches used in production. - -use std::net::SocketAddr; -use std::sync::{Arc, Mutex}; - -use axum::extract::Json; -use axum::extract::State; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::IntoResponse; -use axum::routing::post; -use axum::Router; -use serde_json::{json, Value}; - -use openhuman_core::openhuman::credentials::{ - AuthService, APP_SESSION_PROVIDER, DEFAULT_AUTH_PROFILE_NAME, -}; -use openhuman_core::openhuman::embeddings::catalog; -use openhuman_core::openhuman::embeddings::cloud::{ - OpenHumanCloudEmbedding, DEFAULT_CLOUD_EMBEDDING_DIMENSIONS, DEFAULT_CLOUD_EMBEDDING_MODEL, -}; -use openhuman_core::openhuman::embeddings::cohere::CohereEmbedding; -use openhuman_core::openhuman::embeddings::noop::NoopEmbedding; -use openhuman_core::openhuman::embeddings::ollama::DEFAULT_OLLAMA_URL; -use openhuman_core::openhuman::embeddings::openai::OpenAiEmbedding; -use openhuman_core::openhuman::embeddings::retry_after::{ - backoff_ms_for_attempt, parse_retry_after_ms, BASE_BACKOFF_MS, MAX_BACKOFF_MS, -}; -use openhuman_core::openhuman::embeddings::voyage::VoyageEmbedding; -use openhuman_core::openhuman::embeddings::{ - create_embedding_provider, create_embedding_provider_with_credentials, EmbeddingProvider, - OllamaEmbedding, DEFAULT_OLLAMA_DIMENSIONS, DEFAULT_OLLAMA_MODEL, -}; - -async fn serve_mock_ollama(app: Router) -> String { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind mock ollama"); - let addr: SocketAddr = listener.local_addr().expect("mock ollama addr"); - tokio::spawn(async move { - axum::serve(listener, app).await.expect("mock ollama serve"); - }); - format!("http://127.0.0.1:{}", addr.port()) -} - -#[derive(Clone, Copy)] -enum OpenAiMockBehavior { - RetryThenSuccess, - CountMismatch, - BadEmbeddingItem, - MissingEmbedding, - DimensionMismatch, - MissingData, - Non2xx, -} - -#[derive(Clone)] -struct OpenAiMockState { - behavior: OpenAiMockBehavior, - attempts: Arc>, - requests: Arc>>, - auth_headers: Arc>>>, -} - -impl OpenAiMockState { - fn new(behavior: OpenAiMockBehavior) -> Self { - Self { - behavior, - attempts: Arc::new(Mutex::new(0)), - requests: Arc::new(Mutex::new(Vec::new())), - auth_headers: Arc::new(Mutex::new(Vec::new())), - } - } -} - -async fn serve_mock_openai(behavior: OpenAiMockBehavior) -> (String, OpenAiMockState) { - let state = OpenAiMockState::new(behavior); - let app = Router::new() - .route("/v1/embeddings", post(mock_openai_handler)) - .route("/openai/v1/embeddings", post(mock_openai_handler)) - .route("/api/v2/embeddings", post(mock_openai_handler)) - .route("/embeddings", post(mock_openai_handler)) - .with_state(state.clone()); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind mock openai"); - let addr = listener.local_addr().expect("mock openai addr"); - tokio::spawn(async move { - axum::serve(listener, app).await.expect("mock openai serve"); - }); - (format!("http://127.0.0.1:{}", addr.port()), state) -} - -async fn serve_mock_cohere(app: Router) -> String { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind mock cohere"); - let addr = listener.local_addr().expect("mock cohere addr"); - tokio::spawn(async move { - axum::serve(listener, app).await.expect("mock cohere serve"); - }); - format!("http://127.0.0.1:{}", addr.port()) -} - -fn record_openai_request(state: &OpenAiMockState, headers: &HeaderMap, body: Value) -> usize { - let mut attempts = state.attempts.lock().expect("attempts lock"); - *attempts += 1; - let attempt = *attempts; - drop(attempts); - - state.requests.lock().expect("requests lock").push(body); - state.auth_headers.lock().expect("auth headers lock").push( - headers - .get(axum::http::header::AUTHORIZATION) - .and_then(|value| value.to_str().ok()) - .map(ToOwned::to_owned), - ); - attempt -} - -async fn mock_openai_handler( - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> axum::response::Response { - let attempt = record_openai_request(&state, &headers, body); - - match state.behavior { - OpenAiMockBehavior::RetryThenSuccess if attempt == 1 => ( - StatusCode::TOO_MANY_REQUESTS, - [(axum::http::header::RETRY_AFTER, "0")], - "slow down", - ) - .into_response(), - OpenAiMockBehavior::RetryThenSuccess => Json(json!({ - "data": [ - { "embedding": [1.0, 2.0] }, - { "embedding": [3.0, 4.0] } - ] - })) - .into_response(), - OpenAiMockBehavior::CountMismatch => { - Json(json!({ "data": [{ "embedding": [1.0, 2.0] }] })).into_response() - } - OpenAiMockBehavior::BadEmbeddingItem => { - Json(json!({ "data": [{ "embedding": [1.0, "bad"] }] })).into_response() - } - OpenAiMockBehavior::MissingEmbedding => Json(json!({ "data": [{}] })).into_response(), - OpenAiMockBehavior::DimensionMismatch => { - Json(json!({ "data": [{ "embedding": [1.0, 2.0, 3.0] }] })).into_response() - } - OpenAiMockBehavior::MissingData => Json(json!({ "not_data": [] })).into_response(), - OpenAiMockBehavior::Non2xx => { - (StatusCode::BAD_REQUEST, "bad embedding request").into_response() - } - } -} - -#[tokio::test] -async fn openai_embed_retries_and_round_trips_auth_body_and_vectors() { - let (base_url, state) = serve_mock_openai(OpenAiMockBehavior::RetryThenSuccess).await; - let provider = OpenAiEmbedding::new(&base_url, "test-key", "mock-openai", 2); - - assert_eq!(provider.name(), "openai"); - assert_eq!(provider.model_id(), "mock-openai"); - assert_eq!(provider.dimensions(), 2); - assert_eq!(provider.base_url(), base_url); - assert_eq!(provider.model(), "mock-openai"); - assert_eq!( - provider.embeddings_url(), - format!("{base_url}/v1/embeddings") - ); - - let vectors = provider - .embed(&["first", "second"]) - .await - .expect("openai retry success"); - - assert_eq!(vectors, vec![vec![1.0, 2.0], vec![3.0, 4.0]]); - assert_eq!(*state.attempts.lock().expect("attempts lock"), 2); - - let auth_headers = state.auth_headers.lock().expect("auth headers lock"); - assert_eq!( - auth_headers.as_slice(), - [ - Some("Bearer test-key".to_string()), - Some("Bearer test-key".to_string()) - ] - ); - drop(auth_headers); - - let requests = state.requests.lock().expect("requests lock"); - assert_eq!( - requests[0].get("model").and_then(Value::as_str), - Some("mock-openai") - ); - assert_eq!( - requests[0].pointer("/input/0").and_then(Value::as_str), - Some("first") - ); - assert_eq!( - requests[0].pointer("/input/1").and_then(Value::as_str), - Some("second") - ); -} - -#[tokio::test] -async fn openai_embed_handles_explicit_paths_empty_inputs_and_missing_auth() { - let (base_url, state) = serve_mock_openai(OpenAiMockBehavior::RetryThenSuccess).await; - let api_provider = OpenAiEmbedding::new(&format!("{base_url}/api/v2"), "", "mock-path", 2); - assert_eq!( - api_provider.embeddings_url(), - format!("{base_url}/api/v2/embeddings") - ); - - let vectors = api_provider - .embed(&["first", "second"]) - .await - .expect("explicit api path success"); - assert_eq!(vectors, vec![vec![1.0, 2.0], vec![3.0, 4.0]]); - assert_eq!( - state.auth_headers.lock().expect("auth headers lock").last(), - Some(&None) - ); - - let endpoint_provider = OpenAiEmbedding::new(&format!("{base_url}/embeddings"), "", "m", 2); - assert_eq!( - endpoint_provider.embeddings_url(), - format!("{base_url}/embeddings") - ); - assert_eq!( - endpoint_provider.embed(&[]).await.expect("empty input"), - Vec::>::new() - ); - - let invalid_url_provider = OpenAiEmbedding::new("not-a-url", "", "m", 0); - assert_eq!( - invalid_url_provider.embeddings_url(), - "not-a-url/v1/embeddings" - ); -} - -#[tokio::test] -async fn openai_embed_reports_response_validation_and_http_errors() { - let (count_url, _) = serve_mock_openai(OpenAiMockBehavior::CountMismatch).await; - let count_provider = OpenAiEmbedding::new(&count_url, "k", "m", 2); - assert!(count_provider - .embed(&["a", "b"]) - .await - .expect_err("count mismatch") - .to_string() - .contains("count mismatch")); - - let (bad_item_url, _) = serve_mock_openai(OpenAiMockBehavior::BadEmbeddingItem).await; - let bad_item_provider = OpenAiEmbedding::new(&bad_item_url, "k", "m", 2); - assert!(bad_item_provider - .embed(&["a"]) - .await - .expect_err("non numeric") - .to_string() - .contains("non-numeric")); - - let (missing_item_url, _) = serve_mock_openai(OpenAiMockBehavior::MissingEmbedding).await; - let missing_item_provider = OpenAiEmbedding::new(&missing_item_url, "k", "m", 2); - assert!(missing_item_provider - .embed(&["a"]) - .await - .expect_err("missing embedding") - .to_string() - .contains("missing `embedding`")); - - let (dim_url, _) = serve_mock_openai(OpenAiMockBehavior::DimensionMismatch).await; - let dim_provider = OpenAiEmbedding::new(&dim_url, "k", "m", 2); - assert!(dim_provider - .embed(&["a"]) - .await - .expect_err("dimension mismatch") - .to_string() - .contains("dimension mismatch")); - - let (missing_url, _) = serve_mock_openai(OpenAiMockBehavior::MissingData).await; - let missing_provider = OpenAiEmbedding::new(&missing_url, "k", "m", 2); - assert!(missing_provider - .embed(&["a"]) - .await - .expect_err("missing data") - .to_string() - .contains("missing `data`")); - - let (non_2xx_url, _) = serve_mock_openai(OpenAiMockBehavior::Non2xx).await; - let non_2xx_provider = OpenAiEmbedding::new(&non_2xx_url, "k", "m", 2); - assert!(non_2xx_provider - .embed(&["a"]) - .await - .expect_err("http error") - .to_string() - .contains("returned HTTP")); -} - -#[tokio::test] -async fn cloud_embedding_uses_seeded_session_token_and_reports_missing_auth() { - let (base_url, state) = serve_mock_openai(OpenAiMockBehavior::RetryThenSuccess).await; - let state_dir = tempfile::tempdir().expect("tempdir"); - AuthService::new(state_dir.path(), false) - .store_provider_token( - APP_SESSION_PROVIDER, - DEFAULT_AUTH_PROFILE_NAME, - "cloud-session-token", - Default::default(), - true, - ) - .expect("seed cloud auth"); - - let provider = OpenHumanCloudEmbedding::new( - Some(format!("{base_url}/")), - Some(state_dir.path().to_path_buf()), - false, - "cloud-model", - 2, - ); - assert_eq!(provider.name(), "cloud"); - assert_eq!(provider.model_id(), "cloud-model"); - assert_eq!(provider.dimensions(), 2); - - let vectors = provider - .embed(&["first", "second"]) - .await - .expect("cloud embed"); - assert_eq!(vectors, vec![vec![1.0, 2.0], vec![3.0, 4.0]]); - assert_eq!( - state.auth_headers.lock().expect("auth headers lock").last(), - Some(&Some("Bearer cloud-session-token".to_string())) - ); - - let missing_auth = OpenHumanCloudEmbedding::new( - Some(base_url), - Some( - tempfile::tempdir() - .expect("missing auth dir") - .path() - .to_path_buf(), - ), - false, - "cloud-model", - 2, - ); - assert!(missing_auth - .embed(&["needs-auth"]) - .await - .expect_err("missing backend session") - .to_string() - .contains("No backend session for cloud embeddings")); -} - -#[tokio::test] -async fn cohere_and_voyage_embedding_paths_use_local_compatible_mocks() { - let cohere_requests = Arc::new(Mutex::new(Vec::::new())); - let cohere_auth = Arc::new(Mutex::new(Vec::>::new())); - let cohere_requests_for_route = cohere_requests.clone(); - let cohere_auth_for_route = cohere_auth.clone(); - let cohere_url = serve_mock_cohere(Router::new().route( - "/v2/embed", - post(move |headers: HeaderMap, Json(body): Json| { - let cohere_requests = cohere_requests_for_route.clone(); - let cohere_auth = cohere_auth_for_route.clone(); - async move { - cohere_requests.lock().expect("cohere requests").push(body); - cohere_auth.lock().expect("cohere auth").push( - headers - .get(axum::http::header::AUTHORIZATION) - .and_then(|value| value.to_str().ok()) - .map(ToOwned::to_owned), - ); - Json(json!({ "embeddings": { "float": [[0.1, 0.2], [0.3, 0.4]] } })) - } - }), - )) - .await; - - let cohere = - CohereEmbedding::new("cohere-key", "embed-multilingual-v3.0", 2).with_base_url(cohere_url); - assert_eq!(cohere.name(), "cohere"); - assert_eq!(cohere.model_id(), "embed-multilingual-v3.0"); - assert_eq!(cohere.dimensions(), 2); - assert_eq!( - cohere - .embed(&["alpha", "beta"]) - .await - .expect("cohere embed"), - vec![vec![0.1, 0.2], vec![0.3, 0.4]] - ); - assert_eq!( - cohere_auth.lock().expect("cohere auth").as_slice(), - [Some("Bearer cohere-key".to_string())] - ); - assert_eq!( - cohere_requests.lock().expect("cohere requests")[0].pointer("/texts/0"), - Some(&json!("alpha")) - ); - - let (voyage_url, voyage_state) = serve_mock_openai(OpenAiMockBehavior::RetryThenSuccess).await; - let voyage = VoyageEmbedding::new_with_base_url("voyage-key", "", 2, &voyage_url); - assert_eq!(voyage.name(), "voyage"); - assert_eq!(voyage.model_id(), "voyage-3-large"); - assert_eq!(voyage.dimensions(), 2); - assert_eq!( - voyage - .embed(&["first", "second"]) - .await - .expect("voyage embed"), - vec![vec![1.0, 2.0], vec![3.0, 4.0]] - ); - assert!(voyage_state - .auth_headers - .lock() - .expect("voyage auth") - .iter() - .any(|header| header.as_deref() == Some("Bearer voyage-key"))); -} - -#[tokio::test] -async fn cohere_embedding_reports_parse_count_dimension_and_http_errors() { - let count_url = serve_mock_cohere(Router::new().route( - "/v2/embed", - post(|| async { Json(json!({ "embeddings": { "float": [[1.0, 2.0]] } })) }), - )) - .await; - let count_provider = CohereEmbedding::new("k", "m", 2).with_base_url(count_url); - assert!(count_provider - .embed(&["a", "b"]) - .await - .expect_err("cohere count mismatch") - .to_string() - .contains("count mismatch")); - - let dim_url = serve_mock_cohere(Router::new().route( - "/v2/embed", - post(|| async { Json(json!({ "embeddings": { "float": [[1.0, 2.0, 3.0]] } })) }), - )) - .await; - let dim_provider = CohereEmbedding::new("k", "m", 2).with_base_url(dim_url); - assert!(dim_provider - .embed(&["a"]) - .await - .expect_err("cohere dimension mismatch") - .to_string() - .contains("dimension mismatch")); - - let malformed_url = serve_mock_cohere( - Router::new().route("/v2/embed", post(|| async { (StatusCode::OK, "not-json") })), - ) - .await; - let malformed_provider = CohereEmbedding::new("k", "m", 2).with_base_url(malformed_url); - assert!(malformed_provider - .embed(&["a"]) - .await - .expect_err("cohere parse") - .to_string() - .contains("parse failed")); - - let non_2xx_url = serve_mock_cohere(Router::new().route( - "/v2/embed", - post(|| async { (StatusCode::BAD_REQUEST, "bad cohere request") }), - )) - .await; - let non_2xx_provider = CohereEmbedding::new("k", "m", 2).with_base_url(non_2xx_url); - assert!(non_2xx_provider - .embed(&["a"]) - .await - .expect_err("cohere http error") - .to_string() - .contains("Cohere embed API error")); -} - -#[tokio::test] -async fn embedding_rate_limit_public_paths_cover_disabled_loopback_and_malformed_urls() { - use openhuman_core::openhuman::embeddings::rate_limit::{ - acquire_embedding_slot, embedding_rate_limit, set_embedding_rate_limit, - }; - - let original = embedding_rate_limit(); - set_embedding_rate_limit(0); - assert_eq!(embedding_rate_limit(), 0); - acquire_embedding_slot("https://api.example.invalid/openai/v1").await; - - set_embedding_rate_limit(60_000); - acquire_embedding_slot("http://localhost:11434").await; - acquire_embedding_slot("http://[::1]:11434").await; - acquire_embedding_slot("not-a-url").await; - - set_embedding_rate_limit(original); -} - -#[test] -fn ollama_constructor_normalizes_defaults_and_rejects_runtime_misconfiguration() { - let defaults = OllamaEmbedding::try_new(" ", " ", 0).expect("default ollama config"); - assert_eq!(defaults.base_url(), DEFAULT_OLLAMA_URL); - assert_eq!(defaults.model(), DEFAULT_OLLAMA_MODEL); - assert_eq!(defaults.dimensions(), DEFAULT_OLLAMA_DIMENSIONS); - - let custom = OllamaEmbedding::try_new("http://[::1]:11434/", " nomic-embed-text ", 12) - .expect("custom ollama config"); - assert_eq!(custom.base_url(), "http://[::1]:11434"); - assert_eq!(custom.model(), "nomic-embed-text"); - assert_eq!( - custom.signature(), - "provider=ollama;model=nomic-embed-text;dims=12" - ); - - let explicit = OllamaEmbedding::new("http://127.0.0.1:11434", "mock-model", 3); - assert_eq!(explicit.base_url(), "http://127.0.0.1:11434"); - assert_eq!(explicit.model(), "mock-model"); - - let default = OllamaEmbedding::default(); - assert_eq!(default.base_url(), DEFAULT_OLLAMA_URL); - assert_eq!(default.model(), DEFAULT_OLLAMA_MODEL); - - for bad_url in [ - "ftp://localhost:11434", - "http://user:pass@localhost:11434", - "http://localhost:11434/api", - "http://localhost:11434/v1/chat/completions", - "http://localhost:11434?debug=true", - "http://localhost:11434/#fragment", - ] { - assert!( - OllamaEmbedding::try_new(bad_url, "m", 1).is_err(), - "bad Ollama URL should be rejected: {bad_url}" - ); - } - assert!(OllamaEmbedding::try_new("http://localhost:11434", "local-v1", 1).is_err()); -} - -#[tokio::test] -async fn embedding_catalog_factory_retry_noop_and_cloud_empty_paths_are_reachable() { - let providers = catalog::all_providers(); - assert!(providers.iter().any(|provider| provider.slug == "managed")); - assert!(providers.iter().any(|provider| provider.slug == "cohere")); - assert_eq!( - catalog::find_provider("openai") - .expect("openai provider") - .label, - "OpenAI" - ); - assert!(catalog::find_provider("missing").is_none()); - assert_eq!( - catalog::find_model("voyage", "voyage-3-large") - .expect("voyage model") - .default_dimensions, - 1024 - ); - assert!(catalog::find_model("voyage", "missing").is_none()); - assert_eq!( - catalog::default_model_for("openai") - .expect("default openai model") - .id, - "text-embedding-3-small" - ); - assert!(catalog::default_model_for("none").is_none()); - - assert_eq!(parse_retry_after_ms(Some(" 5 ")), Some(5_000)); - assert_eq!( - parse_retry_after_ms(Some("Wed, 21 Oct 2015 07:28:00 GMT")), - Some(0) - ); - assert_eq!(parse_retry_after_ms(Some("99999")), Some(MAX_BACKOFF_MS)); - assert_eq!(backoff_ms_for_attempt(2, Some("1")), 1_000); - assert_eq!(backoff_ms_for_attempt(1, None), BASE_BACKOFF_MS * 2); - assert_eq!(backoff_ms_for_attempt(10, Some("bad")), MAX_BACKOFF_MS); - - let noop = NoopEmbedding; - assert_eq!(noop.name(), "none"); - assert_eq!(noop.model_id(), "none"); - assert_eq!(noop.dimensions(), 0); - assert_eq!(noop.signature(), "provider=none;model=none;dims=0"); - assert_eq!( - noop.embed(&["ignored"]).await.expect("noop embed"), - Vec::>::new() - ); - assert!(noop - .embed_one("ignored") - .await - .expect_err("noop embed_one") - .to_string() - .contains("Empty embedding result")); - - let cloud = OpenHumanCloudEmbedding::new( - Some("https://api.example.test/".to_string()), - None, - false, - DEFAULT_CLOUD_EMBEDDING_MODEL, - DEFAULT_CLOUD_EMBEDDING_DIMENSIONS, - ); - assert_eq!(cloud.name(), "cloud"); - assert!(cloud - .embed(&[]) - .await - .expect("cloud empty embed") - .is_empty()); - - for (provider, model, dims, expected_name) in [ - ( - "managed", - DEFAULT_CLOUD_EMBEDDING_MODEL, - DEFAULT_CLOUD_EMBEDDING_DIMENSIONS, - "cloud", - ), - ("voyage", "", 0, "voyage"), - ("cohere", "", 0, "cohere"), - ("openai", "text-embedding-3-small", 1536, "openai"), - ("custom:http://127.0.0.1:9", "custom-embedding", 2, "openai"), - ("none", "", 0, "none"), - ] { - let embedder = - create_embedding_provider(provider, model, dims).expect("provider should construct"); - assert_eq!(embedder.name(), expected_name); - } - - match create_embedding_provider("unknown", "m", 1) { - Ok(_) => panic!("unknown provider should fail"), - Err(err) => assert!(err.to_string().contains("unknown embedding provider")), - } - - let default_cloud = openhuman_core::openhuman::embeddings::default_embedding_provider(); - assert_eq!(default_cloud.name(), "cloud"); - let default_local = openhuman_core::openhuman::embeddings::default_local_embedding_provider(); - assert_eq!(default_local.name(), "ollama"); - - for (provider, model, dims, key, endpoint, expected_name) in [ - ( - "managed", - DEFAULT_CLOUD_EMBEDDING_MODEL, - 1024, - "ignored", - None, - "cloud", - ), - ( - "voyage", - "voyage-3-large", - 1024, - "voyage-key", - None, - "voyage", - ), - ("ollama", DEFAULT_OLLAMA_MODEL, 1024, "", None, "ollama"), - ( - "openai", - "text-embedding-3-small", - 1536, - "openai-key", - None, - "openai", - ), - ( - "cohere", - "embed-english-v3.0", - 1024, - "cohere-key", - None, - "cohere", - ), - ( - "custom", - "custom-model", - 768, - "custom-key", - Some("http://127.0.0.1:9"), - "openai", - ), - ( - "custom:http://127.0.0.1:8", - "custom-model", - 768, - "custom-key", - None, - "openai", - ), - ("none", "", 0, "", None, "none"), - ] { - let embedder = - create_embedding_provider_with_credentials(provider, model, dims, key, endpoint) - .expect("provider with credentials should construct"); - assert_eq!(embedder.name(), expected_name); - } - - match create_embedding_provider_with_credentials("bogus", "m", 1, "k", None) { - Ok(_) => panic!("unknown provider with credentials should fail"), - Err(err) => assert!(err.to_string().contains("unknown embedding provider")), - } -} - -#[tokio::test] -async fn ollama_embed_preserves_positions_and_validates_request_and_response() { - let app = Router::new().route( - "/api/embed", - post(|Json(body): Json| async move { - assert_eq!( - body.get("model").and_then(Value::as_str), - Some("mock-ollama") - ); - assert_eq!( - body.pointer("/input/0").and_then(Value::as_str), - Some("alpha") - ); - assert_eq!( - body.pointer("/input/1").and_then(Value::as_str), - Some("beta") - ); - Json(json!({ "embeddings": [[1.0, 2.0], [3.0, 4.0]] })) - }), - ); - let base_url = serve_mock_ollama(app).await; - let provider = OllamaEmbedding::try_new(&base_url, "mock-ollama", 2).expect("provider"); - - let mixed_error = provider - .embed(&[" alpha ", "", "beta", " "]) - .await - .expect_err("mixed blank batch should be rejected"); - assert!(mixed_error.to_string().contains("must not mix blank")); - - let vectors = provider - .embed(&[" alpha ", "beta"]) - .await - .expect("ollama embed"); - assert_eq!(vectors, vec![vec![1.0, 2.0], vec![3.0, 4.0]]); - - let all_blank = provider.embed(&["", " \n\t "]).await.expect("blank embed"); - assert_eq!(all_blank, vec![Vec::::new(), Vec::::new()]); -} - -#[tokio::test] -async fn ollama_embed_reports_malformed_count_dimension_and_transport_errors() { - let count_url = serve_mock_ollama(Router::new().route( - "/api/embed", - post(|| async { Json(json!({ "embeddings": [[1.0]] })) }), - )) - .await; - let count_provider = OllamaEmbedding::try_new(&count_url, "m", 1).expect("count provider"); - assert!(count_provider - .embed(&["a", "b"]) - .await - .expect_err("count mismatch") - .to_string() - .contains("count mismatch")); - - let dim_url = serve_mock_ollama(Router::new().route( - "/api/embed", - post(|| async { Json(json!({ "embeddings": [[1.0, 2.0, 3.0]] })) }), - )) - .await; - let dim_provider = OllamaEmbedding::try_new(&dim_url, "m", 2).expect("dim provider"); - assert!(dim_provider - .embed(&["a"]) - .await - .expect_err("dimension mismatch") - .to_string() - .contains("dimension mismatch")); - - let malformed_url = serve_mock_ollama(Router::new().route( - "/api/embed", - post(|| async { (StatusCode::OK, "not json") }), - )) - .await; - let malformed_provider = - OllamaEmbedding::try_new(&malformed_url, "m", 2).expect("malformed provider"); - assert!(malformed_provider - .embed(&["a"]) - .await - .expect_err("malformed response") - .to_string() - .contains("parse failed")); - - let refused = OllamaEmbedding::try_new("http://127.0.0.1:1", "m", 2).expect("refused provider"); - assert!(refused - .embed(&["a"]) - .await - .expect_err("connection refused") - .to_string() - .contains("is Ollama running")); -} - -#[tokio::test] -async fn ollama_embed_recovers_nan_batch_with_per_text_fallback() { - let app = Router::new().route( - "/api/embed", - post(|Json(body): Json| async move { - let inputs = body - .get("input") - .and_then(Value::as_array) - .expect("input array"); - if inputs.len() > 1 { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - r#"{"error":"failed to encode response: json: unsupported value: NaN"}"# - .to_string(), - ); - } - if inputs.first().and_then(Value::as_str) == Some("bad") { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - "unsupported value: nan".to_string(), - ); - } - ( - StatusCode::OK, - json!({ "embeddings": [[9.0, 8.0]] }).to_string(), - ) - }), - ); - let base_url = serve_mock_ollama(app).await; - let provider = OllamaEmbedding::try_new(&base_url, "mock-ollama", 2).expect("provider"); - - let batch_error = provider - .embed(&["good", "bad"]) - .await - .expect_err("a NaN-producing item should fail the recovered batch"); - assert!(batch_error.to_string().contains("without NaN values")); - - let single_error = provider - .embed(&["bad"]) - .await - .expect_err("single NaN-producing input should fail"); - assert!(single_error.to_string().contains("without NaN values")); -} diff --git a/tests/raw_coverage/memory_sync_providers_raw_coverage_e2e.rs b/tests/raw_coverage/memory_sync_providers_raw_coverage_e2e.rs index fe893a08b..60241c4e2 100644 --- a/tests/raw_coverage/memory_sync_providers_raw_coverage_e2e.rs +++ b/tests/raw_coverage/memory_sync_providers_raw_coverage_e2e.rs @@ -19,7 +19,7 @@ use openhuman_core::openhuman::credentials::{ AuthService, APP_SESSION_PROVIDER, DEFAULT_AUTH_PROFILE_NAME, }; use openhuman_core::openhuman::memory::global as memory_global; -use openhuman_core::openhuman::memory::jobs::drain_until_idle; +use openhuman_core::openhuman::memory_queue::drain_until_idle; use openhuman_core::openhuman::memory_sync::composio::bus::{ ComposioConfigChangedSubscriber, ComposioConnectionCreatedSubscriber, ComposioTriggerSubscriber, }; diff --git a/tests/raw_coverage/memory_tree_embed_round25_raw_coverage_e2e.rs b/tests/raw_coverage/memory_tree_embed_round25_raw_coverage_e2e.rs index c79c21158..62d2cfcc4 100644 --- a/tests/raw_coverage/memory_tree_embed_round25_raw_coverage_e2e.rs +++ b/tests/raw_coverage/memory_tree_embed_round25_raw_coverage_e2e.rs @@ -2,8 +2,9 @@ use axum::extract::Json; use axum::http::StatusCode; use axum::routing::post; use axum::Router; -use openhuman_core::openhuman::memory_tree::score::embed::{ - Embedder, OllamaEmbedder, EMBEDDING_DIM, +use openhuman_core::openhuman::memory_tree::score::embed::EMBEDDING_DIM; +use tinyagents::harness::embeddings::{ + EmbeddingModel, OllamaEmbeddingModel, RECOMMENDED_OLLAMA_CONTEXT_TOKENS, }; use serde_json::{json, Value}; @@ -24,38 +25,43 @@ async fn start_embed_server(app: Router) -> String { async fn round25_ollama_embedder_covers_success_and_error_edges_without_real_ollama() { let success_vec = vec![0.125_f32; EMBEDDING_DIM]; let app = Router::new().route( - "/api/embeddings", + "/api/embed", post({ let success_vec = success_vec.clone(); move |Json(body): Json| { let success_vec = success_vec.clone(); async move { assert_eq!(body["model"], "round25-embed"); - assert_eq!(body["prompt"], "memory tree round25"); + assert_eq!(body["input"][0], "memory tree round25"); assert_eq!(body["options"]["num_ctx"], 8192); - Json(json!({ "embedding": success_vec })) + assert_eq!(body["options"]["num_batch"], 8192); + Json(json!({ "embeddings": [success_vec] })) } } }), ); let url = start_embed_server(app).await; - let embedder = OllamaEmbedder::new(format!("{url}/"), "round25-embed".to_string(), 0); + let embedder = OllamaEmbeddingModel::new(&format!("{url}/"), "round25-embed", EMBEDDING_DIM) + .with_context_options( + RECOMMENDED_OLLAMA_CONTEXT_TOKENS, + RECOMMENDED_OLLAMA_CONTEXT_TOKENS, + ); assert_eq!(embedder.name(), "ollama"); let embedding = embedder - .embed("memory tree round25") + .embed(&["memory tree round25".to_string()]) .await .expect("loopback embedding"); - assert_eq!(embedding.len(), EMBEDDING_DIM); - assert!((embedding[0] - 0.125).abs() < f32::EPSILON); + assert_eq!(embedding[0].len(), EMBEDDING_DIM); + assert!((embedding[0][0] - 0.125).abs() < f32::EPSILON); let missing_model_url = start_embed_server(Router::new().route( - "/api/embeddings", + "/api/embed", post(|| async { (StatusCode::NOT_FOUND, "{\"error\":\"model not found\"}") }), )) .await; - let missing = OllamaEmbedder::new(missing_model_url, "missing-round25".to_string(), 500); + let missing = OllamaEmbeddingModel::new(&missing_model_url, "missing-round25", EMBEDDING_DIM); let missing_err = missing - .embed("text") + .embed(&["text".to_string()]) .await .expect_err("missing model should fail") .to_string(); @@ -63,27 +69,27 @@ async fn round25_ollama_embedder_covers_success_and_error_edges_without_real_oll assert!(missing_err.contains("ollama pull missing-round25")); let dim_url = start_embed_server(Router::new().route( - "/api/embeddings", - post(|| async { Json(json!({ "embedding": [0.1, 0.2, 0.3] })) }), + "/api/embed", + post(|| async { Json(json!({ "embeddings": [[0.1, 0.2, 0.3]] })) }), )) .await; - let dim_mismatch = OllamaEmbedder::new(dim_url, String::new(), 0); + let dim_mismatch = OllamaEmbeddingModel::new(&dim_url, "", EMBEDDING_DIM); let dim_err = dim_mismatch - .embed("text") + .embed(&["text".to_string()]) .await .expect_err("wrong dimensions should fail") .to_string(); - assert!(dim_err.contains("3 dims")); + assert!(dim_err.contains("got 3")); assert!(dim_err.contains("expected 1024")); let bad_json_url = start_embed_server(Router::new().route( - "/api/embeddings", + "/api/embed", post(|| async { (StatusCode::OK, "not-json") }), )) .await; - let bad_json = OllamaEmbedder::new(bad_json_url, String::new(), 0); + let bad_json = OllamaEmbeddingModel::new(&bad_json_url, "", EMBEDDING_DIM); let parse_err = bad_json - .embed("text") + .embed(&["text".to_string()]) .await .expect_err("invalid json should fail") .to_string(); diff --git a/vendor/tinyagents b/vendor/tinyagents index 2583fccc2..07467c34a 160000 --- a/vendor/tinyagents +++ b/vendor/tinyagents @@ -1 +1 @@ -Subproject commit 2583fccc213a00f2a3d94744ff1e0d1541368f97 +Subproject commit 07467c34a69bb21409732b739aa84b161dd4cd07 diff --git a/vendor/tinycortex b/vendor/tinycortex index daaaf6ba5..55bf0661d 160000 --- a/vendor/tinycortex +++ b/vendor/tinycortex @@ -1 +1 @@ -Subproject commit daaaf6ba5f02635c08deae2b2b2ed7fcc8c06b6a +Subproject commit 55bf0661d003dd38dc33e8da45b46dd675710c9c