mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-27 21:08:00 +00:00
refactor(memory): finish TinyCortex consolidation (#5140)
This commit is contained in:
@@ -856,7 +856,7 @@ jobs:
|
||||
key: tinycortex
|
||||
|
||||
- name: Run TinyCortex engine and Composio sync tests
|
||||
run: bash scripts/ci-cancel-aware.sh cargo test --manifest-path vendor/tinycortex/Cargo.toml --features git-diff,sync
|
||||
run: bash scripts/ci-cancel-aware.sh cargo test --manifest-path vendor/tinycortex/Cargo.toml --features git-diff,sync,persona
|
||||
|
||||
pr-ci-gate:
|
||||
name: PR CI Gate
|
||||
|
||||
Generated
+5
-22
@@ -3079,7 +3079,7 @@ dependencies = [
|
||||
"js-sys",
|
||||
"log",
|
||||
"wasm-bindgen",
|
||||
"windows-core 0.62.2",
|
||||
"windows-core 0.57.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4746,7 +4746,7 @@ dependencies = [
|
||||
"tar",
|
||||
"tempfile",
|
||||
"thiserror 2.0.18",
|
||||
"tinyagents 2.1.0",
|
||||
"tinyagents",
|
||||
"tinychannels",
|
||||
"tinycortex",
|
||||
"tinyflows",
|
||||
@@ -7332,23 +7332,6 @@ dependencies = [
|
||||
"crunchy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyagents"
|
||||
version = "2.0.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"futures",
|
||||
"reqwest 0.12.28",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.11.0",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyagents"
|
||||
version = "2.1.0"
|
||||
@@ -7433,7 +7416,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"thiserror 2.0.18",
|
||||
"tinyagents 2.0.0",
|
||||
"tinyagents",
|
||||
"tokio",
|
||||
"toml 0.8.23",
|
||||
"tracing",
|
||||
@@ -7453,7 +7436,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.18",
|
||||
"tinyagents 2.1.0",
|
||||
"tinyagents",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
@@ -8864,7 +8847,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
+5
-9
@@ -109,10 +109,12 @@ tinyagents = { version = "2.1", features = ["sqlite"] }
|
||||
# queue/ingest/score + long tail), vendored as a git submodule and patched
|
||||
# below to `vendor/tinycortex`. OpenHuman's memory subsystem migrates onto this
|
||||
# crate through the adapter seam in `src/openhuman/tinycortex/` (mirroring the
|
||||
# tinyagents seam): engine logic in the crate; RPC, agent tools, live sync,
|
||||
# security gating, and the global singleton stay host-side. rusqlite/git2 are
|
||||
# tinyagents seam): engine logic (including provider sync pipelines) in the
|
||||
# crate; RPC, agent tools, sync scheduling/credentials/events, security gating,
|
||||
# and the global singleton stay host-side. rusqlite/git2 are
|
||||
# aligned to the host pins (=0.40 / 0.21) so one bundled SQLite + one libgit2
|
||||
# link. Keep the version pin in lockstep with the submodule tag.
|
||||
# link. The submodule intentionally tracks reviewed upstream main commits;
|
||||
# keep this semver requirement compatible with the vendored crate version.
|
||||
tinycortex = { version = "0.1", features = ["git-diff", "persona", "sync"] }
|
||||
tinychannels = { version = "0.1", features = ["relay-websocket"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
@@ -704,12 +706,6 @@ tinyjuice = { path = "vendor/tinyjuice" }
|
||||
tinychannels = { path = "vendor/tinychannels" }
|
||||
tinyplace = { path = "vendor/tinyplace/sdk/rust" }
|
||||
|
||||
# TinyCortex temporarily pins the embedding API port while tinyagents #58 is
|
||||
# pending. Resolve that git source to the same vendored crate so host adapters
|
||||
# and TinyCortex share one `EmbeddingModel` trait identity.
|
||||
[patch."https://github.com/senamakel/tinyagents"]
|
||||
tinyagents = { path = "vendor/tinyagents" }
|
||||
|
||||
# Emit just enough DWARF in release builds for Sentry to symbolicate Rust
|
||||
# panics + render surrounding source lines. `line-tables-only` keeps the
|
||||
# binary small (only file+line tables, no full type info) while still
|
||||
|
||||
Generated
+16
-33
@@ -205,7 +205,7 @@ version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -216,7 +216,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2061,7 +2061,7 @@ dependencies = [
|
||||
"libc",
|
||||
"option-ext",
|
||||
"redox_users 0.5.2",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2479,7 +2479,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3706,7 +3706,7 @@ dependencies = [
|
||||
"js-sys",
|
||||
"log",
|
||||
"wasm-bindgen",
|
||||
"windows-core 0.62.2",
|
||||
"windows-core 0.57.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4107,7 +4107,7 @@ dependencies = [
|
||||
"portable-atomic",
|
||||
"portable-atomic-util",
|
||||
"serde_core",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4991,7 +4991,7 @@ version = "0.50.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5603,7 +5603,7 @@ dependencies = [
|
||||
"tar",
|
||||
"tempfile",
|
||||
"thiserror 2.0.18",
|
||||
"tinyagents 2.1.0",
|
||||
"tinyagents",
|
||||
"tinychannels",
|
||||
"tinycortex",
|
||||
"tinyflows",
|
||||
@@ -7133,7 +7133,7 @@ dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7192,7 +7192,7 @@ dependencies = [
|
||||
"security-framework 3.7.0",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7949,7 +7949,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8809,7 +8809,7 @@ dependencies = [
|
||||
"getrandom 0.4.2",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8968,23 +8968,6 @@ dependencies = [
|
||||
"strict-num",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyagents"
|
||||
version = "2.0.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"futures",
|
||||
"reqwest 0.12.28",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.11.0",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyagents"
|
||||
version = "2.1.0"
|
||||
@@ -9064,7 +9047,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"thiserror 2.0.18",
|
||||
"tinyagents 2.0.0",
|
||||
"tinyagents",
|
||||
"tokio",
|
||||
"toml 0.8.2",
|
||||
"tracing",
|
||||
@@ -9084,7 +9067,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.18",
|
||||
"tinyagents 2.1.0",
|
||||
"tinyagents",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
@@ -9670,7 +9653,7 @@ checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e"
|
||||
dependencies = [
|
||||
"memoffset",
|
||||
"tempfile",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10466,7 +10449,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -366,10 +366,10 @@ End-to-end coverage of the agent harness via the web-chat RPC surface against an
|
||||
| ----- | ------------------------------------------ | ----- | -------------------------------------------------------------------------------------------------------------- | ------ | ----------------------------------------------------------------------- |
|
||||
| 8.4.1 | Save Preference (general / situational) | RU | `src/openhuman/agent/tools/save_preference_tests.rs` | ✅ | `save_preference` tool → `user_pref_{general,situational}`, topic-keyed |
|
||||
| 8.4.2 | Lane A — Standing Prefs in System Prompt | RU | `src/openhuman/learning/prompt_sections.rs`, `src/openhuman/agent/harness/session/turn_tests.rs` | ✅ | General prefs rendered into the system prompt at thread start |
|
||||
| 8.4.3 | Lane B — Situational Recall (vector-gated) | RU | `src/openhuman/memory/store/unified/query_tests.rs::recall_relevant_by_vector_gates_on_similarity` | ✅ | Per-turn; relevant query injects, unrelated suppresses |
|
||||
| 8.4.3 | Lane B — Situational Recall (vector-gated) | RU | `src/openhuman/memory_store/namespace_store/query_tests.rs::recall_relevant_by_vector_gates_on_similarity` | ✅ | Per-turn; relevant query injects, unrelated suppresses |
|
||||
| 8.4.4 | Same-Topic Contradiction (replace) | RU | `src/openhuman/agent/tools/save_preference_tests.rs::recategorising_moves_pref_between_namespaces` | ✅ | `ON CONFLICT REPLACE`; a topic lives in exactly one scope |
|
||||
| 8.4.5 | Cross-Topic Contradiction Surfacing | RU | `src/openhuman/agent/tools/save_preference_tests.rs::save_surfaces_related_preference_for_contradiction_check` | ✅ | Related prefs surfaced in the tool result for the chat agent to resolve |
|
||||
| 8.4.6 | vector_chunks Model-Signature Recall Guard | RU | `src/openhuman/memory/store/unified/query_tests.rs::vector_recall_excludes_other_model_signature` | ✅ | Excludes cross-model vectors; dim-guards legacy rows |
|
||||
| 8.4.6 | vector_chunks Model-Signature Recall Guard | RU | `src/openhuman/memory_store/namespace_store/query_tests.rs::vector_recall_excludes_other_model_signature` | ✅ | Excludes cross-model vectors; dim-guards legacy rows |
|
||||
|
||||
### 8.5 Long-term Goals
|
||||
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
# TinyCortex API Gap Audit (Phase 0.2)
|
||||
|
||||
**Status (2026-07-22):** Historical gap inventory for the completed engine
|
||||
cutover. Counts and source anchors below describe the audit base; unresolved
|
||||
host consolidation work, including G1 re-homing and sync dedupe, is tracked in
|
||||
`tinycortex-migration-plan-2026-07-22.md`.
|
||||
|
||||
**Purpose.** Map every host call site into the engine-mapping modules
|
||||
(`memory_store`, `memory_tree`, `memory_queue`, and the long tail) onto a TinyCortex
|
||||
public API. Where no crate API covers a host need, record a **gap** — each gap becomes a
|
||||
@@ -48,16 +53,16 @@ The crate exposes every seam the plan expects, plus more. Host implements these:
|
||||
| `Summariser` (time-tree, string) | `tree/runtime/engine.rs:29` | **divergent — see G6** |
|
||||
| `EntityExtractor` (regex + LLM) | `score/extract/composite.rs:16` | `tinycortex/chat.rs` (LLM variant) |
|
||||
| `QueueDelegates` | `queue/handlers.rs:96` | `tinycortex/queue_driver.rs` |
|
||||
| `TreeJobSink` | `ingest/types.rs:15` | `tinycortex/sinks.rs` |
|
||||
| `TreeLeafSink` | `archivist/sink.rs:44` | `tinycortex/sinks.rs` |
|
||||
| `SnapshotItemSource` | `diff/source.rs:28` | `tinycortex/sinks.rs` |
|
||||
| `EntityOccurrenceIndex` | `graph/types.rs:44` | `tinycortex/sinks.rs` |
|
||||
| `TreeJobSink` | `ingest/types.rs:15` | `tinycortex/ingest.rs` |
|
||||
| `TreeLeafSink` | `archivist/sink.rs:44` | `tinycortex/ingest.rs` |
|
||||
| `SnapshotItemSource` | `diff/source.rs:28` | host diff adapter |
|
||||
| `EntityOccurrenceIndex` | `graph/types.rs:44` | host graph adapter |
|
||||
| `SelfIdentity` | `store/entity_index/store.rs:25` | host identity registry |
|
||||
| `GoalsGenerator` | `goals/reflect.rs:57` | `tinycortex/chat.rs` |
|
||||
| `SourceReader` | `sources/readers/mod.rs:1732` | local readers only *(amended: live-sync readers join the crate in W-SYNC, plan §8)* |
|
||||
| `SyncEventSink` *(new, W-SYNC)* | `sync/traits.rs` (planned) | `tinycortex/sync_sink.rs` → `MemorySyncStage` bus events |
|
||||
| `SkillDocSink` *(new, W-SYNC)* | `sync/traits.rs` (planned) | forwards to `MemoryClient::store_skill_sync` (host-retained unified tier) |
|
||||
| `ConversationEventBus` / `ChannelEventHandler` | `conversations/bus.rs:106/95` | `tinycortex/bus.rs` (translate to `DomainEvent`) |
|
||||
| `ConversationEventBus` / `ChannelEventHandler` | `conversations/bus.rs:106/95` | host `memory_conversations/bus.rs` (translate to `DomainEvent`) |
|
||||
|
||||
Config seam: `MemoryConfig{workspace, embedding: EmbeddingConfig{dim,model,strict},
|
||||
tree: TreeConfig{input_token_budget,output_token_budget,summary_fanout,flush_age_secs},
|
||||
@@ -86,7 +91,7 @@ retrieval: RetrievalConfig{default_profile}, sync_budget: SyncBudgetConfig}` +
|
||||
|
||||
### G2 — Graph relation-edge persistence (E2GraphRAG accumulation) — **HARD**
|
||||
|
||||
- **Host:** `memory_store/unified/graph.rs`, `unified/query.rs` persist and query co-occurrence
|
||||
- **Host:** `memory_store/namespace_store/{graph,query}.rs` persist and query co-occurrence
|
||||
/ LLM-triple relations; `NamespaceMemoryHit.supporting_relations: Vec<GraphRelationRecord>`
|
||||
is populated at retrieval.
|
||||
- **Crate:** persists the **entity occurrence index** at persist time
|
||||
|
||||
@@ -25,7 +25,9 @@ touched an engine-mapping memory module after the port line, and classifies each
|
||||
| Host audit SHA | `7850cf363559bcbb7ba688cbc4fccdb6bd9ce754` (`main`, 2026-07-04) |
|
||||
| TinyCortex submodule | `vendor/tinycortex` → `tinyhumansai/tinycortex` |
|
||||
| TinyCortex audit SHA | `d1a8c7be2babc8fff7a72ed93861f459f3d6fa58` |
|
||||
| **TinyCortex pinned SHA (current)** | `a8e10f7dd8ebdb9b0905e1380fefcc6bf5a65207` — audit SHA **+ #59** (native-dep alignment, §0.4) **+ #63/#64** (D2/D1 drift ports) merged; this is the live gitlink |
|
||||
| TinyCortex historical cutover SHA | `a8e10f7dd8ebdb9b0905e1380fefcc6bf5a65207` — audit SHA **+ #59** (native-dep alignment, §0.4) **+ #63/#64** (D2/D1 drift ports) merged |
|
||||
| **TinyCortex reviewed gitlink (2026-07-22)** | `daaaf6ba5f02635c08deae2b2b2ed7fcc8c06b6a` — includes OpenHuman #4794/#4820/#4863-era crate work; upstream currently has no tags |
|
||||
| **TinyCortex migration branch (current)** | `7b4b115` — TinyAgents 2.1 dependency alignment, standalone patch configuration, and corrected sync ownership docs |
|
||||
| TinyCortex crate version | `0.1.1` |
|
||||
| **Port line (derived)** | **after 2026-06-25, before 2026-06-28** (see below) |
|
||||
|
||||
@@ -38,6 +40,11 @@ touched an engine-mapping memory module after the port line, and classifies each
|
||||
> (conversations) are now unblocked per the gate rule. D4 (memory_sync corpus,
|
||||
> W-SYNC) remains its own separate track.
|
||||
|
||||
> **Consolidation status (2026-07-22): D1–D4 CLOSED.** D1–D3 remain closed.
|
||||
> D4 was re-audited against the crate sync implementation and the live host
|
||||
> dispatch path. The host schedulers, credentials, RPC, source-scope/redaction
|
||||
> policy, product task normalizers, and event-bus adapters remain host-owned.
|
||||
|
||||
### How the port line was located
|
||||
|
||||
The port commits in `vendor/tinycortex` are all dated **2026-06-29**, but that is the date the
|
||||
@@ -132,26 +139,23 @@ its `tools/` stay host (agent tools), its `vector`/`scoring` are engine (W5) —
|
||||
|
||||
## D4 — memory_sync corpus (2026-07-09 reclassification, W-SYNC)
|
||||
|
||||
The plan's §8 amendment moves the sync engine into the crate, so the blanket **HOST-OWNED
|
||||
"live sync"** classification above is superseded for engine-mapping sync code. Unlike D1–D3 there
|
||||
is **no existing crate port to diff against** — the entire `memory_sync/` engine (plus the
|
||||
`memory_sources` dispatcher/reconcile parts) is port scope. The port baseline for W-SYNC is taken
|
||||
**fresh from host `main` at W-SYNC.1 branch time**, so ordinary drift cannot accumulate; this entry
|
||||
exists to (a) record the reclassification and (b) pin the post-port-line commits that already
|
||||
touched the corpus, so the W-SYNC.1 port provably includes them:
|
||||
The plan's §8 amendment moved the generic sync engine into the crate, so the
|
||||
blanket **HOST-OWNED "live sync"** classification above is superseded for
|
||||
engine-mapping sync code. The crate port landed in `0333d10` and is the path
|
||||
called by `memory_sync::composio::run_connection_sync` and the default
|
||||
`ComposioProvider::sync` implementation.
|
||||
|
||||
| # | Host commit | Files in corpus | Note |
|
||||
| --- | --- | --- | --- |
|
||||
| D4.1 | `c43f79641` (07-03) | `composio/providers/{sync_state,traits}.rs` | TinyAgents-cutover import churn |
|
||||
| D4.2 | `27b00b539` (07-05) | `sources/rebuild.rs` | test-parity cleanup (1 line) |
|
||||
| D4.3 | `653e6e143` (07-06) | `memory_sources/sync.rs` (+312) | self-heal drifted content-sha tokens + prune vanished folder items |
|
||||
| D4.4 | `e456b7799` (07-07) | `memory_sources/{rpc,sync}.rs`, `canonicalize/email.rs`, `composio/providers/{gmail/post_process,notion/source,orchestrator}.rs`, `sources/github.rs` | orchestration-fixes wave |
|
||||
| D4.1 | `c43f79641` (07-03) | `composio/providers/{sync_state,traits}.rs` | ✅ **CLOSED.** Import churn only; persistence and provider dispatch are implemented by the `SyncStateStore` seam and crate dispatcher. |
|
||||
| D4.2 | `27b00b539` (07-05) | `sources/rebuild.rs` | ✅ **CLOSED.** Test-only parity cleanup; no engine semantic delta. |
|
||||
| D4.3 | `653e6e143` (07-06) | `memory_sources/sync.rs` (+312) | ✅ **CLOSED.** Crate `sync/workspace.rs` updates changed items and prunes vanished items through `LocalDocumentSink::delete`; covered by workspace sync tests. |
|
||||
| D4.4 | `e456b7799` (07-07) | `memory_sources/{rpc,sync}.rs`, `canonicalize/email.rs`, `composio/providers/{gmail/post_process,notion/source,orchestrator}.rs`, `sources/github.rs` | ✅ **CLOSED.** Generic provider fetch/pagination/canonical memory records are in the crate port; orchestration, product task normalization, action-result presentation, and RPC fixes are host policy by design. |
|
||||
|
||||
**Gate:** W-SYNC.1 (crate scaffolding PR) requires this enumeration current as of its branch point
|
||||
(re-run the scan: `git log --since=2026-06-25 --oneline -- src/openhuman/memory_sync
|
||||
src/openhuman/memory_sources`); the W-SYNC.3 host flip requires D4 **CLOSED** — every listed commit
|
||||
(and any accrued since) verifiably contained in the crate port or explicitly waived. Host-retained
|
||||
parts (schedulers, bus subscribers, RPC wrappers, keychain/OAuth) remain HOST-OWNED as before.
|
||||
**Gate result:** D4 is **CLOSED**. Gmail's Composio 413 mitigation is additionally
|
||||
present in crate commit `ba9e12e` (25-message fetch pages). Host-retained parts
|
||||
(schedulers, bus subscribers, RPC wrappers, keychain/OAuth, action tools and
|
||||
product task/profile projections) remain HOST-OWNED.
|
||||
|
||||
## Closing the ledger (procedure)
|
||||
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
# TinyCortex Memory Migration — Plan & Audit
|
||||
|
||||
**Status:** W1–W2 landed; W3 partial (chunks sub-store flipped). Amended 2026-07-09 (see §8) to a **sync-inclusive** scope.
|
||||
**Status:** Historical execution plan. W1–W8 and the engine test port are landed
|
||||
(OpenHuman #4794/#4820); persona/coding-session ingest followed in #4863. The
|
||||
remaining consolidation work is tracked in
|
||||
[`tinycortex-migration-plan-2026-07-22.md`](tinycortex-migration-plan-2026-07-22.md).
|
||||
**Anchor precedent:** the TinyAgents harness migration (#4249 / #4399 / #4473 and follow-ups).
|
||||
**Target:** migrate large portions of the OpenHuman memory subsystem onto the `tinycortex` crate, vendored as a git submodule at **`vendor/tinycortex`** (`https://github.com/tinyhumansai/tinycortex`).
|
||||
|
||||
@@ -83,8 +86,8 @@ New sibling module mirroring `src/openhuman/tinyagents/`, holding every impl of
|
||||
- `chat.rs` — `impl ChatProvider` + `impl Summariser` over `memory::chat::build_chat_provider` / `inference::provider`.
|
||||
- `queue_driver.rs` — tokio worker loop calling `queue::run_once`/`drain_until_idle`, owned by `memory/global.rs`, with event-bus progress emission and Sentry hooks (host-side, since TinyCortex dropped its scheduler).
|
||||
- `config.rs` — `Config` → `MemoryConfig` (workspace roots, `EmbeddingConfig`, `TreeConfig`, `WeightProfile`, `SyncBudgetConfig`) with `tree_policy.rs` flavour overlays.
|
||||
- `sinks.rs` — `TreeJobSink`, `SnapshotItemSource` impls bridging to host state.
|
||||
- `bus.rs` — translate engine outcomes into `core::event_bus` `DomainEvent`s (host-side only; the crate stays bus-free).
|
||||
- `ingest.rs` / `seal.rs` — `TreeJobSink`, seal observer, and host state bridges.
|
||||
- `sync.rs` — sync persistence/policy plus translation into host `DomainEvent`s (the crate stays bus-free).
|
||||
- `mod.rs` — facade re-exports (`pub use tinycortex::memory::{…}`) so the rest of the host imports through one seam, plus module-doc explaining the boundary (the tinyagents seam's `mod.rs` header is the template).
|
||||
|
||||
---
|
||||
@@ -103,7 +106,7 @@ This phase produces documents and upstream issues only; it is the gate for every
|
||||
|
||||
**0.5 Type-unification decision.** Host `memory::traits` types and crate types are wire-compatible twins. Decide: host re-exports crate types (preferred — one source of truth, 30+ consumer sites unchanged via `pub use`), vs. keeping host types + `From` conversions (fallback if serde/API divergence is found in 0.3). Special care: `MemoryTaint` is **security-critical provenance** (fails closed to `ExternalSync`, drives external-effect-tool gating) — its semantics, serde representation, and fail-closed defaults must be proven identical before re-exporting.
|
||||
|
||||
**0.6 Spec doc.** Write `docs/tinycortex-migration-spec.md` version-anchored to exact reviewed SHAs (host + tinycortex), with the ownership lists above, the drift/gap/parity ledgers, and a **deletion ledger** skeleton (every legacy file, with preconditions for deletion) — directly modeled on `docs/tinyagents-migration-spec.md` + `99-deletion-ledger.md`.
|
||||
**0.6 Spec doc.** Write `docs/tinycortex-migration-spec.md` version-anchored to exact reviewed SHAs (host + tinycortex), with the ownership lists above, the drift/gap/parity ledgers, and a **deletion ledger** skeleton (every legacy file, with preconditions for deletion) — coordinated with `docs/tinyagents-migration-plan-2026-07-22.md`; the TinyCortex deletion ledger lives in §2 of the spec.
|
||||
|
||||
---
|
||||
|
||||
@@ -146,7 +149,7 @@ W3–W5 are the risky core (user data on disk). W1–W2 can land quickly; W6–W
|
||||
|
||||
Ordering rule for this migration: **within each workstream, the implementation (adapter + cutover + deletion) lands first; the test work follows as the second slice.** Tests are not skipped — they are sequenced after the implementation is proven to compile and pass the *existing* suites. Concretely:
|
||||
|
||||
1. **Slice A (impl):** adapters + cutover + legacy deletion. Gate: `cargo check` both worlds, existing crate-level integration tests (`tests/memory_roundtrip_e2e.rs`, `memory_tree_sync_deep_raw_coverage_e2e.rs`, etc.) still green — these exercise the public `openhuman::memory::` surface and act as the built-in parity harness for every flip.
|
||||
1. **Slice A (impl):** adapters + cutover + legacy deletion. Gate: `cargo check` both worlds, existing crate-level integration tests (`tests/memory_roundtrip_e2e.rs`, `tests/raw_coverage/memory_tree_sync_deep_raw_coverage_e2e.rs`, etc.) still green — these exercise the public `openhuman::memory::` surface and act as the built-in parity harness for every flip.
|
||||
2. **Slice B (tests):** port and extend the test estate for the new boundary:
|
||||
- **Engine-internal tests move upstream.** The host's `#[path]`-included sibling tests (`ops_tests.rs` pattern) that poke engine internals migrate into `vendor/tinycortex` as crate tests (via tinycortex PRs), following the crate's own `*_tests.rs` sibling convention.
|
||||
- **Host keeps boundary tests:** RPC schema/handler tests, tool gating tests, seam adapter tests (config mapping, embedding signature, taint fail-closed, source-scope enforcement), and the E2E files under `tests/`.
|
||||
|
||||
@@ -0,0 +1,417 @@
|
||||
# TinyCortex Migration — Current-State Audit & Consolidated Plan (2026-07-22)
|
||||
|
||||
**Status:** **DONE** on `feat/tinycortex-migration-2026-07-22`.
|
||||
[CI Full run 29925645209](https://github.com/senamakel/openhuman/actions/runs/29925645209)
|
||||
is green, including the final gate and every Linux, macOS, and Windows desktop
|
||||
shard
|
||||
(companion to `tinyagents-migration-plan-2026-07-22.md`;
|
||||
supersedes the *status/phasing* in `tinycortex-memory-migration-plan.md`,
|
||||
`tinycortex-migration-spec.md`, `tinycortex-api-gap-audit.md`, and
|
||||
`tinycortex-parity-checklist.md` — those docs' ownership splits, gap lists, and
|
||||
parity dimensions remain the reference detail. `tinycortex-drift-ledger.md`
|
||||
stays the row-level ledger but its anchors are stale — see §2).
|
||||
**Scope:** finish moving the remaining generic memory-engine code from
|
||||
`src/openhuman/` into the vendored `tinycortex` crate (`vendor/tinycortex`),
|
||||
delete the in-tree duplicates and staging code, migrate/retire the affected
|
||||
tests, and clean up every dangling doc/code reference.
|
||||
**Method:** fresh four-way audit — (1) the crate surface, (2) the core memory
|
||||
domains, (3) the periphery + the `src/openhuman/tinycortex/` seam, (4) the
|
||||
docs/tests/git history — against the working tree at `main` (`5b8a9f269`,
|
||||
2026-07-22).
|
||||
|
||||
---
|
||||
|
||||
## 1. Executive summary
|
||||
|
||||
**The engine migration is essentially done and the docs don't know it.** All
|
||||
five `docs/tinycortex-*` docs self-describe a "W1–W2 landed; W3 partial" state,
|
||||
but git shows the full W-track landed: W1 seam (#4526), W2 type re-export
|
||||
(#4529), the W3 chunk-store cutover chain (#4532–#4559), W4 queue flip
|
||||
(#4781), W5 vector+scoring (#4790), the W7 long-tail shims (#4785–#4789),
|
||||
**engine-migration completion (#4794)**, **engine test port into the crate
|
||||
(#4820)**, and post-completion feature work (persona/coding-session ingest
|
||||
#4863) on top. The drift ledger's D1–D3 rows are all CLOSED.
|
||||
|
||||
What remains is not "migrate the engine" but five bounded packages (§5):
|
||||
|
||||
1. **Housekeeping** — the pin/tag story is broken (crate has *no git tags*
|
||||
despite the Cargo.toml comment demanding tag-lockstep), the host CI vendor
|
||||
suite omits the `persona` feature the host enables, and there are ~8
|
||||
broken/stale references including a seam module doc naming files that were
|
||||
never created.
|
||||
2. **Delete `memory_store/unified/`** — the explicitly "staging for removal"
|
||||
tier (~5k LOC + ~3.7k test LOC), gated by the G1 `sqlite_conn()` escape
|
||||
hatch.
|
||||
3. **W-SYNC completion** — `memory_sync/` (17.7k LOC, the largest remaining
|
||||
host mass) still runs the host provider pipelines while the crate now ships
|
||||
a full `memory::sync` Composio engine; the D4 ledger rows gate the flip.
|
||||
4. **Embedding-provider dedupe** — host `embeddings/` adapters and
|
||||
`memory_tree/score/embed/` providers now duplicate what
|
||||
`tinyagents::harness::embeddings` ships since tinyagents #58 (this package
|
||||
lands in **tinyagents**, not tinycortex — the crates split the job).
|
||||
5. **Shim retirement + funnel decision** — the compatibility shims left behind
|
||||
by W3–W7, and a policy call on the fact that domains import
|
||||
`tinycortex::memory::*` directly while the seam's re-export facade has
|
||||
**zero** consumers.
|
||||
|
||||
---
|
||||
|
||||
## 2. Ground truth: pins, versions, and discrepancies (fix first)
|
||||
|
||||
| Fact | Value | Where |
|
||||
| --- | --- | --- |
|
||||
| Host requirement | `tinycortex = { version = "0.1", features = ["git-diff", "persona", "sync"] }` | root `Cargo.toml:116` |
|
||||
| Path override | `[patch.crates-io] tinycortex = { path = "vendor/tinycortex" }` | `Cargo.toml:682` |
|
||||
| Vendored crate | `0.1.1`, edition 2021, **MIT** (not GPL — unlike tinyagents), repo `tinyhumansai/tinycortex` | `vendor/tinycortex/Cargo.toml` |
|
||||
| Submodule HEAD | `daaaf6ba` (`heads/main`, dependabot-era, ~50+ commits past every SHA the docs cite) | `git submodule status` |
|
||||
| Tags | **none exist** — `git describe --tags` fails; yet host `Cargo.toml:115` says "Keep the version pin in lockstep with the submodule tag" | submodule |
|
||||
| Docs' claimed pin | gitlink `a8e10f7` (exists in history, long since passed) | drift ledger / spec anchors |
|
||||
| Nested dep | tinycortex requires `tinyagents = { version = "2", path = "vendor/tinyagents" }`, **its own nested submodule pinned at v2.0.0** | `vendor/tinycortex/Cargo.toml:69` |
|
||||
| Host tinyagents req | `2.1` | root `Cargo.toml:107` |
|
||||
| Fork patch | `[patch."https://github.com/senamakel/tinyagents"]` exists *for tinycortex's sake* ("temporarily pins the embedding API port while tinyagents #58 is pending" — #58 is now merged at the tinyagents submodule HEAD) | `Cargo.toml:685-691` |
|
||||
| Host CI vendor suite | `cargo test --manifest-path vendor/tinycortex/Cargo.toml --features git-diff,sync` — **omits `persona`**, which the host enables | `.github/workflows/ci-lite.yml:859` |
|
||||
|
||||
Version skew summary: three different tinyagents expectations (host `2.1`,
|
||||
tinycortex `2`, nested submodule `2.0.0`) resolve only via path patches. The
|
||||
tinyagents plan's WP-0 (bump `vendor/tinyagents` to v2.1.0) should be executed
|
||||
together with bumping tinycortex's *nested* `vendor/tinyagents` so both crates
|
||||
share one trait identity at one version, after which the `senamakel` patch
|
||||
block and its "temporarily pins" comment can likely be retired.
|
||||
|
||||
---
|
||||
|
||||
## 3. What the crate already ships
|
||||
|
||||
`tinycortex 0.1.1` exposes a single `memory` module — the engine "ported from
|
||||
OpenHuman" — with features `tokio`, `git-diff` (git2-backed `memory::diff`),
|
||||
`providers-http` (reqwest embedding/LLM providers, host does **not** enable),
|
||||
`sync` (live Composio + workspace-scan engine), `persona` (doc-06 persona
|
||||
distillation). Modules: `store` (content/vectors/kv/entity_index/safety+PII),
|
||||
`chunks`, `sources` (+readers), `score` (embed/extract/signals), `tree`
|
||||
(+runtime, bucket/document seal, flavoured trees), `queue` (SQLite job queue +
|
||||
tokio runtime), `retrieval` (hybrid search, MMR, rerank), `entities`, `graph`,
|
||||
`goals`, `tool_memory`, `conversations`, `archivist`, `ingest`
|
||||
(canonicalize→chunk→score→tree), `diff`, `sync` (composio
|
||||
clickup/github/linear/notion/slack/gmail), `providers`, `persona`
|
||||
(readers for claude_code/codex/git history), `fsutil`.
|
||||
|
||||
Host-facing extension points (~20 public traits) with the seam's current
|
||||
implementations in parentheses: `Memory`, `MemoryStore`, `EmbeddingBackend` +
|
||||
`Embedder` (`SeamEmbedder`), `ChatProvider` (`SeamChatProvider`), `Summariser`
|
||||
(`HostSummariser`), `SealObserver` + seal-time `Embedder` bridge (`seal.rs`),
|
||||
`TreeJobSink` (`HostTreeJobSink`), `QueueDelegates` (`HostQueueDelegates`),
|
||||
`SourceReader`, `SelfIdentity`, `EntityOccurrenceIndex`, `SnapshotItemSource`,
|
||||
`GoalsGenerator`, `ConversationEventBus`/`ChannelEventHandler`,
|
||||
`PersonaStateStore`, and the sync seams `SyncStateStore` / `SyncEventSink` /
|
||||
`SkillDocSink` / `LocalDocumentSink` / `ExternalSourceReader` / `SyncPipeline`
|
||||
/ `ActionExecutor` / `IncrementalSource` (`sync.rs`, `HostSyncAdapter`).
|
||||
|
||||
The crate consumes `tinyagents::harness::embeddings::{EmbeddingModel,
|
||||
format_embedding_signature}` in `store/vectors/embedding.rs` — that is the
|
||||
whole tinyagents dependency (the W-EMB decision made real by tinyagents #58).
|
||||
|
||||
Crate CI: fmt, clippy `-D warnings`, build+test `--all-features`, doc build
|
||||
with warnings denied, plus a per-feature matrix (`core`, `tokio`, `git-diff`,
|
||||
`sync`, all). Offline by convention; Composio/live tests are wiremock-mocked or
|
||||
env-key-gated.
|
||||
|
||||
---
|
||||
|
||||
## 4. Audit: what remains host-side
|
||||
|
||||
### 4.1 Per-domain state (crate-backed file counts via `use tinycortex::`)
|
||||
|
||||
| Domain | Files / LOC | Crate-backed | State |
|
||||
| --- | --- | --- | --- |
|
||||
| `memory/` | 69 / 18.0k | 3 | Orchestration + policy + RPC layer (by design "no SQLite here"). Predominantly host: `read_rpc/`, `schema*/`, `rpc_models`, agent tools, sync orchestration, tree *policy*. Stays, minus shim cleanup. |
|
||||
| `memory_store/` | 54 / 17.7k | 15 | Sub-stores flipped to crate (chunks, content, vectors, kv, entities, trees, safety/PII per W3/W7 markers). **`unified/` (~5k + ~3.7k tests) is README-marked "Staging for removal"** — documents/query/segments/events/profile await per-kind flips. `factories.rs` (config/provider selection) stays host. |
|
||||
| `memory_tree/` | 47 / 10.3k | 16 | Largest crate-backed surface; engine parts delegate. Host keeps RPC (`tree/rpc.rs` 1,387, `retrieval/rpc.rs`), CLI, bus subscriber, `health/` doctor (G5, host-retained by decision), and `score/embed/` **providers** (~1.9k — see WP-3). |
|
||||
| `memory_queue/` | 7 / 1.4k | 2 | W4 flipped: `worker.rs` delegates claim→dispatch→settle to `tinycortex::memory::queue::run_once` via `HostQueueDelegates`; per-kind handlers deleted. Host keeps worker pool, scheduler tick, `scheduler_gate` throttle, SQLite glue. Done, minus the `memory::jobs` re-export shim. |
|
||||
| `memory_search/` | 8 / 0.8k | 2 | W5 flipped: `scoring.rs`/`vector/mmr.rs` are near-empty shims over crate `WeightProfile`/`retrieval::mmr`. Host keeps the three agent tools. Done. |
|
||||
| `memory_tools/` | 9 / 1.4k | 3 | W7 flipped: types/store/prompt are shims over `tool_memory::{types,store,render}`. Host keeps `capture.rs` (`PostTurnHook`) + put/list tools. Done. |
|
||||
| `memory_archivist/` | 1 / 44 | 1 | Pure W7 shim over `memory::archivist`. Done. |
|
||||
| `memory_conversations/` | 2 / 0.9k | 2 | W7 shim over `memory::conversations`; host keeps `bus.rs` event-bus glue (+ a `tinychannels` legacy-message conversion). Done. |
|
||||
| `memory_diff/` | 7 / 2.0k | 11 uses | W7 flipped: engine crate-owned; host keeps RPC/tools + `DomainEvent::MemoryDiff*` publishing. Done. |
|
||||
| `memory_goals/` | 6 / 0.9k | 4 uses | W7 flipped: types/store crate-owned (byte-identical `MEMORY_GOALS.md`); host keeps agent wiring. Done. |
|
||||
| `memory_sources/` | 16 / 5.0k | 12 uses | Product registry (config-persisted connectors) with crate readers underneath. Host by design. |
|
||||
| `memory_sync/` | **73 / 17.7k** | 5 uses | **The largest remaining mass.** Host Composio provider pipelines (gmail/slack/github/notion/linear/clickup) still run host-side while the crate's `memory::sync` ships the same provider engine. W-SYNC seam (`sync.rs`, 679) is landed; the host flip (W-SYNC.3) has not happened. See WP-2. |
|
||||
| `embeddings/` | 13 / 3.1k | **0** | Bridges **tinyagents** (9 files), not tinycortex. Post-#58, the host per-provider adapters duplicate crate `harness::embeddings` providers. See WP-4. |
|
||||
| `agent_memory/`, `subconscious/` (memory parts), `learning/` (memory parts) | — | 0 | Host product consumers of the memory stack; no crate duplication. Stay. |
|
||||
|
||||
Consumer counts (host memory tiers are deeply embedded and *stay* host, so
|
||||
these are context, not deletion blockers): `memory::` 166 files,
|
||||
`memory_store::` 132, `memory_tree::` 67, `memory_queue::` 18,
|
||||
`memory_tools::` 11, `memory_search::` 3.
|
||||
|
||||
### 4.2 The seam — `src/openhuman/tinycortex/` (11 files, ~3.2k LOC)
|
||||
|
||||
`config.rs`, `embeddings.rs`, `chat.rs`, `summariser.rs`, `ingest.rs`,
|
||||
`seal.rs`, `sync.rs` (679), `queue_driver.rs` (1,007), `persona.rs` (446),
|
||||
`parity.rs` (`#[cfg(test)]` format pins), `mod.rs`. Healthy and intentionally
|
||||
host-owned. Two findings:
|
||||
|
||||
- **The seam is not the funnel.** `mod.rs` re-exports crate types
|
||||
(`MemoryEntry`, `MemoryTaint`, `RecallOpts`, …) as the intended
|
||||
type-unification funnel, but **zero** files import `crate::…::tinycortex::`
|
||||
re-exports — every memory domain imports `tinycortex::memory::*` directly.
|
||||
Either convention works; pick one and record it (WP-5). Direct crate imports
|
||||
are simpler and match how `tinyagents` is consumed; if so, shrink the facade
|
||||
instead of promoting it.
|
||||
- **Stale internal doc:** `mod.rs:28-30` names sibling adapters `sinks.rs` and
|
||||
`bus.rs` that were never created (their roles landed as
|
||||
`ingest.rs`/`seal.rs`/`sync.rs`). `queue_driver.rs:21-26` still says "This
|
||||
brick is additive: nothing is flipped yet" — the flip landed in #4781.
|
||||
|
||||
Coupling is bidirectional by design (seam reaches back into
|
||||
`memory_store`/`memory_tree`/`memory_sync` to do host work inside crate
|
||||
callbacks); not a defect, but `queue_driver.rs`'s 11 host imports will thin as
|
||||
WP-1/WP-3 shims retire.
|
||||
|
||||
### 4.3 Broken/dangling references (resolved in WP-0)
|
||||
|
||||
| Resolved issue | Resolution |
|
||||
| --- | --- |
|
||||
| Missing TinyAgents companion-spec link | Replaced with the existing July 22 migration plan. |
|
||||
| Missing standalone deletion-ledger link | Replaced with §2 of the TinyCortex migration spec. |
|
||||
| Wrong agent-harness documentation path | Pointed at the architecture subdirectory. |
|
||||
| Never-landed golden-workspace generator | Removed after the post-cutover parity decision descoped synthetic pre-cutover fixtures. |
|
||||
| Wrong raw-coverage test path | Corrected to the `tests/raw_coverage/` location. |
|
||||
| Stale line anchors `Cargo.toml:82`, `src/openhuman/mod.rs:130` | `tinycortex-migration-spec.md` §0.4 (actual: `Cargo.toml:116`, `mod.rs:140`) |
|
||||
| Never-created seam sibling names | Replaced with the adapters that actually landed. |
|
||||
| Developer-specific absolute path in crate docs | Replaced with repository-relative paths. |
|
||||
|
||||
Also stale-in-place: all five docs' status headers (pre-#4794), the drift
|
||||
ledger's pin anchors, and the crate migration doc's claim that "TinyCortex
|
||||
will not own memory sync" (superseded by the plan's §8 sync-inclusive
|
||||
amendment and the crate's shipped `sync` module — the *scheduler/credentials/
|
||||
RPC* stay host, the *engine* is crate; restate the boundary in one place).
|
||||
|
||||
### 4.4 Test inventory
|
||||
|
||||
- **In-crate (done):** engine coverage was ported by #4820; the crate runs its
|
||||
own suite offline (wiremock Composio mock; live tests env-gated). W8's CI
|
||||
wiring exists: `ci-lite.yml:859` runs the vendor suite on submodule-pointer
|
||||
changes — **but with `--features git-diff,sync` only; `persona` is untested
|
||||
in host CI while enabled in the host build.**
|
||||
- **Dies with WP-1:** `memory_store/unified/*_tests.rs` (~3.7k LOC:
|
||||
`documents_tests` 1,275, `query_tests` 1,000, `profile_tests` 763,
|
||||
`segments_tests` 393, `events_tests` 287) go with the unified tier; port any
|
||||
still-unique assertions to the per-kind crate backends first.
|
||||
- **Retargets with WP-2:** the memory_sync suites (389 inline tests across 35
|
||||
files + provider `*_tests.rs`: github 655, gmail 301+354, slack 180, linear
|
||||
172, clickup 155, notion 119) — post-flip, provider *engine* behavior is
|
||||
crate-tested; host keeps orchestration/bus/config tests. Integration:
|
||||
`memory_sync_pipeline_e2e.rs` (570), `memory_golden_parity_e2e.rs` (275),
|
||||
`raw_coverage/memory_sync_*` retarget to the crate-backed path.
|
||||
- **Golden parity harness:** comparators 1 & 5 green; 2/3/4 TODO;
|
||||
`frontmatter_parity` Layer-1 asserter pending; fixture generator script
|
||||
missing. Decide finish-vs-descope in WP-0 (post-cutover, parity pins matter
|
||||
mainly for `MemoryTaint`/on-disk stability — keep those, consider dropping
|
||||
the rest).
|
||||
- **Stays host:** `json_rpc_e2e.rs` memory sections, `memory_sources_e2e`
|
||||
(837), `memory_tree_summarizer_e2e` (583), `memory_roundtrip_e2e`,
|
||||
`memory_graph_sync_e2e`, `memory_fast_retrieve_e2e`, `autocomplete_memory_e2e`,
|
||||
`memory_artifacts_e2e`, the 15 `raw_coverage/memory_*` modules, and all
|
||||
subconscious/learning suites.
|
||||
|
||||
---
|
||||
|
||||
## 5. The plan — five work packages
|
||||
|
||||
Same conventions as the tinyagents companion plan: WP-0 first (hygiene,
|
||||
unblocks honest review), then largely independent packages; drift-ledger rows
|
||||
before deletions; failing-before/passing-after regression tests; small
|
||||
validated commits on feature branches per submodule.
|
||||
|
||||
### WP-0 — Version/tag + CI + docs housekeeping (no engine behavior change)
|
||||
|
||||
1. **Establish the tag discipline the Cargo comment promises:** tag
|
||||
`vendor/tinycortex` (e.g. `v0.1.1` at the current release point, or cut
|
||||
`v0.2.0` at HEAD), and record the bump workflow the plan prescribed
|
||||
(`chore(vendor): bump tinycortex …` — zero such commits exist in host
|
||||
history). Alternatively amend the comment to describe reality
|
||||
(main-tracking submodule). Either is fine; the current state (comment
|
||||
demands tags, none exist) is the worst option.
|
||||
2. **Align the nested tinyagents:** when the tinyagents plan's WP-0 bumps
|
||||
`vendor/tinyagents` to v2.1.0, bump `vendor/tinycortex/vendor/tinyagents`
|
||||
in lockstep and update tinycortex's `tinyagents = "2"` if needed; then
|
||||
evaluate retiring the `[patch."…senamakel/tinyagents"]` block and its
|
||||
obsolete "while tinyagents #58 is pending" comment (#58 is merged).
|
||||
3. **Close the persona CI hole:** add `persona` to the vendor-suite features
|
||||
in `.github/workflows/ci-lite.yml:859` (and confirm the crate's own
|
||||
feature-matrix covers `persona`, which today it does not list either).
|
||||
4. **Docs refresh:** update the five docs' status headers to post-#4794/#4820
|
||||
reality; update drift-ledger anchors (pin `daaaf6ba`+tag, note #4794/#4820/
|
||||
#4863); document the `persona` feature + seam (currently invisible in all
|
||||
five docs); restate the sync ownership boundary once (engine=crate,
|
||||
scheduler/credentials/RPC/bus=host) and fix the crate-side migration doc's
|
||||
contradictory "will not own sync" line + its absolute local path.
|
||||
5. **Fix the §4.3 broken references**, including the two code-adjacent ones:
|
||||
correct the seam module inventory and the queue-driver's stale pre-flip
|
||||
description.
|
||||
6. **Golden-parity decision:** finish comparators 2/3/4 + `frontmatter_parity`
|
||||
+ the fixture script, **or** descope to the security-relevant pins
|
||||
(`MemoryTaint` byte-identity, chunk/vector on-disk format) with the
|
||||
decision recorded in the parity checklist.
|
||||
|
||||
**Exit:** pins/tags/CI/docs agree with the tree; `grep` for the phantom paths
|
||||
comes back clean.
|
||||
|
||||
### WP-1 — Retire `memory_store/unified/` (the "staging for removal" tier)
|
||||
|
||||
1. Inventory the five remaining unified facets (documents, query, segments,
|
||||
events, profile — ~5k LOC) against their per-kind crate backends; flip
|
||||
callers facet-by-facet (the W3 sub-store pattern, already proven for
|
||||
chunks/content/vectors/kv/entities/trees/safety).
|
||||
2. The blocker is **G1** (`sqlite_conn()` escape hatch, ~312 refs at audit
|
||||
time): re-audit the current count, then either land the crate-side access
|
||||
the gap audit sketched or explicitly re-scope G1 to the host-retained
|
||||
namespace-document tables and record it.
|
||||
3. Decide the fate of the 10-table host-retained `UnifiedMemory`
|
||||
namespace-document tier: keep host (spec's standing decision) but move it
|
||||
out of `unified/` into a clearly-named home so "staging for removal" can
|
||||
actually be removed.
|
||||
4. Tests: port unique assertions from the ~3.7k LOC of `unified/*_tests.rs`
|
||||
to per-kind backends (crate-side where engine behavior, host-side where
|
||||
glue), then delete with the code.
|
||||
|
||||
**Exit:** `memory_store/unified/` gone or reduced to the renamed host tier;
|
||||
`memory_store/` ≈ 9–10k LOC of glue + host tier; `memory_roundtrip_e2e` +
|
||||
`memory_golden_parity_e2e` + full mock-suite green.
|
||||
|
||||
### WP-2 — W-SYNC.3: flip `memory_sync/` onto the crate sync engine
|
||||
|
||||
The seam (`sync.rs`: `ExternalSourceReader`, `SkillDocSink`,
|
||||
`LocalDocumentSink`, `SyncStateStore`, `SyncEventSink`) is landed; the crate
|
||||
ships the Composio engine incl. per-provider modules. Remaining:
|
||||
|
||||
1. Close the **D4.1–D4.4** drift rows (ledger requires this before the flip).
|
||||
2. Flip provider-by-provider (gmail first — the crate already carries the
|
||||
Gmail 413-page fix #73), keeping host-side: scheduler/periodic trigger,
|
||||
credentials, `config.toml` source registry (`memory_sources/`), event-bus
|
||||
publishing (`ChannelMessageReceived` et al.), RPC/status surface, and
|
||||
redaction/`source_scope` gating.
|
||||
3. Dedupe post-processing: host `providers/*/post_process*` vs crate provider
|
||||
modules — upstream generic normalization, keep host product policy.
|
||||
4. Tests per §4.4: engine behavior moves to crate suites (wiremock Composio),
|
||||
host keeps orchestration/bus/e2e; `memory_sync_pipeline_e2e` retargets.
|
||||
|
||||
**Exit:** `memory_sync/` shrinks from 17.7k toward ~6–8k LOC of
|
||||
orchestration/product glue; D4 rows CLOSED; no duplicated provider parsing.
|
||||
|
||||
### WP-3 — Embedding-provider dedupe (lands in *tinyagents*, coordinated here)
|
||||
|
||||
Two host clusters now duplicate `tinyagents::harness::embeddings` (post-#58:
|
||||
openai/cohere/voyage/ollama/cloud + rate-limit/retry-after):
|
||||
|
||||
1. `src/openhuman/embeddings/` adapters (`openai/voyage/cohere/ollama/
|
||||
cloud_adapter.rs`, `provider_trait.rs`) → construct crate
|
||||
`EmbeddingModel`s directly; host keeps `rpc.rs` (1,479), `schemas.rs`,
|
||||
`catalog.rs`, `factory.rs` (config/BYOK selection, #4056 dimension
|
||||
gating).
|
||||
2. `memory_tree/score/embed/{openai_compat,ollama,cloud,factory}.rs` (~1.9k)
|
||||
→ same crate models via the seam's `SeamEmbedder`/`EmbedderBridge`.
|
||||
|
||||
Because tinycortex consumes the same `EmbeddingModel` trait, this single
|
||||
change serves both crates — it is the concrete payoff of the W-EMB decision.
|
||||
Sequence after WP-0's tinyagents version alignment.
|
||||
|
||||
**Exit:** one embedding-provider implementation per provider across the
|
||||
workspace; `embeddings_rpc_e2e` (822) + `ollama_embeddings_fallback_e2e` (234)
|
||||
green against the crate-backed path.
|
||||
|
||||
### WP-4 — Shim retirement + funnel decision
|
||||
|
||||
1. **Funnel policy:** adopt direct `tinycortex::memory::*` imports as the
|
||||
convention (matches reality and the tinyagents precedent), shrink the seam
|
||||
`mod.rs` facade to the types host code genuinely re-brands
|
||||
(`MemoryTaint` stays pinned by parity tests) — or the reverse; either way,
|
||||
record it in the spec and stop carrying both.
|
||||
2. Retire compatibility shims whose consumers can move: `memory_tree/tools.rs`
|
||||
(points at `memory`), `memory_store/trees` legacy paths, the
|
||||
`memory_queue`→`memory::jobs` re-export, `memory_archivist` (44-line shim —
|
||||
move its 2 call sites and delete the domain), and the W5/W7 near-empty
|
||||
shims once import counts hit zero (`memory_search/scoring.rs`,
|
||||
`vector/mmr.rs`, `memory_tools/{types,store}.rs`).
|
||||
3. Thin `queue_driver.rs` (1,007) as WP-1 removes its `memory_store`
|
||||
reach-backs.
|
||||
|
||||
**Exit:** shim count measurably down (each deletion = one commit with its
|
||||
import-migration); seam ≤ ~2.5k LOC; no module whose entire body is a
|
||||
re-export except deliberate facades recorded in the spec.
|
||||
|
||||
### WP-5 — Exit gate
|
||||
|
||||
- Full suite: `scripts/test-rust-with-mock.sh` (incl. the 15
|
||||
`raw_coverage/memory_*` modules), vendor suite
|
||||
`cargo test --manifest-path vendor/tinycortex/Cargo.toml --features
|
||||
git-diff,sync,persona`, slim disabled build (`--no-default-features
|
||||
--features tokenjuice-treesitter` — memory domains are ungated but the
|
||||
standing repo rule applies), `pnpm rust:check`.
|
||||
- Drift ledger: D4 CLOSED; new rows for every WP-1/WP-2 deletion; the spec's
|
||||
§2 deletion-ledger skeleton filled in with actuals.
|
||||
- Docs: the five tinycortex docs + `src/openhuman/memory*/README.md` files +
|
||||
`gitbooks/developing/architecture.md` reflect the post-flip reality; this
|
||||
plan stamped done.
|
||||
|
||||
### Execution record (2026-07-22)
|
||||
|
||||
| Package | Result |
|
||||
| --- | --- |
|
||||
| WP-0 | TinyAgents unified at 2.1 across OpenHuman, TinyCortex, and TinyFlows; obsolete fork patch removed; TinyCortex CI covers `persona`; docs and phantom references repaired. TinyCortex intentionally tracks reviewed upstream commits rather than nonexistent tags. |
|
||||
| WP-1 | G1 count is zero. The ten-table namespace/document product tier is host-owned, so the complete `unified/` directory was renamed to `namespace_store/` instead of deleted as engine duplication. Its 153 targeted tests pass. |
|
||||
| WP-2 | The live default path was already TinyCortex-backed. D4 is CLOSED; dead Gmail duplicate removed; remaining provider task/profile projections explicitly classified as product policy. Provider tests pass (301). |
|
||||
| WP-3 | Concrete provider transports deduplicated into TinyAgents. The memory tree now uses a thin `ProviderEmbedder`; Ollama's 8k context/batch and missing-model guidance moved upstream before the host client was deleted. TinyAgents embedding tests pass (38); host library check passes. |
|
||||
| WP-4 | Archivist, search scoring/MMR, tool-memory type/store, tree-tool, jobs-alias, and unused seam type facades retired. Direct crate imports are canonical. Seam production code is 2,229 LOC (below the 2.5k exit target). |
|
||||
| WP-5 | Local focused validation is recorded above; the slim `--no-default-features --features tokenjuice-treesitter` build also passes. [CI Full run 29925645209](https://github.com/senamakel/openhuman/actions/runs/29925645209) is green: core quality and full tests, TinyCortex, Tauri, frontend, mock-backend Rust E2E, Playwright, three desktop builds, every launched desktop shard, and the final gate. Two first-attempt Linux jobs ended without uploaded logs; rerunning only failed jobs passed the Linux build, Rust integration suite, and all eight Linux shards. |
|
||||
|
||||
---
|
||||
|
||||
## 6. Risks and standing gotchas
|
||||
|
||||
- **`MemoryTaint` is security-critical.** The type is proven byte-identical
|
||||
host↔crate and fails closed to `ExternalSync`; the parity pins in
|
||||
`tinycortex/parity.rs` are the regression guard — whatever WP-0 decides
|
||||
about the golden harness, **keep these**.
|
||||
- **Redaction and `source_scope` gating stay host.** WP-2 must not let crate
|
||||
sync code become a path around host redaction or the source allowlist —
|
||||
review the `SyncEventSink`/`LocalDocumentSink` implementations for every
|
||||
provider flipped.
|
||||
- **On-disk compat:** chunk/vector formats, `MEMORY_GOALS.md` byte-identity,
|
||||
and the P5 fresh-DB divergence (3 legacy inline-embedding columns in
|
||||
`mem_tree_chunks`) — migrations of user data must be forward-only and
|
||||
tested against a pre-migration workspace fixture.
|
||||
- **MIT vs GPL:** tinycortex is MIT (tinyagents is GPL-3.0) — moving host code
|
||||
*down* into tinycortex relicenses it; fine for our own code, but keep the
|
||||
"no product policy / no keys / nothing openhuman-branded as API" rule, and
|
||||
don't move anything derived from GPL-only sources into the MIT crate.
|
||||
- **Two Cargo worlds + nested submodule:** vendor bumps now touch up to three
|
||||
lockfiles (root, `app/src-tauri`, and tinycortex's own) and two submodule
|
||||
pointers (`vendor/tinycortex`, `vendor/tinycortex/vendor/tinyagents`). CI
|
||||
clones with `submodules: recursive` for the vendor suite — verify after any
|
||||
bump.
|
||||
- **Event-bus surface:** `memory_sync` publishes 7 distinct `DomainEvent`
|
||||
variants; the WP-2 flip must keep publishing them from host callbacks (the
|
||||
crate has no event bus by design).
|
||||
|
||||
---
|
||||
|
||||
## 7. Quick reference — disposition of every audited module
|
||||
|
||||
| Module | Verdict |
|
||||
| --- | --- |
|
||||
| `memory_store/unified/` | DELETE after per-kind flips; host tier re-homed (WP-1) |
|
||||
| `memory_store/` sub-stores, `factories.rs` | DONE (crate-backed) / STAYS (host glue) |
|
||||
| `memory_sync/` provider engines | FLIP to crate `memory::sync` (WP-2) |
|
||||
| `memory_sync/` scheduler, bus, RPC, credentials | STAYS |
|
||||
| `embeddings/` per-provider adapters | REPLACE with `tinyagents::harness::embeddings` models (WP-3) |
|
||||
| `embeddings/` rpc/schemas/catalog/factory | STAYS |
|
||||
| `memory_tree/score/embed/` providers | REPLACE via seam + crate models (WP-3) |
|
||||
| `memory_tree/` engine files | DONE (crate-backed); RPC/CLI/bus/health STAY |
|
||||
| `memory_queue/`, `memory_search/`, `memory_tools/`, `memory_diff/`, `memory_goals/`, `memory_conversations/` | DONE — retire residual shims when import counts reach zero (WP-4) |
|
||||
| `memory_archivist/` | DELETE the 44-line shim after moving 2 call sites (WP-4) |
|
||||
| `memory/` (orchestration/RPC/tools/tree-policy) | STAYS |
|
||||
| `memory_sources/` | STAYS (product registry over crate readers) |
|
||||
| `agent_memory/`, `subconscious/`, `learning/` | STAYS (consumers, no duplication) |
|
||||
| Seam `src/openhuman/tinycortex/` | STAYS, shrinks; fix stale module docs (WP-0), thin `queue_driver.rs` (WP-4) |
|
||||
| Golden-parity harness | FINISH or DESCOPE by decision (WP-0); `MemoryTaint`/format pins KEEP |
|
||||
@@ -1,11 +1,10 @@
|
||||
# TinyCortex Memory Migration — Spec (Phase 0.5 / 0.6)
|
||||
|
||||
**Status:** Phase 0 baseline **+ execution underway** — #59 (native-dep alignment) merged and the dep
|
||||
is active (§0.4); W1 seam + W2 type re-export + W3-chunks partial landed; **drift closure COMPLETE**
|
||||
(**D3** via #59, **D2** via tinycortex#63, **D1** via tinycortex#64; gitlink `a8e10f7`) — so **W4
|
||||
(queue) and W7 (conversations) are unblocked**. Anchors the migration to exact reviewed SHAs and ties
|
||||
together the drift, gap, and parity ledgers. Modeled on `docs/tinyagents-migration-spec.md` + its
|
||||
deletion ledger.
|
||||
**Status:** Post-engine-cutover reference. W1–W8 and the crate-owned engine test
|
||||
port landed in OpenHuman #4794/#4820, with persona/coding-session ingest in
|
||||
#4863. The remaining host consolidation is tracked by
|
||||
[`tinycortex-migration-plan-2026-07-22.md`](tinycortex-migration-plan-2026-07-22.md).
|
||||
This document retains the ownership contract and deletion ledger detail.
|
||||
|
||||
**Companion plan:** [`tinycortex-memory-migration-plan.md`](tinycortex-memory-migration-plan.md)
|
||||
**Ledgers:** [`tinycortex-drift-ledger.md`](tinycortex-drift-ledger.md) ·
|
||||
@@ -19,7 +18,10 @@ deletion ledger.
|
||||
| `tinyhumansai/openhuman` | `7850cf363559bcbb7ba688cbc4fccdb6bd9ce754` | host audit base (`main`, 2026-07-04) |
|
||||
| `tinyhumansai/tinycortex` | `d1a8c7be2babc8fff7a72ed93861f459f3d6fa58` | crate audit base (v0.1.1) |
|
||||
| `tinyhumansai/tinycortex` | `33dda943053e61ef585fc39647cf1854344b6323` | audit base **+ #59** (native-dep alignment, §0.4) merged |
|
||||
| `tinyhumansai/tinycortex` | `a8e10f7dd8ebdb9b0905e1380fefcc6bf5a65207` | **current pinned gitlink** — **+ #63/#64** (D2/D1 drift closure) merged; all drift rows CLOSED |
|
||||
| `tinyhumansai/tinycortex` | `a8e10f7dd8ebdb9b0905e1380fefcc6bf5a65207` | historical cutover gitlink — **+ #63/#64** (D2/D1 drift closure) merged |
|
||||
| `tinyhumansai/openhuman` | `5b8a9f269` | 2026-07-22 consolidation audit base |
|
||||
| `tinyhumansai/tinycortex` | `daaaf6ba5f02635c08deae2b2b2ed7fcc8c06b6a` | 2026-07-22 reviewed gitlink; no upstream tags exist, so the host tracks reviewed main commits |
|
||||
| `tinyhumansai/tinycortex` | `7b4b115` | current migration branch: TinyAgents 2.1 alignment + corrected ownership docs |
|
||||
|
||||
Port line (derived by content, §0.1): **after 2026-06-25, before 2026-06-28** for engine features.
|
||||
|
||||
@@ -72,14 +74,14 @@ ledger. Re-export covers the data types (`MemoryEntry`, `MemoryCategory`, `Memor
|
||||
| **rusqlite alignment** | ✅ **resolved & merged (#59).** Crate was pinned `0.32` (bundled), host pins `=0.40.0` (bundled). Two `links = "sqlite3"` = hard Cargo error. Fixed in #59 (bump to `0.40` + `usize`→`i64`/`try_from` — the same sweep that closed drift **D3**). |
|
||||
| **git2 alignment** | ✅ **resolved & merged (#59).** Crate was pinned `0.19`, host `0.21` (vendored-libgit2). Two `links = "git2"`. Fixed in #59 (bump to `0.21` + API deltas: `Tag::message`, `StringArray::Iter`, `Buf::as_str`). |
|
||||
| Crate compiles with aligned deps | ✅ `cargo check --all-targets` clean; 38 diff/checkpoint tests pass. |
|
||||
| **Host root world compiles with dep active** | ✅ `cargo check --manifest-path Cargo.toml --lib` **exit 0** with `tinycortex = "0.1"` active (`Cargo.toml:82`) + submodule at `33dda94`. **No `multiple packages link to native library` error** — one bundled SQLite + one libgit2 confirmed. **Now landed** (post-#59-merge): the `[dependencies]` line and gitlink are committed on the working branch, not reverted. Re-verified 2026-07-09. |
|
||||
| **Host root world compiles with dep active** | ✅ `cargo check --manifest-path Cargo.toml --lib` **exit 0** with `tinycortex = "0.1"` active (`Cargo.toml:116`) + submodule at `33dda94`. **No `multiple packages link to native library` error** — one bundled SQLite + one libgit2 confirmed. **Now landed** (post-#59-merge): the `[dependencies]` line and gitlink are committed on the working branch, not reverted. Re-verified 2026-07-09. |
|
||||
| Host `app/src-tauri` world | to verify in W1 (separate Cargo world / lockfile). |
|
||||
| `GGML_NATIVE=OFF` macOS ARM | to verify on a macOS runner in W1 (no macOS host here). |
|
||||
|
||||
**Activation landed (post-#59).** The native-dep alignment merged upstream as **#59** and the host
|
||||
gitlink was bumped to `33dda94`, so — per the submodule rule (bump only to a **merged** SHA) — W1's
|
||||
activation is now in place: `[dependencies] tinycortex = "0.1"` is active in the root world
|
||||
(`Cargo.toml:82`), the seam (`src/openhuman/tinycortex/`) is wired (`src/openhuman/mod.rs:130`), and
|
||||
(`Cargo.toml:116`), the seam (`src/openhuman/tinycortex/`) is wired (`src/openhuman/mod.rs:140`), and
|
||||
`cargo check --lib` is **exit 0**. Remaining §0.4 follow-ups: the `app/src-tauri` world and the
|
||||
`GGML_NATIVE=OFF` macOS-runner check still verify in W1 (see rows above).
|
||||
|
||||
@@ -117,7 +119,7 @@ activation is now in place: `[dependencies] tinycortex = "0.1"` is active in the
|
||||
coexist in the shared DB but **do not move**: `memory_docs`, `graph_global`, `graph_namespace`,
|
||||
`episodic_log` (+ `episodic_fts` + triggers), `event_log` (+ `event_fts`, `event_embeddings`,
|
||||
triggers), `conversation_segments`, `segment_embeddings`, `vector_chunks`, `user_profile`.
|
||||
These live in `memory_store/unified/{init,fts5,events,segments,profile}.rs` and remain host — the
|
||||
These live in `memory_store/namespace_store/{init,fts5,events,segments,profile}.rs` and remain host — the
|
||||
crate is the **primitive substrate**, not a drop-in for the whole DB.
|
||||
- **Content-store host surfaces the crate explicitly excludes:** `content::wiki_git`,
|
||||
`content::obsidian`, `content::obsidian_registry`.
|
||||
@@ -128,8 +130,11 @@ activation is now in place: `[dependencies] tinycortex = "0.1"` is active in the
|
||||
`embeddings.rs` (`EmbeddingBackend`/`Embedder`), `chat.rs` (`ChatProvider`/`Summariser`×2/
|
||||
`EntityExtractor`/`GoalsGenerator`), `queue_driver.rs` (`QueueDelegates` + tokio worker loop +
|
||||
Sentry/bus), `config.rs` (`Config`→`MemoryConfig`), `sinks.rs` (`TreeJobSink`/`TreeLeafSink`/
|
||||
`SnapshotItemSource`/`EntityOccurrenceIndex`), `bus.rs` (engine outcomes → `DomainEvent`),
|
||||
`mod.rs` (facade re-exports + boundary doc). All 17 W1 seam traits confirmed present (§0.2).
|
||||
`SnapshotItemSource`/`EntityOccurrenceIndex`), `sync.rs` (sync outcomes → `DomainEvent`),
|
||||
`mod.rs` (adapter namespace + compatibility re-exports + boundary doc). New
|
||||
engine consumers import `tinycortex::memory::*` directly; the seam owns
|
||||
implementations rather than serving as a second type funnel. All 17 W1 seam
|
||||
traits confirmed present (§0.2).
|
||||
|
||||
**Later seam additions (amended 2026-07-09):**
|
||||
|
||||
@@ -144,7 +149,7 @@ Sentry/bus), `config.rs` (`Config`→`MemoryConfig`), `sinks.rs` (`TreeJobSink`/
|
||||
|
||||
---
|
||||
|
||||
## 2. Deletion ledger (skeleton)
|
||||
## 2. Deletion ledger
|
||||
|
||||
Every legacy engine file is deleted only when its module's **drift row is closed**, its **gaps are
|
||||
resolved**, and the **golden-workspace parity harness is green** for its flip. Counts from the host
|
||||
@@ -152,7 +157,7 @@ audit SHA.
|
||||
|
||||
| Legacy module | Files (test files) | Deletes in | Preconditions |
|
||||
| --- | --- | --- | --- |
|
||||
| `memory_store/` | 66 (11) | W3 | drift **D3** closed; gap **G1** (escape hatch) migrated to `with_connection`; parity P3/P5/P11/P12 green; `unified/` tier re-homed as host-retained (kept, not deleted) |
|
||||
| `memory_store/` | 66 (11) | W3/WP-1 | drift **D3** closed; gap **G1** migrated completely (zero `sqlite_conn()` call sites at the 2026-07-22 audit); parity P3/P5/P11/P12 green; namespace tier re-homed to `namespace_store/` as host-retained |
|
||||
| `memory_tree/` | 65 (7) | W5 | gaps **G3** (seal-embed), **G6** (2× Summariser) resolved; `source_scope` allowlist re-verified; parity P7/P11 green; `health/` + `tree_policy.rs` kept host (G5) |
|
||||
| `memory_queue/` | 10 (1) | W4 | drift **D2** closed (predicate upstreamed); job payload_json parity (P4/P9); host worker loop + Sentry/degraded wiring kept host |
|
||||
| `memory_conversations/` | 7 (1) | W7 | drift **D1** closed; `bus.rs` kept host |
|
||||
@@ -168,10 +173,19 @@ audit SHA.
|
||||
| `memory_search/` | 8 (0) | W5 | `vector`/`scoring` → crate `retrieval`/`score`; `tools/` kept host |
|
||||
| `memory/ingest_pipeline.rs` internals | (thin entry points kept) | W6 | `ingest_chat`/`ingest_document_with_scope` signatures unchanged; 11 call sites untouched |
|
||||
|
||||
### 2026-07-22 consolidation actuals
|
||||
|
||||
| Package | Actual result |
|
||||
| --- | --- |
|
||||
| WP-1 namespace tier | `memory_store/unified/` removed and re-homed byte-for-byte as `memory_store/namespace_store/`; G1 re-audit found zero `sqlite_conn()` call sites. The retained ten-table product store explains why total `memory_store` remains 17.7k LOC rather than the plan's speculative 9–10k target. |
|
||||
| WP-2 sync | The default provider `sync()` and `run_connection_sync` already call the TinyCortex engine. Deleted the dead host Gmail sync parser; renamed GitHub/Notion/Linear/ClickUp product projections from `sync.rs` to `normalization.rs`. D4.1-D4.4 are CLOSED. The retained 17.2k LOC is schedulers, bus/RPC, action tools/catalogs, credentials, profiles, post-processing, and product task projections; 6.9k LOC is provider catalogs/tools/normalization/profile/RPC alone. |
|
||||
| WP-3 embeddings | Deleted host OpenAI, Cohere, Voyage, general Ollama, memory-tree cloud, and memory-tree Ollama provider implementations (746 production LOC) plus their obsolete 828-line raw-coverage suite. Provider transport now has one implementation in TinyAgents; OpenHuman retains selection and credential/privacy adapters. |
|
||||
| WP-4 shims | Deleted `memory_archivist`, `memory_search::{scoring,vector}`, `memory_tools::{types,store}`, `memory_tree::tools`, and the `memory::jobs` alias. Removed the unused TinyCortex type facade; direct `tinycortex::memory::*` imports are the convention. The seam is 2,229 pre-test LOC. |
|
||||
|
||||
**Kept host (never deleted):** `memory/{ops,schemas,schema,read_rpc,tools,query,tree_source,
|
||||
ingestion,util}`, `memory/{global,source_scope,chat,sync,preferences,remember,tree_policy,rpc_models,
|
||||
traits(→re-exports)}.rs`, `memory_sync/`'s host-retained shell only (schedulers `periodic.rs`,
|
||||
`bus.rs` subscribers, RPC registration — the engine moves in W-SYNC, plan §8), `memory_store/unified/*` (the namespace-document
|
||||
`bus.rs` subscribers, RPC registration — the engine moves in W-SYNC, plan §8), `memory_store/namespace_store/*` (the namespace-document
|
||||
tier), `memory_store/content/{wiki_git,obsidian,obsidian_registry}`, `memory_tree/health/`, and the
|
||||
new `src/openhuman/tinycortex/` seam.
|
||||
|
||||
|
||||
@@ -1,5 +1,13 @@
|
||||
# TinyCortex Data-Format Parity Checklist (Phase 0.3)
|
||||
|
||||
**Status (2026-07-22):** The engine cutover and crate test port are complete.
|
||||
The proposed pre-migration golden-fixture generator and differential
|
||||
comparators 2–4 are deliberately **descoped**: there is no trustworthy
|
||||
pre-cutover binary/fixture left to generate them from after cutover. Retained
|
||||
gates are the security-critical `MemoryTaint` byte/serde pins, deterministic
|
||||
chunk IDs, vector encoding/signature, content paths, schema composition,
|
||||
idempotent reopen, and the host public-surface E2E suites.
|
||||
|
||||
**Purpose.** Existing user workspaces must open **unchanged** after every cutover flip.
|
||||
This checklist enumerates every on-disk format shared between the host engine and TinyCortex,
|
||||
records the audit result, and specifies the **golden-workspace parity harness** that gates W3, W5, W6.
|
||||
@@ -7,7 +15,8 @@ records the audit result, and specifies the **golden-workspace parity harness**
|
||||
**Hard rule (plan §0.3/§6):** any mismatch is fixed **upstream in tinycortex**, never papered over
|
||||
with a host shim.
|
||||
|
||||
Anchors: host `7850cf363` · tinycortex `d1a8c7be` (v0.1.1).
|
||||
Anchors: historical audit host `7850cf363` · crate `d1a8c7be`; consolidation
|
||||
base host `5b8a9f269` · reviewed crate `daaaf6ba` · migration branch `7b4b115`.
|
||||
|
||||
## Ownership tiers in the shared workspace (key finding)
|
||||
|
||||
@@ -28,10 +37,10 @@ A user workspace's `chunks.db` (and content vault) holds **two tiers**:
|
||||
`event_ai/ad/au` triggers), `conversation_segments`, `segment_embeddings`, `vector_chunks`,
|
||||
`user_profile` **(10 tables + FTS + triggers)**.
|
||||
|
||||
These live in `src/openhuman/memory_store/unified/{init,fts5,events,segments,profile}.rs`.
|
||||
**W3 must keep the host creating/reading these in the same DB the crate now manages** —
|
||||
These live in `src/openhuman/memory_store/namespace_store/{init,fts5,events,segments,profile}.rs`.
|
||||
**The host continues creating/reading these in the same DB the crate now manages** —
|
||||
the crate's `chunks::with_connection` opens the shared handle; host `UnifiedMemory` schema
|
||||
init runs alongside the crate's. Parity requirement: crate schema init and host `unified`
|
||||
init runs alongside the crate's. Parity requirement: crate schema init and host namespace-store
|
||||
init must **compose without collision** on both fresh and existing DBs.
|
||||
|
||||
## Parity results by format dimension
|
||||
@@ -93,17 +102,15 @@ The core mechanism from plan §0.3: **one on-disk workspace, opened by both engi
|
||||
> tables) and the host `UnifiedMemory` tier (10 tables) **coexist without collision** (P3/P5/P11/P12),
|
||||
> and that re-running the flow adds/drops no tables (comparator 5). Still TODO: `vectors`/`store_meta`/
|
||||
> `kv_*` (created by the chunk/embed pipeline, need a widened ingest flow), the seeded golden fixture +
|
||||
> `scripts/gen-golden-workspace.sh`, and comparators **2** (recall/retrieval snapshot), **3** (tree
|
||||
> read), **4** (byte-compare vault) — these require a populated, sealed fixture and the W5 retrieval
|
||||
> surface, so they land alongside the W3/W5 flips.
|
||||
> Comparators **2** (recall/retrieval snapshot), **3** (tree read), and **4**
|
||||
> (byte-compare vault) were never landed before cutover and are now descoped by
|
||||
> the decision above. Comparator 1 (schema composition) and comparator 5
|
||||
> (idempotent reopen) remain active.
|
||||
|
||||
**Fixture.** Check in a small, deterministic `tests/fixtures/golden-workspace/` produced by the
|
||||
*pre-migration* build: a real `chunks.db` + content vault + diff `.git`, seeded via a fixed script
|
||||
(`scripts/gen-golden-workspace.sh`) with: a handful of chat + document + email sources across ≥2
|
||||
namespaces, ingested + scored + sealed to ≥2 tree levels, some entities/edges, a few queue jobs in
|
||||
mixed states, and both tiers populated (episodic/event/segment rows present). Store the **generator
|
||||
script** and a **manifest** (expected chunk-ids, summary paths, recall snapshots) so the fixture is
|
||||
regenerable and reviewable, not an opaque blob.
|
||||
**Fixture decision.** Do not synthesize a fixture and label it “pre-migration”
|
||||
after the fact. Compatibility is guarded by the retained format pins, schema
|
||||
composition/idempotency test, and real host E2E fixtures. A future format
|
||||
migration may add a versioned fixture captured before that migration starts.
|
||||
|
||||
**Comparators (read-only, both engines open the SAME copied workspace):**
|
||||
|
||||
@@ -124,7 +131,8 @@ regenerable and reviewable, not an opaque blob.
|
||||
|
||||
**Wiring.** Runs under `pnpm test:rust` (host-side, counts toward coverage) and as a dedicated
|
||||
`tests/memory_golden_parity_e2e.rs`. The existing crate-level integration tests
|
||||
(`tests/memory_roundtrip_e2e.rs`, `memory_tree_sync_deep_raw_coverage_e2e.rs`) act as the
|
||||
(`tests/memory_roundtrip_e2e.rs`,
|
||||
`tests/raw_coverage/memory_tree_sync_deep_raw_coverage_e2e.rs`) act as the
|
||||
"public-surface still green" guard (plan §5.1); this harness adds the **differential** guard that a
|
||||
flip preserves *existing* data, not just that the API still functions.
|
||||
|
||||
@@ -154,13 +162,13 @@ Engine tests now run at their ownership boundary rather than through OpenHuman r
|
||||
surfaces, agent-tool response post-processing, source registry side effects, and the
|
||||
security-critical `MemoryTaint` seam.
|
||||
- OpenHuman CI now runs `cargo test --manifest-path vendor/tinycortex/Cargo.toml --features
|
||||
git-diff,sync` when the submodule pointer changes and in the reusable full Rust suite. This is
|
||||
git-diff,sync,persona` when the submodule pointer changes and in the reusable full Rust suite. This is
|
||||
required because Cargo does not run dependency test targets while testing `openhuman`.
|
||||
|
||||
The focused local verification commands are:
|
||||
|
||||
```bash
|
||||
cargo test --manifest-path vendor/tinycortex/Cargo.toml --features git-diff,sync
|
||||
cargo test --manifest-path vendor/tinycortex/Cargo.toml --features git-diff,sync,persona
|
||||
cargo test --test raw_coverage_all memory_sync -- --test-threads=1
|
||||
cargo test --test memory_sync_pipeline_e2e --test memory_artifacts_e2e \
|
||||
--test memory_golden_parity_e2e --test memory_roundtrip_e2e --test memory_sources_e2e
|
||||
|
||||
@@ -291,10 +291,10 @@ Every layer is async and non-blocking. The Rust core processes thousands of conc
|
||||
|
||||
## Vendored crate family & recent shifts
|
||||
|
||||
Core subsystems are being re-platformed onto published `tiny*` crates, vendored as git submodules under `vendor/` (`tinyagents`, `tinyflows`, `tinycortex`, `tinychannels`, `tinyjuice`, `tinyplace`) so crate changes can be tested in-tree before publishing. The major ongoing shifts:
|
||||
Core subsystems run on published `tiny*` crates, vendored as git submodules under `vendor/` (`tinyagents`, `tinyflows`, `tinycortex`, `tinychannels`, `tinyjuice`, `tinyplace`) so crate changes can be tested in-tree before publishing. The major ownership boundaries are:
|
||||
|
||||
- **Agent engine on tinyagents** — every agent turn runs through the `tinyagents` crate harness via the seam in `src/openhuman/tinyagents/`; see [Agent Harness](architecture/agent-harness.md).
|
||||
- **Memory on tinycortex** — memory modules (`memory_diff`, `memory_conversations`, `memory_queue`, …) are being deleted or shimmed onto the `tinycortex` crate engine (W7 migration; #4785–#4788).
|
||||
- **Memory on tinycortex** — the generic store/tree/queue/retrieval/sync engine is crate-owned. OpenHuman keeps RPC, tools, scheduling, credentials, security/event policy, worker orchestration, and the host namespace-document store; `src/openhuman/tinycortex/` implements those seams. Concrete embedding transports are shared through `tinyagents::harness::embeddings`.
|
||||
- **Inference on the crate ModelRouter** — host workload-tier model routing and cloud provider slugs now use the crate-native `ModelRouter`/`OpenAiModel` (#4782, #4783).
|
||||
- **Hosted-only brain** — the client-local orchestration graph engine (`src/openhuman/orchestration/graph/`) was retired (#4738); the client is a thin hosted-brain participant (pushers, effect/tool executors, wire allowlist — #4725) surfaced in the `/orchestration` and `/brain/tinyplace-orchestration` routes.
|
||||
|
||||
|
||||
@@ -674,7 +674,7 @@ pub fn expected_error_kind(message: &str) -> Option<ExpectedErrorKind> {
|
||||
/// disk-full condition during its own page bookkeeping (journal/WAL extension)
|
||||
/// before the next syscall surfaces an errno, rusqlite renders the `SQLITE_FULL`
|
||||
/// result code as `"database or disk is full"` (Sentry TAURI-RUST-B6N, hit at
|
||||
/// `memory_store::unified::documents::tx.commit()` during
|
||||
/// `memory_store::namespace_store::documents::tx.commit()` during
|
||||
/// `openhuman.memory_doc_ingest`). `SQLITE_FULL` has only two causes:
|
||||
/// genuine ENOSPC/ERROR_DISK_FULL (always the case in practice — the same
|
||||
/// burst always produces an os-error-28/112 sibling event) or a
|
||||
@@ -683,7 +683,7 @@ pub fn expected_error_kind(message: &str) -> Option<ExpectedErrorKind> {
|
||||
/// rusqlite renders `SQLITE_FULL` in one of two shapes. The **bare** shape is
|
||||
/// the five words `"database or disk is full"` — Our local memory-store write
|
||||
/// call-sites wrap it with `format!("<verb>: {e}")` (e.g. `"commit tx: ..."` /
|
||||
/// `"clear_namespace commit tx: ..."` in `memory_store::unified::documents`),
|
||||
/// `"clear_namespace commit tx: ..."` in `memory_store::namespace_store::documents`),
|
||||
/// so the phrase lands as the **suffix** of the local emit. The **extended**
|
||||
/// shape carries the full error-code envelope, `"database or disk is full:
|
||||
/// Error code 13: Insertion failed because database is full"` (Sentry
|
||||
@@ -1190,9 +1190,8 @@ fn is_loopback_unavailable(lower: &str) -> bool {
|
||||
/// the local Ollama daemon — pure user-state errors the UI already surfaces
|
||||
/// (toast / settings page warning) where Sentry has no remediation path.
|
||||
///
|
||||
/// Several canonical wire shapes are covered, all emitted by
|
||||
/// `openhuman::embeddings::ollama::OllamaEmbedding::embed` and the embed
|
||||
/// service fallback path:
|
||||
/// Several canonical wire shapes are covered, all emitted by the TinyAgents
|
||||
/// Ollama embedder and the host embed service fallback path:
|
||||
///
|
||||
/// - **TAURI-RUST-XS** (~376 events on self-hosted Sentry): user pointed the
|
||||
/// embedder at a chat / vision model id with a temperature suffix (e.g.
|
||||
@@ -4250,7 +4249,7 @@ mod tests {
|
||||
// SQLITE_FULL rendering from rusqlite — engine-level disk-full
|
||||
// detection during page-bookkeeping (journal/WAL extension) that
|
||||
// beats the next syscall to the errno. Production hit at
|
||||
// `memory_store::unified::documents::tx.commit()` during
|
||||
// `memory_store::namespace_store::documents::tx.commit()` during
|
||||
// `openhuman.memory_doc_ingest`, in the same burst that emits
|
||||
// os-error-112 siblings (Sentry TAURI-RUST-B6N).
|
||||
"commit tx: database or disk is full",
|
||||
|
||||
@@ -80,15 +80,17 @@ impl PostTurnHook for ArchivistHook {
|
||||
|
||||
tracing::debug!("[archivist] episodic rows written: session={session_id}");
|
||||
|
||||
// Dual-write into memory_archivist::store (md-backed) so we can
|
||||
// Dual-write into the crate-owned archivist store (md-backed) so we can
|
||||
// validate the FTS5 → md migration before flipping the read side.
|
||||
// Best-effort: a write failure here must not break the turn. The
|
||||
// user turn's assigned seq is captured into `current_seq` so the
|
||||
// segment ops can store it alongside the FTS5 episodic id.
|
||||
let mut current_seq: Option<u32> = None;
|
||||
if let Some(cfg) = self.config.as_ref() {
|
||||
let engine_config =
|
||||
crate::openhuman::tinycortex::memory_config_from(cfg, cfg.workspace_dir.clone());
|
||||
let ts_ms = (timestamp * 1000.0) as i64;
|
||||
let user_turn = crate::openhuman::memory_archivist::ArchivedTurn {
|
||||
let user_turn = tinycortex::memory::archivist::types::ArchivedTurn {
|
||||
session_id: session_id.to_string(),
|
||||
seq: 0, // assigned by record_turn
|
||||
timestamp_ms: ts_ms,
|
||||
@@ -98,7 +100,7 @@ impl PostTurnHook for ArchivistHook {
|
||||
tool_calls_json: None,
|
||||
cost_microdollars: 0,
|
||||
};
|
||||
match crate::openhuman::memory_archivist::store::record_turn(cfg, user_turn) {
|
||||
match tinycortex::memory::archivist::store::record_turn(&engine_config, user_turn) {
|
||||
Ok(stored) => current_seq = Some(stored.seq),
|
||||
Err(e) => {
|
||||
tracing::warn!("[archivist] memory_archivist user dual-write failed: {e}");
|
||||
@@ -113,7 +115,7 @@ impl PostTurnHook for ArchivistHook {
|
||||
} else {
|
||||
Some(serde_json::to_string(&ctx.tool_calls).unwrap_or_default())
|
||||
};
|
||||
let assistant_turn = crate::openhuman::memory_archivist::ArchivedTurn {
|
||||
let assistant_turn = tinycortex::memory::archivist::types::ArchivedTurn {
|
||||
session_id: session_id.to_string(),
|
||||
seq: 0,
|
||||
timestamp_ms: ts_ms + 1,
|
||||
@@ -124,7 +126,7 @@ impl PostTurnHook for ArchivistHook {
|
||||
cost_microdollars: 0,
|
||||
};
|
||||
if let Err(e) =
|
||||
crate::openhuman::memory_archivist::store::record_turn(cfg, assistant_turn)
|
||||
tinycortex::memory::archivist::store::record_turn(&engine_config, assistant_turn)
|
||||
{
|
||||
tracing::warn!("[archivist] memory_archivist assistant dual-write failed: {e}");
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ use std::sync::Arc;
|
||||
|
||||
impl ArchivistHook {
|
||||
/// Read every entry recorded for `session_id`, preferring the
|
||||
/// md-backed `memory_archivist::store` when `self.config` is set and
|
||||
/// crate-owned md-backed archivist store when `self.config` is set and
|
||||
/// falling back to the legacy FTS5 episodic table otherwise.
|
||||
///
|
||||
/// Returns `EpisodicEntry` so the existing call sites (segment
|
||||
@@ -23,7 +23,10 @@ impl ArchivistHook {
|
||||
session_id: &str,
|
||||
) -> Vec<EpisodicEntry> {
|
||||
if let Some(cfg) = self.config.as_ref() {
|
||||
match crate::openhuman::memory_archivist::store::session_entries(cfg, session_id) {
|
||||
let engine_config =
|
||||
crate::openhuman::tinycortex::memory_config_from(cfg, cfg.workspace_dir.clone());
|
||||
match tinycortex::memory::archivist::store::session_entries(&engine_config, session_id)
|
||||
{
|
||||
Ok(turns) => {
|
||||
return turns
|
||||
.into_iter()
|
||||
|
||||
@@ -1,106 +1,31 @@
|
||||
# embeddings
|
||||
|
||||
Embedding providers for the OpenHuman memory system. Converts text into numerical vectors for semantic search, abstracting over multiple backends behind a single `EmbeddingProvider` trait. Owns provider construction (factory + slug catalog), per-endpoint client-side rate limiting, 429/503 retry-with-backoff logic, and the JSON-RPC surface that the Settings UI uses to pick a provider, store API keys, test connectivity, and embed text. The default provider is the OpenHuman backend ("managed" / "cloud", Voyage-backed); other paths include direct Voyage, OpenAI, Cohere, local Ollama, any OpenAI-compatible `custom:<url>` endpoint, and a no-op (keyword-only) fallback.
|
||||
Host policy and RPC surface for vector embeddings. Concrete OpenAI-compatible,
|
||||
Cohere, Voyage, Ollama, cloud-transport, retry, and rate-limit implementations
|
||||
live in `tinyagents::harness::embeddings`; this domain selects and adapts those
|
||||
models for OpenHuman.
|
||||
|
||||
## Responsibilities
|
||||
## Host-owned responsibilities
|
||||
|
||||
- Define `EmbeddingProvider` (async trait) and the canonical embedding-space signature format (`provider=…;model=…;dims=…`) — the single source of truth so config-derived and live-provider signatures stay byte-identical (#1574).
|
||||
- Implement each provider: cloud/managed, Voyage, OpenAI(-compatible), Cohere, Ollama, noop.
|
||||
- Construct providers from a slug + model + dims (with or without credentials) via the factory.
|
||||
- Maintain the static provider/model catalog (slugs, labels, API-key/endpoint requirements, dimension presets) consumed by the frontend picker.
|
||||
- Throttle outbound cloud-embedding HTTP requests with a process-global, per-endpoint token bucket (loopback exempt).
|
||||
- Parse `Retry-After` and apply 429/503 exponential backoff in HTTP-based providers.
|
||||
- Expose RPC handlers for settings, API-key management, embed, and connection testing; trigger memory wipe / re-embed backfill when the embedding signature changes.
|
||||
- `factory.rs`: provider slug/model/dimension selection and construction of
|
||||
TinyAgents models.
|
||||
- `provider_trait.rs`: compatibility contract used by existing OpenHuman
|
||||
consumers plus `TinyAgentsEmbeddingProvider`, the one trait adapter.
|
||||
- `cloud_adapter.rs`: OpenHuman session-token resolution, egress disclosure,
|
||||
and local-only privacy enforcement around TinyAgents `CloudEmbeddingModel`.
|
||||
- `catalog.rs`, `rpc.rs`, `schemas.rs`: Settings catalog, credentials, JSON-RPC,
|
||||
connection tests, and re-embed/wipe policy.
|
||||
- `noop.rs`: unit-struct compatibility adapter over TinyAgents' no-op model.
|
||||
|
||||
## Key files
|
||||
Provider HTTP behavior and its tests belong in `vendor/tinyagents`. Do not add
|
||||
new per-provider clients here.
|
||||
|
||||
| File | Role |
|
||||
| --- | --- |
|
||||
| `src/openhuman/embeddings/mod.rs` | Module docstring + module decls + embedding provider `pub use` re-exports; exposes `all_embeddings_controller_schemas` / `all_embeddings_registered_controllers`. |
|
||||
| `src/openhuman/embeddings/provider_trait.rs` | `EmbeddingProvider` trait (`name`/`model_id`/`dimensions`/`signature`/`embed`/`embed_one`) and `format_embedding_signature`. |
|
||||
| `src/openhuman/embeddings/factory.rs` | `create_embedding_provider`, `create_embedding_provider_with_credentials`, `default_embedding_provider` (cloud), `default_local_embedding_provider` (Ollama). Maps provider slugs to concrete impls; unknown slugs error. |
|
||||
| `src/openhuman/embeddings/catalog.rs` | Static catalog of `EmbeddingProviderEntry` + `EmbeddingModelPreset`; slug constants; `all_providers` / `find_provider` / `find_model` / `default_model_for`. |
|
||||
| `src/openhuman/embeddings/cloud.rs` | `OpenHumanCloudEmbedding` — default provider; resolves session JWT + API URL per call, delegates HTTP to `OpenAiEmbedding` against `<api>/openai/v1`. `DEFAULT_CLOUD_EMBEDDING_MODEL` = `embedding-v1`, dims 1024. |
|
||||
| `src/openhuman/embeddings/openai.rs` | `OpenAiEmbedding` — OpenAI-compatible `POST /v1/embeddings`; URL inference, 429/503 retry, rate-limit gating, dimension/count validation. Used directly by cloud, voyage, and custom paths. |
|
||||
| `src/openhuman/embeddings/voyage.rs` | `VoyageEmbedding` — thin wrapper delegating to `OpenAiEmbedding` against `api.voyageai.com`. |
|
||||
| `src/openhuman/embeddings/cohere.rs` | `CohereEmbedding` — Cohere-native `POST /v2/embed` wire format (`texts`, `embedding_types`, nested `embeddings.float`) with its own 429/503 retry loop. |
|
||||
| `src/openhuman/embeddings/ollama.rs` | `OllamaEmbedding` — local Ollama `POST /api/embed`; base-url/model normalization, blank-input zero-vector preservation, NaN-encoding 500 per-text recovery (TAURI-RUST-AZ). Defaults `bge-m3`/1024. |
|
||||
| `src/openhuman/embeddings/noop.rs` | `NoopEmbedding` — returns empty vectors; keyword-only fallback (`name`/`model_id` = "none", dims 0). |
|
||||
| `src/openhuman/embeddings/rate_limit.rs` | Process-global, per-endpoint token-bucket request limiter; `set_embedding_rate_limit`, `embedding_rate_limit`, `acquire_embedding_slot`. Default 60/min; loopback exempt; `0` disables. |
|
||||
| `src/openhuman/embeddings/retry_after.rs` | `parse_retry_after_ms`, `backoff_ms_for_attempt`, and the `MAX_429_RETRIES` / backoff constants. |
|
||||
| `src/openhuman/embeddings/rpc.rs` | RPC business logic: `get_settings`, `update_settings`, `set_api_key`, `clear_api_key`, `embed`, `test_connection`, plus `provider_from_config` and `resolve_api_key`. |
|
||||
| `src/openhuman/embeddings/schemas.rs` | Controller schemas + `handle_*` param-deserializing wrappers delegating to `rpc.rs`. |
|
||||
| `src/openhuman/embeddings/mod_tests.rs` | Module-level tests (via `#[path]`). |
|
||||
| `src/openhuman/embeddings/openai_tests.rs` / `ollama_tests.rs` | Sibling test suites for the OpenAI and Ollama providers (via `#[path]`). |
|
||||
The canonical embedding-space signature is
|
||||
`provider={name};model={model};dims={dims}`. Configuration-derived and live
|
||||
provider signatures must remain byte-identical or stored vectors split into
|
||||
incompatible spaces.
|
||||
|
||||
## Public surface
|
||||
|
||||
From `mod.rs` re-exports:
|
||||
|
||||
- Trait + helper: `EmbeddingProvider`, `format_embedding_signature`.
|
||||
- Providers: `OpenHumanCloudEmbedding`, `OpenAiEmbedding`, `OllamaEmbedding`, `NoopEmbedding` (Voyage/Cohere are public via their submodules through the factory).
|
||||
- Factory fns: `create_embedding_provider`, `default_embedding_provider`, `default_local_embedding_provider`.
|
||||
- Config-driven builder: `provider_from_config` (re-exported from `rpc`) — same construction `embed` uses, for callers like `codegraph` that need a provider without a JSON-RPC round-trip.
|
||||
- Defaults/consts: `DEFAULT_CLOUD_EMBEDDING_MODEL`, `DEFAULT_CLOUD_EMBEDDING_DIMENSIONS`, `DEFAULT_OLLAMA_MODEL`, `DEFAULT_OLLAMA_DIMENSIONS`.
|
||||
- RPC registry: `all_embeddings_controller_schemas`, `all_embeddings_registered_controllers`.
|
||||
|
||||
## RPC / controllers
|
||||
|
||||
Namespace `embeddings` (6 controllers, registered through `src/core/all.rs`):
|
||||
|
||||
| Method | Description |
|
||||
| --- | --- |
|
||||
| `embeddings.get_settings` | Current provider/model/dims/rate-limit + full provider catalog with `has_api_key` flags and `vector_search_enabled`. |
|
||||
| `embeddings.update_settings` | Update provider/model/dimensions/custom_endpoint/rate_limit. Requires `confirm_wipe=true` when **dimensions** change (returns `EMBEDDINGS_DIMENSION_CHANGE_REQUIRES_WIPE` otherwise); wipes memory on dim change and queues a re-embed backfill on any signature change. Also syncs `config.embeddings_provider` workload routing. |
|
||||
| `embeddings.set_api_key` | Store an API key under credential provider `embeddings:<slug>`. |
|
||||
| `embeddings.clear_api_key` | Remove the stored key for `embeddings:<slug>`. |
|
||||
| `embeddings.embed` | Embed input texts using the configured provider; returns vectors, dimensions, count. |
|
||||
| `embeddings.test_connection` | Run a single test embed against the configured or specified provider/model/dims. |
|
||||
|
||||
All handlers return `RpcOutcome<serde_json::Value>` via `into_cli_compatible_json`.
|
||||
|
||||
## Agent tools
|
||||
|
||||
None. This module owns no `tools.rs`; agent-facing memory tools live in `memory/tools/` and consume embeddings indirectly through the memory store.
|
||||
|
||||
## Events
|
||||
|
||||
None. No `bus.rs` / `EventHandler` impls. Cross-module side effects in `update_settings` are direct calls (`memory::read_rpc::wipe_all_rpc`, `memory_queue::ensure_reembed_backfill`), not domain events.
|
||||
|
||||
## Persistence
|
||||
|
||||
No `store.rs`. Settings live in the global `Config` (`config.memory.embedding_provider` / `embedding_model` / `embedding_dimensions` / `embedding_rate_limit_per_min`, plus `config.embeddings_provider`). API keys are persisted via the credentials domain (`AuthService`) under provider key `embeddings:<slug>`. Vectors themselves are stored by `memory_store::vectors`.
|
||||
|
||||
## Dependencies
|
||||
|
||||
- `crate::openhuman::config` — `Config`, `config::ops::load_config_with_timeout`, `build_runtime_proxy_client` (proxy-aware reqwest), config save.
|
||||
- `crate::openhuman::credentials` — `AuthService`, `APP_SESSION_PROVIDER` for resolving the session JWT (cloud) and storing/loading provider API keys.
|
||||
- `crate::openhuman::memory_store::vectors` — canonical vector store owner used by memory storage and retrieval.
|
||||
- `crate::openhuman::memory::read_rpc` — `wipe_all_rpc` on a dimension change.
|
||||
- `crate::openhuman::memory_queue` — `ensure_reembed_backfill` on a signature change.
|
||||
- `crate::openhuman::inference::local` — `ollama_base_url()` when constructing the Ollama provider.
|
||||
- `crate::api::config` — `effective_api_url` for the cloud backend base URL.
|
||||
- `crate::core::all` — `ControllerFuture`, `RegisteredController` for the RPC registry.
|
||||
- `crate::core::{ControllerSchema, FieldSchema, TypeSchema}` — schema definitions.
|
||||
- `crate::core::observability` — `report_error_or_expected` so transient upstream HTTP failures demote to warning breadcrumbs instead of Sentry errors.
|
||||
- `crate::rpc::RpcOutcome` — handler return contract.
|
||||
|
||||
## Used by
|
||||
|
||||
- `memory_store/*` — factories, vectors store, chunks store, unified store, retrieval, client (primary consumer; constructs providers and uses signatures to partition the embedding space).
|
||||
- `memory/*` — ingestion queue, preferences, tools (recall/store/forget), `read_rpc`.
|
||||
- `memory_tree/score/embed/cloud.rs` — scoring path.
|
||||
- `codegraph/*` — index/search/tools obtain a provider via `provider_from_config` for `signature()` + direct embedding.
|
||||
- `voice/factory.rs`, `screen_intelligence`, `channels`, `agent` tools — indirect consumers.
|
||||
- `config/schema/load.rs` — wires `memory.embedding_rate_limit_per_min` into `rate_limit::set_embedding_rate_limit`.
|
||||
- `core/all.rs` — registers the RPC controllers; `core/observability.rs` — classifies the canonical embedding error strings.
|
||||
|
||||
## Notes / gotchas
|
||||
|
||||
- **Signature drift is the core hazard.** `format_embedding_signature` is the single source of truth; both config-derived and live-provider signatures route through it. A mismatch silently splits one embedding space into two (#1574). `update_settings` keys memory wipe off **dimension** change (vectors stay comparable across provider/model swaps at the same dimensionality) but queues a re-embed backfill on **any** signature change.
|
||||
- **Rate limiting is account-wide and process-global**, keyed by resolved base URL, with capacity = 1 token (no burst) to stay strictly under a hard per-minute cap. Ephemeral provider instances share one budget per endpoint. Loopback hosts are exempt; `limit==0` disables. The acquire chokepoint sits **inside** the retry loop so retried attempts consume tokens.
|
||||
- **Cloud provider resolves auth lazily per call** — it can be constructed before login; the first `embed()` errors clearly if unauthenticated. It honors `OPENHUMAN_WORKSPACE` for the auth-profiles directory so non-default workspaces (tests/multi-instance) don't silently lose their session.
|
||||
- **Ollama NaN recovery (TAURI-RUST-AZ):** a single bad input can poison a whole batch with a 500 `unsupported value: NaN`; the provider re-issues per-text and substitutes empty embeddings for the offending entries. Blank/whitespace inputs are preserved as zero-vectors so result length always matches input length. `local-*` model IDs are rejected (they're virtual routing aliases).
|
||||
- **Custom endpoints** are encoded in the provider slug as `custom:<url>`; `resolve_api_key` and the RPC handlers normalize this back to the `custom` credential slug.
|
||||
- **Voyage and cloud reuse `OpenAiEmbedding`** for HTTP plumbing (Voyage's API is an OpenAI superset). **Cohere** speaks a distinct wire format and has its own retry loop.
|
||||
- Error strings from OpenAI/Cohere providers intentionally preserve the `(429 ` / status substring so `core::observability`'s `TransientUpstreamHttp` classifier downgrades them.
|
||||
The memory tree uses the same factory through
|
||||
`memory_tree/score/embed::ProviderEmbedder`; its fixed on-disk dimension remains
|
||||
1024. Ollama requests use TinyAgents'
|
||||
`RECOMMENDED_OLLAMA_CONTEXT_TOKENS` for both `num_ctx` and `num_batch`.
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
//! Compatibility wrapper for tinyagents' Cohere model.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use tinyagents::harness::embeddings::{CohereEmbeddingModel, EmbeddingModel};
|
||||
|
||||
use super::EmbeddingProvider;
|
||||
|
||||
pub struct CohereEmbedding {
|
||||
inner: CohereEmbeddingModel,
|
||||
}
|
||||
|
||||
impl CohereEmbedding {
|
||||
pub fn new(api_key: &str, model: &str, dimensions: usize) -> Self {
|
||||
Self {
|
||||
inner: CohereEmbeddingModel::new(api_key)
|
||||
.with_model(model)
|
||||
.with_dimensions(dimensions),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_base_url(mut self, base_url: impl Into<String>) -> Self {
|
||||
self.inner = self.inner.with_base_url(base_url);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl EmbeddingProvider for CohereEmbedding {
|
||||
fn name(&self) -> &str {
|
||||
self.inner.name()
|
||||
}
|
||||
fn model_id(&self) -> &str {
|
||||
self.inner.model_id()
|
||||
}
|
||||
fn dimensions(&self) -> usize {
|
||||
self.inner.dimensions()
|
||||
}
|
||||
fn signature(&self) -> String {
|
||||
self.inner.signature()
|
||||
}
|
||||
async fn embed(&self, texts: &[&str]) -> anyhow::Result<Vec<Vec<f32>>> {
|
||||
let texts = texts
|
||||
.iter()
|
||||
.map(|text| (*text).to_owned())
|
||||
.collect::<Vec<_>>();
|
||||
self.inner
|
||||
.embed(&texts)
|
||||
.await
|
||||
.map_err(|error| anyhow::anyhow!(error))
|
||||
}
|
||||
}
|
||||
@@ -15,14 +15,8 @@
|
||||
pub mod catalog;
|
||||
#[path = "cloud_adapter.rs"]
|
||||
pub mod cloud;
|
||||
#[path = "cohere_adapter.rs"]
|
||||
pub mod cohere;
|
||||
mod factory;
|
||||
pub mod noop;
|
||||
#[path = "ollama_adapter.rs"]
|
||||
pub mod ollama;
|
||||
#[path = "openai_adapter.rs"]
|
||||
pub mod openai;
|
||||
mod provider_trait;
|
||||
pub mod rate_limit {
|
||||
pub use tinyagents::harness::embeddings::{
|
||||
@@ -42,8 +36,6 @@ pub mod retry_after {
|
||||
}
|
||||
mod rpc;
|
||||
mod schemas;
|
||||
#[path = "voyage_adapter.rs"]
|
||||
pub mod voyage;
|
||||
|
||||
pub use catalog::non_embedding_model_reason;
|
||||
pub use cloud::{
|
||||
@@ -60,8 +52,6 @@ pub(crate) use factory::model_supports_dimensions;
|
||||
// #002 FR-015: the memory-tree OpenAI-compat embedder reuses the same key
|
||||
// resolution the embeddings RPC uses, so there is one source of truth.
|
||||
pub use noop::NoopEmbedding;
|
||||
pub use ollama::OllamaEmbedding;
|
||||
pub use openai::OpenAiEmbedding;
|
||||
pub use provider_trait::{
|
||||
format_embedding_signature, EmbeddingProvider, TinyAgentsEmbeddingProvider,
|
||||
};
|
||||
@@ -71,7 +61,9 @@ pub use schemas::{
|
||||
all_controller_schemas as all_embeddings_controller_schemas,
|
||||
all_registered_controllers as all_embeddings_registered_controllers,
|
||||
};
|
||||
pub use tinyagents::harness::embeddings::{DEFAULT_OLLAMA_DIMENSIONS, DEFAULT_OLLAMA_MODEL};
|
||||
pub use tinyagents::harness::embeddings::{
|
||||
DEFAULT_OLLAMA_DIMENSIONS, DEFAULT_OLLAMA_MODEL, DEFAULT_OLLAMA_URL,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "mod_tests.rs"]
|
||||
|
||||
@@ -21,15 +21,13 @@ fn noop_name_and_dims() {
|
||||
async fn noop_embed_returns_empty() {
|
||||
let p = NoopEmbedding;
|
||||
let result = p.embed(&["hello"]).await.unwrap();
|
||||
assert!(result.is_empty());
|
||||
assert_eq!(result, vec![Vec::<f32>::new()]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn noop_embed_one_returns_error() {
|
||||
// embed returns empty vec → pop() returns None → error from default impl
|
||||
async fn noop_embed_one_returns_empty_vector() {
|
||||
let p = NoopEmbedding;
|
||||
let err = p.embed_one("hello").await.unwrap_err();
|
||||
assert!(err.to_string().contains("Empty embedding result"));
|
||||
assert!(p.embed_one("hello").await.unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! No-op embedding provider for keyword-only search fallback.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use tinyagents::harness::embeddings::{EmbeddingModel, NoopEmbeddingModel};
|
||||
|
||||
use super::EmbeddingProvider;
|
||||
|
||||
@@ -22,7 +23,14 @@ impl EmbeddingProvider for NoopEmbedding {
|
||||
0
|
||||
}
|
||||
|
||||
async fn embed(&self, _texts: &[&str]) -> anyhow::Result<Vec<Vec<f32>>> {
|
||||
Ok(Vec::new())
|
||||
async fn embed(&self, texts: &[&str]) -> anyhow::Result<Vec<Vec<f32>>> {
|
||||
let texts = texts
|
||||
.iter()
|
||||
.map(|text| (*text).to_owned())
|
||||
.collect::<Vec<_>>();
|
||||
NoopEmbeddingModel
|
||||
.embed(&texts)
|
||||
.await
|
||||
.map_err(|error| anyhow::anyhow!(error))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
//! Compatibility wrapper for tinyagents' Ollama model.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use tinyagents::harness::embeddings::{EmbeddingModel, OllamaEmbeddingModel};
|
||||
|
||||
use super::EmbeddingProvider;
|
||||
|
||||
pub use tinyagents::harness::embeddings::{
|
||||
DEFAULT_OLLAMA_DIMENSIONS, DEFAULT_OLLAMA_MODEL, DEFAULT_OLLAMA_URL,
|
||||
};
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct OllamaEmbedding {
|
||||
inner: OllamaEmbeddingModel,
|
||||
}
|
||||
|
||||
impl OllamaEmbedding {
|
||||
pub fn try_new(base_url: &str, model: &str, dimensions: usize) -> anyhow::Result<Self> {
|
||||
Ok(Self {
|
||||
inner: OllamaEmbeddingModel::try_new(base_url, model, dimensions)?,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn new(base_url: &str, model: &str, dimensions: usize) -> Self {
|
||||
Self::try_new(base_url, model, dimensions).expect("invalid Ollama embedding configuration")
|
||||
}
|
||||
|
||||
pub fn base_url(&self) -> &str {
|
||||
self.inner.base_url()
|
||||
}
|
||||
|
||||
pub fn model(&self) -> &str {
|
||||
self.inner.model()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl EmbeddingProvider for OllamaEmbedding {
|
||||
fn name(&self) -> &str {
|
||||
self.inner.name()
|
||||
}
|
||||
fn model_id(&self) -> &str {
|
||||
self.inner.model_id()
|
||||
}
|
||||
fn dimensions(&self) -> usize {
|
||||
self.inner.dimensions()
|
||||
}
|
||||
fn signature(&self) -> String {
|
||||
self.inner.signature()
|
||||
}
|
||||
async fn embed(&self, texts: &[&str]) -> anyhow::Result<Vec<Vec<f32>>> {
|
||||
let texts = texts
|
||||
.iter()
|
||||
.map(|text| (*text).to_owned())
|
||||
.collect::<Vec<_>>();
|
||||
self.inner
|
||||
.embed(&texts)
|
||||
.await
|
||||
.map_err(|error| anyhow::anyhow!(error))
|
||||
}
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
//! Compatibility wrapper for tinyagents' OpenAI-compatible model.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use tinyagents::harness::embeddings::{EmbeddingModel, OpenAiEmbeddingModel};
|
||||
|
||||
use super::EmbeddingProvider;
|
||||
|
||||
pub struct OpenAiEmbedding {
|
||||
inner: OpenAiEmbeddingModel,
|
||||
}
|
||||
|
||||
impl OpenAiEmbedding {
|
||||
pub fn new(base_url: &str, api_key: &str, model: &str, dimensions: usize) -> Self {
|
||||
Self {
|
||||
inner: OpenAiEmbeddingModel::new(api_key)
|
||||
.with_base_url(base_url)
|
||||
.with_model(model)
|
||||
.with_dimensions(dimensions)
|
||||
.with_send_dimensions(false)
|
||||
.with_required_api_key(false),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_send_dimensions(mut self, send: bool) -> Self {
|
||||
self.inner = self.inner.with_send_dimensions(send);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_required_api_key(mut self, required: bool) -> Self {
|
||||
self.inner = self.inner.with_required_api_key(required);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn base_url(&self) -> &str {
|
||||
self.inner.base_url()
|
||||
}
|
||||
|
||||
pub fn model(&self) -> &str {
|
||||
self.inner.model()
|
||||
}
|
||||
|
||||
pub fn embeddings_url(&self) -> String {
|
||||
self.inner.embeddings_url()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl EmbeddingProvider for OpenAiEmbedding {
|
||||
fn name(&self) -> &str {
|
||||
self.inner.name()
|
||||
}
|
||||
fn model_id(&self) -> &str {
|
||||
self.inner.model_id()
|
||||
}
|
||||
fn dimensions(&self) -> usize {
|
||||
self.inner.dimensions()
|
||||
}
|
||||
fn signature(&self) -> String {
|
||||
self.inner.signature()
|
||||
}
|
||||
async fn embed(&self, texts: &[&str]) -> anyhow::Result<Vec<Vec<f32>>> {
|
||||
let texts = texts
|
||||
.iter()
|
||||
.map(|text| (*text).to_owned())
|
||||
.collect::<Vec<_>>();
|
||||
self.inner
|
||||
.embed(&texts)
|
||||
.await
|
||||
.map_err(|error| anyhow::anyhow!(error))
|
||||
}
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
//! Compatibility wrapper for tinyagents' Voyage model.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use tinyagents::harness::embeddings::{
|
||||
EmbeddingModel, VoyageEmbeddingModel, VOYAGE_API_BASE, VOYAGE_DEFAULT_DIMENSIONS,
|
||||
VOYAGE_DEFAULT_MODEL,
|
||||
};
|
||||
|
||||
use super::EmbeddingProvider;
|
||||
|
||||
pub struct VoyageEmbedding {
|
||||
inner: VoyageEmbeddingModel,
|
||||
}
|
||||
|
||||
impl VoyageEmbedding {
|
||||
pub fn new(api_key: &str, model: &str, dimensions: usize) -> Self {
|
||||
Self::new_with_base_url(api_key, model, dimensions, VOYAGE_API_BASE)
|
||||
}
|
||||
|
||||
pub fn new_with_base_url(
|
||||
api_key: &str,
|
||||
model: &str,
|
||||
dimensions: usize,
|
||||
base_url: &str,
|
||||
) -> Self {
|
||||
Self {
|
||||
inner: VoyageEmbeddingModel::with_options(
|
||||
api_key,
|
||||
if model.is_empty() {
|
||||
VOYAGE_DEFAULT_MODEL
|
||||
} else {
|
||||
model
|
||||
},
|
||||
if dimensions == 0 {
|
||||
VOYAGE_DEFAULT_DIMENSIONS
|
||||
} else {
|
||||
dimensions
|
||||
},
|
||||
base_url,
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl EmbeddingProvider for VoyageEmbedding {
|
||||
fn name(&self) -> &str {
|
||||
self.inner.name()
|
||||
}
|
||||
fn model_id(&self) -> &str {
|
||||
self.inner.model_id()
|
||||
}
|
||||
fn dimensions(&self) -> usize {
|
||||
self.inner.dimensions()
|
||||
}
|
||||
fn signature(&self) -> String {
|
||||
self.inner.signature()
|
||||
}
|
||||
async fn embed(&self, texts: &[&str]) -> anyhow::Result<Vec<Vec<f32>>> {
|
||||
let texts = texts
|
||||
.iter()
|
||||
.map(|text| (*text).to_owned())
|
||||
.collect::<Vec<_>>();
|
||||
self.inner
|
||||
.embed(&texts)
|
||||
.await
|
||||
.map_err(|error| anyhow::anyhow!(error))
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
//!
|
||||
//! The memory tree's embedder (`bge-m3`) is requested with
|
||||
//! `num_ctx = 8192` (see
|
||||
//! [`crate::openhuman::memory_tree::score::embed::ollama::EMBED_NUM_CTX`])
|
||||
//! [`tinyagents::harness::embeddings::RECOMMENDED_OLLAMA_CONTEXT_TOKENS`])
|
||||
//! and the summariser hard-caps its output to fit that 8192-token embed
|
||||
//! ceiling. A local model whose native context window is below this floor
|
||||
//! silently truncates chunks/summaries and corrupts recall, so we refuse
|
||||
@@ -16,12 +16,12 @@ use serde::Serialize;
|
||||
/// Minimum native context window (tokens) a local model must advertise to
|
||||
/// be accepted by the memory layer.
|
||||
///
|
||||
/// Re-exported from the embedder's own `EMBED_NUM_CTX` so this gate can
|
||||
/// Re-exported from TinyAgents' canonical Ollama context setting so this gate can
|
||||
/// never drift from what the memory pipeline actually requests at embed
|
||||
/// time. Changing the embedder's context request automatically moves the
|
||||
/// acceptance floor with it.
|
||||
pub const MIN_CONTEXT_TOKENS: u64 =
|
||||
crate::openhuman::memory_tree::score::embed::ollama::EMBED_NUM_CTX as u64;
|
||||
tinyagents::harness::embeddings::RECOMMENDED_OLLAMA_CONTEXT_TOKENS as u64;
|
||||
|
||||
/// Verdict for a single model's context window against
|
||||
/// [`MIN_CONTEXT_TOKENS`]. Serialized into the diagnostics payload so the
|
||||
@@ -79,7 +79,7 @@ mod tests {
|
||||
// requests; this guards against the two drifting apart.
|
||||
assert_eq!(
|
||||
MIN_CONTEXT_TOKENS,
|
||||
crate::openhuman::memory_tree::score::embed::ollama::EMBED_NUM_CTX as u64
|
||||
tinyagents::harness::embeddings::RECOMMENDED_OLLAMA_CONTEXT_TOKENS as u64
|
||||
);
|
||||
assert_eq!(MIN_CONTEXT_TOKENS, 8_192);
|
||||
}
|
||||
|
||||
@@ -111,7 +111,7 @@ These are subscriber registrations rather than a single `bus.rs`; subscriptions
|
||||
- `src/core/all.rs` — registers the `learning.*` controllers + schemas.
|
||||
- `src/openhuman/agent/harness/session/{builder,turn}.rs` and `agent_memory/memory_loader.rs` — wire the post-turn hooks, prompt sections, and learned-context loading into the agent loop.
|
||||
- `src/openhuman/channels/runtime/startup.rs` — likely registers schedulers/subscribers at startup.
|
||||
- `src/openhuman/memory_store/unified/profile.rs`, `memory_sync/composio/providers/profile.rs`, `memory_tools/{capture,mod}.rs`, `tools/impl/system/tool_stats.rs`, `tools/schemas.rs` — consume facet/learning types.
|
||||
- `src/openhuman/memory_store/namespace_store/profile.rs`, `memory_sync/composio/providers/profile.rs`, `memory_tools/{capture,mod}.rs`, `tools/impl/system/tool_stats.rs`, `tools/schemas.rs` — consume facet/learning types.
|
||||
|
||||
## Notes / gotchas
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ use crate::openhuman::memory_store::profile::{self, ProfileFacet, UserState};
|
||||
/// Thin wrapper around the `user_profile` table.
|
||||
///
|
||||
/// All methods delegate to the standalone helpers in
|
||||
/// `memory::store::unified::profile`. This type exists so callers
|
||||
/// `memory_store::namespace_store::profile`. This type exists so callers
|
||||
/// (stability detector, prompt sections, RPCs) share a single typed
|
||||
/// entry-point that can be constructed from any `Arc<Mutex<Connection>>`.
|
||||
pub struct FacetCache {
|
||||
|
||||
@@ -26,7 +26,7 @@ one job. memory orchestrates and routes between them.
|
||||
| ------------------------------------------ | --------------------------------------------------------------------------------------------------- |
|
||||
| [`memory_store`](../memory_store/) | Storage primitives: raw / chunks / entities / trees / vectors / kv / contacts. SQLite + on-disk md. |
|
||||
| [`memory_tree`](../memory_tree/) | Generic tree mechanics: bucket-seal, flush, summarise, and retrieval/traversal backends. |
|
||||
| [`memory_archivist`](../memory_archivist/) | Chat conversation → clip tool-calls → push to tree (shim over `tinycortex::memory::archivist`). |
|
||||
| `tinycortex::memory::archivist` | Chat conversation → clip tool-calls → push to tree; called directly by the host harness. |
|
||||
| [`memory_tools`](../memory_tools/) | Tool-scoped rules + agent read/write tools. |
|
||||
| [`memory_sync`](../memory_sync/) | Composio + workspace + MCP sync pipelines. |
|
||||
|
||||
|
||||
@@ -57,6 +57,5 @@ pub use traits::{Memory, MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSumm
|
||||
// Re-export types that external tests and consumers historically imported
|
||||
// from `memory::*`. The definitions moved to sibling crates during the
|
||||
// memory refactor; these aliases keep the public surface stable.
|
||||
pub use crate::openhuman::memory_queue as jobs;
|
||||
pub use crate::openhuman::memory_store::types::NamespaceDocumentInput;
|
||||
pub use crate::openhuman::memory_store::{MemoryClient, UnifiedMemory};
|
||||
|
||||
@@ -4,7 +4,7 @@ use crate::openhuman::embeddings::NoopEmbedding;
|
||||
use crate::openhuman::memory::ingest_pipeline::ingest_chat;
|
||||
use crate::openhuman::memory_queue::drain_until_idle;
|
||||
use crate::openhuman::memory_store::content::raw::{write_raw_items, RawItem, RawKind};
|
||||
use crate::openhuman::memory_store::unified::UnifiedMemory;
|
||||
use crate::openhuman::memory_store::namespace_store::UnifiedMemory;
|
||||
use crate::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage};
|
||||
use chrono::{TimeZone, Utc};
|
||||
use rusqlite::params;
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
//! Memory archivist — thin host shim over `tinycortex::memory::archivist` (W7).
|
||||
//!
|
||||
//! The archivist engine (clip / compose / tree-writer / episodic store + its
|
||||
//! types) lives in the crate now. This module keeps the host's
|
||||
//! `memory_archivist::…` import paths stable and adapts the host [`Config`] to
|
||||
//! the crate's `MemoryConfig` for the two episodic-store functions host callers
|
||||
//! use (`store::record_turn` / `store::session_entries`, from the agent
|
||||
//! archivist hook + recap).
|
||||
//!
|
||||
//! On-disk layout is unchanged — the crate writes to the same path the host
|
||||
//! did: `<workspace>/memory_tree/content/episodic/<sanitized-session>/<seq>.md`.
|
||||
//! `Config::workspace_dir` maps to the crate `MemoryConfig.workspace`, so
|
||||
//! `memory_tree_content_root()` (`<workspace>/memory_tree/content`) resolves
|
||||
//! identically on both sides.
|
||||
|
||||
pub use tinycortex::memory::archivist::types::{ArchivedTurn, Turn};
|
||||
|
||||
/// Episodic conversation archive — thin adapters over the crate store that
|
||||
/// convert the host [`Config`](crate::openhuman::config::Config) into the
|
||||
/// crate's `MemoryConfig`.
|
||||
pub mod store {
|
||||
use anyhow::Result;
|
||||
|
||||
use super::ArchivedTurn;
|
||||
use crate::openhuman::config::Config;
|
||||
use crate::openhuman::tinycortex::memory_config_from;
|
||||
|
||||
fn engine_config(config: &Config) -> tinycortex::memory::MemoryConfig {
|
||||
memory_config_from(config, config.workspace_dir.clone())
|
||||
}
|
||||
|
||||
/// Append one turn to its session's episodic archive, assigning the next
|
||||
/// per-session sequence number.
|
||||
pub fn record_turn(config: &Config, turn: ArchivedTurn) -> Result<ArchivedTurn> {
|
||||
tinycortex::memory::archivist::store::record_turn(&engine_config(config), turn)
|
||||
}
|
||||
|
||||
/// All turns recorded for a session, ordered by sequence.
|
||||
pub fn session_entries(config: &Config, session_id: &str) -> Result<Vec<ArchivedTurn>> {
|
||||
tinycortex::memory::archivist::store::session_entries(&engine_config(config), session_id)
|
||||
}
|
||||
}
|
||||
|
||||
pub use store::{record_turn, session_entries};
|
||||
@@ -28,7 +28,6 @@
|
||||
//! This queue used to live under `openhuman::memory::jobs`; it now has a
|
||||
//! dedicated top-level home (`openhuman::memory_queue`) because it is an
|
||||
//! execution/runtime concern rather than a leaf of the memory policy API.
|
||||
//! `openhuman::memory` re-exports it as `memory::jobs` during the migration.
|
||||
|
||||
mod ops;
|
||||
pub mod scheduler;
|
||||
|
||||
@@ -5,9 +5,7 @@
|
||||
//! `memory_tree`) provide persistence and tree traversal; this module composes
|
||||
//! them into tools the agent can invoke.
|
||||
|
||||
pub mod scoring;
|
||||
pub mod tools;
|
||||
pub mod vector;
|
||||
|
||||
// ── Public re-exports ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
//! Scoring weight profiles for hybrid retrieval — thin host shim over
|
||||
//! `tinycortex::memory::WeightProfile` (W5).
|
||||
//!
|
||||
//! The weight profile (graph/vector/keyword/freshness weights + the
|
||||
//! `BALANCED`/`SEMANTIC`/`LEXICAL`/`GRAPH_FIRST` presets + `by_name`) is the
|
||||
//! crate's, a byte-identical port. The host keeps only [`compose_score`] — the
|
||||
//! trivial weighted combination the crate expresses via
|
||||
//! `retrieval::scoring::hybrid_score` at its own call sites; exposed here as a
|
||||
//! free function so `memory_search::tools::hybrid_search` keeps its call shape.
|
||||
|
||||
pub use tinycortex::memory::WeightProfile;
|
||||
|
||||
/// Weighted composite of the four retrieval signals under `profile`.
|
||||
///
|
||||
/// `graph·graph_relevance + vector·vector_similarity + keyword·keyword_relevance
|
||||
/// + freshness·freshness`.
|
||||
pub fn compose_score(
|
||||
profile: &WeightProfile,
|
||||
graph_relevance: f64,
|
||||
vector_similarity: f64,
|
||||
keyword_relevance: f64,
|
||||
freshness: f64,
|
||||
) -> f64 {
|
||||
(profile.graph * graph_relevance)
|
||||
+ (profile.vector * vector_similarity)
|
||||
+ (profile.keyword * keyword_relevance)
|
||||
+ (profile.freshness * freshness)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn compose_score_is_weighted_sum() {
|
||||
let p = WeightProfile::BALANCED;
|
||||
let s = compose_score(&p, 1.0, 1.0, 1.0, 1.0);
|
||||
assert!((s - (p.graph + p.vector + p.keyword + p.freshness)).abs() < 1e-9);
|
||||
}
|
||||
}
|
||||
@@ -12,10 +12,10 @@ use std::sync::Arc;
|
||||
|
||||
use crate::openhuman::config::rpc as config_rpc;
|
||||
use crate::openhuman::embeddings::{provider_from_config, EmbeddingProvider};
|
||||
use crate::openhuman::memory_search::scoring::WeightProfile;
|
||||
use crate::openhuman::memory_store::types::MemoryItemKind;
|
||||
use crate::openhuman::memory_store::UnifiedMemory;
|
||||
use crate::openhuman::tools::traits::{Tool, ToolResult};
|
||||
use tinycortex::memory::WeightProfile;
|
||||
|
||||
pub struct MemoryHybridSearchTool;
|
||||
|
||||
@@ -178,13 +178,14 @@ impl Tool for MemoryHybridSearchTool {
|
||||
.enumerate()
|
||||
.map(|(i, hit)| {
|
||||
let bd = &hit.score_breakdown;
|
||||
let score = crate::openhuman::memory_search::scoring::compose_score(
|
||||
let score = tinycortex::memory::retrieval::scoring::hybrid_score(
|
||||
&profile,
|
||||
bd.graph_relevance,
|
||||
bd.vector_similarity,
|
||||
bd.keyword_relevance,
|
||||
bd.freshness,
|
||||
);
|
||||
)
|
||||
.final_score;
|
||||
(i, score)
|
||||
})
|
||||
.filter(|(_, score)| *score > 0.0)
|
||||
|
||||
@@ -11,13 +11,13 @@ use std::fmt::Write;
|
||||
|
||||
use crate::openhuman::config::rpc as config_rpc;
|
||||
use crate::openhuman::embeddings::provider_from_config;
|
||||
use crate::openhuman::memory_search::vector::mmr::{mmr_select, MmrCandidate};
|
||||
use crate::openhuman::memory_store::chunks::store::{
|
||||
get_chunk_embeddings_for_signature_batch, list_chunks, ListChunksQuery,
|
||||
};
|
||||
use crate::openhuman::memory_store::chunks::types::SourceKind;
|
||||
use crate::openhuman::memory_store::vectors::cosine_similarity;
|
||||
use crate::openhuman::tools::traits::{Tool, ToolResult};
|
||||
use tinycortex::memory::retrieval::mmr::{mmr_select, MmrCandidate};
|
||||
|
||||
pub struct MemoryVectorSearchTool;
|
||||
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
//! Maximal Marginal Relevance selection — thin host re-export of
|
||||
//! `tinycortex::memory::retrieval::mmr` (W5).
|
||||
//!
|
||||
//! The MMR algorithm (relevance–diversity tradeoff over embeddings) is the
|
||||
//! crate's, a byte-identical port. Host consumers (`memory_search::tools`) keep
|
||||
//! their `memory_search::vector::mmr::{MmrCandidate, MmrResult, mmr_select}`
|
||||
//! import paths unchanged.
|
||||
|
||||
pub use tinycortex::memory::retrieval::mmr::{mmr_select, MmrCandidate, MmrResult};
|
||||
@@ -1,3 +0,0 @@
|
||||
//! Vector diversity algorithms.
|
||||
|
||||
pub mod mmr;
|
||||
@@ -12,9 +12,8 @@ trees/ summary tree persistence (one table, kind-parameterized)
|
||||
vectors/ local vector DB (cosine, brute-force)
|
||||
kv/ global + namespace key-value (kv_global, kv_namespace)
|
||||
contacts/ [removed] facade over people::store (Person/Handle/Interaction)
|
||||
unified/ [staging for removal] UnifiedMemory's remaining SQLite surface
|
||||
(documents/query/segments/events/profile) — replaced as the
|
||||
Memory trait callers migrate to per-kind backends
|
||||
namespace_store/ host-retained namespace documents, graph, episodic/event/
|
||||
segment/profile tables, and retrieval policy
|
||||
```
|
||||
|
||||
## Cross-cutting modules
|
||||
@@ -43,7 +42,7 @@ unified/ [staging for removal] UnifiedMemory's remaining SQLite surface
|
||||
| [`vectors/`](vectors/) | Standalone vector store. `VectorStore` over SQLite, byte-codec for f32 vectors, cosine similarity. |
|
||||
| [`kv.rs`](kv.rs) | Global + namespace key-value (`kv_global`, `kv_namespace` tables). |
|
||||
| `contacts/` | Removed. Contact access now lives outside `memory_store` via `people::store`. |
|
||||
| [`unified/`](unified/) | **Staging for removal.** Shrinking SQLite surface that still backs the `Memory` trait while callers migrate to per-kind modules. Active pieces today: `documents`, `query`, `segments`, `events`, `profile`. The `fts5` episodic surface is replaced by [`memory_archivist`](../memory_archivist/) and `graph` by the crate's `tinycortex::memory::graph` (the host `memory_graph` placeholder was deleted in the W7 migration). See [`unified/README.md`](unified/README.md). |
|
||||
| [`namespace_store/`](namespace_store/) | Host-retained namespace/document tier over the shared SQLite database: documents, persisted product graph relations, episodic/events, segments, profile facets, and host retrieval policy. TinyCortex owns the generic chunk/vector/tree/queue substrate; this tier remains the stable `Memory` implementation. See [`namespace_store/README.md`](namespace_store/README.md). |
|
||||
|
||||
## Layer rules
|
||||
|
||||
|
||||
@@ -17,10 +17,10 @@ use crate::openhuman::memory::ingestion::{
|
||||
IngestionJob, IngestionQueue, IngestionState, MemoryIngestionConfig, MemoryIngestionRequest,
|
||||
MemoryIngestionResult,
|
||||
};
|
||||
use crate::openhuman::memory_store::namespace_store::UnifiedMemory;
|
||||
use crate::openhuman::memory_store::types::{
|
||||
NamespaceDocumentInput, NamespaceMemoryHit, NamespaceRetrievalContext,
|
||||
};
|
||||
use crate::openhuman::memory_store::unified::UnifiedMemory;
|
||||
|
||||
/// Reference-counted handle to a `MemoryClient`.
|
||||
pub type MemoryClientRef = Arc<MemoryClient>;
|
||||
@@ -56,7 +56,7 @@ pub struct MemoryClient {
|
||||
impl MemoryClient {
|
||||
/// Returns a handle to the underlying SQLite connection for direct
|
||||
/// profile-facet writes via
|
||||
/// [`crate::openhuman::memory_store::unified::profile::profile_upsert`].
|
||||
/// [`crate::openhuman::memory_store::namespace_store::profile::profile_upsert`].
|
||||
///
|
||||
/// Intentionally `pub(crate)` — external consumers should use the
|
||||
/// higher-level `MemoryClient` API; this escape hatch exists so
|
||||
|
||||
@@ -21,7 +21,7 @@ use crate::openhuman::embeddings::{
|
||||
DEFAULT_CLOUD_EMBEDDING_MODEL, DEFAULT_OLLAMA_DIMENSIONS, DEFAULT_OLLAMA_MODEL,
|
||||
};
|
||||
use crate::openhuman::memory::traits::Memory;
|
||||
use crate::openhuman::memory_store::unified::UnifiedMemory;
|
||||
use crate::openhuman::memory_store::namespace_store::UnifiedMemory;
|
||||
|
||||
/// One-shot guard so the Ollama health-gate fallback only reports to Sentry
|
||||
/// once per process lifetime. Memory is constructed many times per session
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
|
||||
use tinycortex::memory::store::kv::KvStore;
|
||||
|
||||
use crate::openhuman::memory_store::namespace_store::UnifiedMemory;
|
||||
use crate::openhuman::memory_store::types::MemoryKvRecord;
|
||||
use crate::openhuman::memory_store::unified::UnifiedMemory;
|
||||
|
||||
impl UnifiedMemory {
|
||||
fn tinycortex_kv(&self) -> Result<KvStore, String> {
|
||||
|
||||
@@ -17,11 +17,11 @@ use serde_json::json;
|
||||
use crate::openhuman::memory::traits::{
|
||||
Memory, MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary, RecallOpts,
|
||||
};
|
||||
use crate::openhuman::memory_store::namespace_store::fts5;
|
||||
use crate::openhuman::memory_store::types::{NamespaceDocumentInput, GLOBAL_NAMESPACE};
|
||||
use crate::openhuman::memory_store::unified::fts5;
|
||||
use anyhow::Context;
|
||||
|
||||
use super::unified::UnifiedMemory;
|
||||
use super::namespace_store::UnifiedMemory;
|
||||
|
||||
/// Convert a UNIX timestamp (f64) to RFC3339 string.
|
||||
fn timestamp_to_rfc3339(ts: f64) -> String {
|
||||
|
||||
@@ -11,7 +11,8 @@
|
||||
//! ## Submodules
|
||||
//!
|
||||
//! - `types`: Common data structures and types used across the memory store.
|
||||
//! - `unified`: The primary SQLite-based memory implementation.
|
||||
//! - `namespace_store`: Host-retained SQLite namespace documents, graph,
|
||||
//! episodic/event/segment/profile tables, and their query policy.
|
||||
//! - `client`: High-level client interface for interacting with the memory system.
|
||||
//! - `factories`: Factory functions for creating and initializing memory instances.
|
||||
//! - `memory_trait`: Defines the `Memory` trait that all implementations must satisfy.
|
||||
@@ -21,13 +22,13 @@ pub mod content;
|
||||
pub mod entities;
|
||||
pub mod kinds;
|
||||
pub mod kv;
|
||||
pub mod namespace_store;
|
||||
pub mod retrieval;
|
||||
pub mod safety;
|
||||
pub mod tools;
|
||||
pub mod traits;
|
||||
pub mod trees;
|
||||
pub mod types;
|
||||
pub mod unified;
|
||||
pub mod vectors;
|
||||
|
||||
mod client;
|
||||
@@ -42,16 +43,16 @@ pub use factories::{
|
||||
active_embedding_signature, create_memory, create_memory_for_migration,
|
||||
create_memory_with_local_ai, effective_embedding_settings, effective_memory_backend_name,
|
||||
};
|
||||
pub use namespace_store::events;
|
||||
pub use namespace_store::fts5;
|
||||
pub use namespace_store::profile;
|
||||
pub use namespace_store::segments;
|
||||
pub use namespace_store::UnifiedMemory;
|
||||
pub use types::{
|
||||
GraphRelationRecord, MemoryItemKind, MemoryKvRecord, NamespaceDocumentInput,
|
||||
NamespaceMemoryHit, NamespaceQueryResult, NamespaceRetrievalContext, RetrievalScoreBreakdown,
|
||||
StoredMemoryDocument,
|
||||
};
|
||||
pub use unified::events;
|
||||
pub use unified::fts5;
|
||||
pub use unified::profile;
|
||||
pub use unified::segments;
|
||||
pub use unified::UnifiedMemory;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
+7
-2
@@ -1,6 +1,11 @@
|
||||
# Unified memory store
|
||||
# Namespace memory store
|
||||
|
||||
SQLite-backed implementation of the memory store. One `UnifiedMemory` struct owns a WAL-mode connection plus the on-disk markdown sidecar tree and vector storage path; the rest of this directory adds capabilities to it via per-domain `impl` blocks.
|
||||
Host-retained SQLite namespace/document tier. One `UnifiedMemory` struct owns
|
||||
the shared connection plus the on-disk markdown sidecar and compatibility
|
||||
embedding handle; the rest of this directory adds the product-owned document,
|
||||
graph, episodic, event, segment, profile, and retrieval policy via `impl`
|
||||
blocks. TinyCortex owns the generic chunk/vector/tree/queue engine beside this
|
||||
tier; this directory is intentionally not migration staging.
|
||||
|
||||
## Files
|
||||
|
||||
+2
-2
@@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS conversation_segments (
|
||||
status TEXT NOT NULL DEFAULT 'open',
|
||||
created_at REAL NOT NULL,
|
||||
updated_at REAL NOT NULL,
|
||||
-- Per-session sequence numbers from memory_archivist::store, populated
|
||||
-- Per-session sequence numbers from tinycortex::memory::archivist::store, populated
|
||||
-- alongside start_episodic_id / end_episodic_id during the FTS5 -> md
|
||||
-- migration. Once STM recall switches its segment-span dedup to use
|
||||
-- (session_id, seq) the legacy episodic_id columns can be dropped.
|
||||
@@ -108,7 +108,7 @@ pub struct ConversationSegment {
|
||||
pub status: SegmentStatus,
|
||||
pub created_at: f64,
|
||||
pub updated_at: f64,
|
||||
/// Per-session seq number assigned by `memory_archivist::store::record_turn`
|
||||
/// Per-session seq number assigned by the TinyCortex archivist store.
|
||||
/// for the user turn that opened this segment. `None` on legacy rows
|
||||
/// written before the FTS5 -> md migration began.
|
||||
#[serde(default)]
|
||||
@@ -1,82 +1,28 @@
|
||||
# memory_sync
|
||||
|
||||
Every "pull data from upstream → land it in memory_store" pipeline in
|
||||
one place, organised by the kind of upstream they talk to.
|
||||
OpenHuman orchestration and product policy around the TinyCortex sync engine.
|
||||
|
||||
## Three pipeline kinds
|
||||
TinyCortex owns generic Composio provider fetch/pagination, canonical memory
|
||||
records, sync budgets/state, workspace reconciliation, and persistence traits.
|
||||
The live host path calls it through `src/openhuman/tinycortex/sync.rs` from
|
||||
`composio::run_connection_sync` and the default provider `sync()` method.
|
||||
|
||||
| Kind | Submodule | Owns |
|
||||
| --- | --- | --- |
|
||||
| **Composio** | [`composio/`](composio/) | Per-provider sync via the Composio Edge API: gmail, slack, github, notion, linear, clickup, … |
|
||||
| **Workspace** | [`workspace/`](workspace/) | Vault file watch, harness turn capture, dictation transcripts — anything local. |
|
||||
| **MCP** | [`mcp/`](mcp/) | Third-party MCP servers via `mcp_clients/` transport. |
|
||||
OpenHuman retains:
|
||||
|
||||
## Trait
|
||||
- periodic scheduling and connection selection;
|
||||
- credentials and Composio action execution;
|
||||
- source-scope and redaction policy;
|
||||
- translation into host `DomainEvent`s;
|
||||
- JSON-RPC/status/connect surfaces;
|
||||
- agent-facing action tools and result post-processing;
|
||||
- product task/profile projections for GitHub, Notion, Linear, and ClickUp;
|
||||
- local workspace watching and MCP orchestration.
|
||||
|
||||
Every pipeline implements [`SyncPipeline`]:
|
||||
The provider directories therefore are not alternate sync engines. Their
|
||||
remaining `provider.rs`, `tools.rs`, `normalization.rs`, profile, catalog, and
|
||||
post-processing files implement host product surfaces over the crate-backed
|
||||
sync path. New generic parsing or persistence behavior belongs in
|
||||
`vendor/tinycortex/src/memory/sync/`.
|
||||
|
||||
```rust
|
||||
async fn init(&self, &Config) -> anyhow::Result<()>;
|
||||
async fn tick(&self, &Config) -> anyhow::Result<SyncOutcome>;
|
||||
fn id(&self) -> &str;
|
||||
fn kind(&self) -> SyncPipelineKind;
|
||||
```
|
||||
|
||||
`SyncOutcome { records_ingested, more_pending, note }` is the
|
||||
orchestrator-facing result; pipelines own their own pagination cursors
|
||||
and retry policy behind that.
|
||||
|
||||
## Layout
|
||||
|
||||
| Path | Role |
|
||||
| --- | --- |
|
||||
| [`mod.rs`](mod.rs) | Module root + re-exports. |
|
||||
| [`traits.rs`](traits.rs) | `SyncPipeline`, `SyncOutcome`, `SyncPipelineKind`. |
|
||||
| [`composio/`](composio/) | Per-provider pipelines (gmail, slack, github, notion, linear, clickup). |
|
||||
| [`workspace/`](workspace/) | Vault, harness, dictation pipelines. |
|
||||
| [`mcp/`](mcp/) | MCP-server pipelines (one per connected server). |
|
||||
|
||||
## Schedulers & self-healing
|
||||
|
||||
Three background loops keep memory synced and compressed without manual
|
||||
"Sync now" clicks:
|
||||
|
||||
| Loop | Module | Cadence | Covers |
|
||||
| --- | --- | --- | --- |
|
||||
| Composio periodic | [`composio/periodic.rs`](composio/periodic.rs) | 20-min tick, per-connection interval | Composio connections (gmail, slack, …) |
|
||||
| Workspace periodic | [`workspace/periodic.rs`](workspace/periodic.rs) | 20-min tick, `memory_sync_interval_secs` (24h default, `0` = manual only) | `github_repo`, `folder`, `rss_feed`, `web_page` sources |
|
||||
| Queue scheduler | `memory_queue::scheduler` | 3 h | flush stale L0 buffers + requeue transient-failed jobs |
|
||||
|
||||
**Raw-archive coverage** ([`sources/rebuild.rs`](sources/rebuild.rs)): every
|
||||
summary batch records its raw files in `mem_tree_ingested_sources`
|
||||
(`source_kind = "raw_file"`). After each sync, `check_and_rebuild_tree`
|
||||
diffs disk vs gate and summarises only the uncovered remainder — so an
|
||||
interrupted sync self-heals on the next pass instead of stranding raw
|
||||
files forever. Inspect/trigger over RPC with
|
||||
`openhuman.memory_sources_reconcile`.
|
||||
|
||||
⚠️ A source's **tree scope** and **raw-archive id** slugify to different
|
||||
directories for GitHub (`github:owner/repo` vs `github.com/owner/repo`) —
|
||||
always carry both (`memory_sources::sync::SourceScope`).
|
||||
|
||||
## Status
|
||||
|
||||
**Pipelines: scaffold only.** Today's sync code still lives in:
|
||||
|
||||
- `composio/providers/<provider>/ingest.rs` + `bin/{slack_backfill,gmail_backfill_3d}.rs`
|
||||
- `vault/sync.rs`, `agent_experience/`, `dictation_hotkeys/`
|
||||
- `mcp_clients/` (transport only; no drain loop yet)
|
||||
|
||||
Each migrates here as its own per-pipeline PR. The job-queue orchestration
|
||||
in `memory::jobs` stays put — it just gains the ability to iterate over a
|
||||
registered `Vec<Box<dyn SyncPipeline>>`.
|
||||
|
||||
## Layer rules
|
||||
|
||||
- Sync writes go through `memory::ingest_pipeline` so every record
|
||||
lands as raw md → chunks → tree leaves like any other ingest.
|
||||
- No direct writes into trees or unified. No upstream-specific data
|
||||
models leak past the pipeline boundary.
|
||||
- One pipeline per upstream service. Composio's GitHub and MCP's GitHub
|
||||
are distinct pipelines because they hit different surfaces with
|
||||
different cadence and auth.
|
||||
D4.1-D4.4 are closed in `docs/tinycortex-drift-ledger.md`. Gmail's bounded
|
||||
25-message page is crate-owned and prevents Composio 413 responses.
|
||||
|
||||
@@ -6,15 +6,15 @@
|
||||
//! re-learning a new shape:
|
||||
//!
|
||||
//! - `provider.rs` — `impl ComposioProvider for ClickUpProvider`
|
||||
//! - `sync.rs` — payload-shape helpers (results extraction, title)
|
||||
//! - `normalization.rs` — payload-shape helpers (results extraction, title)
|
||||
//! - `ingest.rs` — memory_tree document ingest (issue #2885)
|
||||
//! - `tools.rs` — `CLICKUP_CURATED` whitelist of Composio actions
|
||||
//! - `tests.rs` — unit tests for the helpers + trait metadata
|
||||
//!
|
||||
//! Issue: #2288 (introduction); #2885 (memory_tree migration).
|
||||
|
||||
mod normalization;
|
||||
mod provider;
|
||||
mod sync;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
pub mod tools;
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
//! ClickUp sync helpers — result extraction, task-title extraction,
|
||||
//! ClickUp host normalization helpers — result extraction, task-title extraction,
|
||||
//! and time utilities.
|
||||
//!
|
||||
//! ClickUp's REST API (and therefore Composio's wrapping of it) returns
|
||||
@@ -27,7 +27,7 @@
|
||||
use async_trait::async_trait;
|
||||
use serde_json::json;
|
||||
|
||||
use super::sync;
|
||||
use super::normalization;
|
||||
use crate::openhuman::memory_sync::composio::providers::{
|
||||
first_array_str, merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider,
|
||||
CuratedTool, NormalizedTask, ProviderContext, ProviderUserProfile, TaskFetchFilter, TaskKind,
|
||||
@@ -105,7 +105,7 @@ impl ComposioProvider for ClickUpProvider {
|
||||
let data = &resp.data;
|
||||
let display_name = pick_str(data, &["user.username", "data.user.username", "username"]);
|
||||
let email = pick_str(data, &["user.email", "data.user.email", "email"]);
|
||||
let username = sync::extract_user_id(data);
|
||||
let username = normalization::extract_user_id(data);
|
||||
let avatar_url = pick_str(
|
||||
data,
|
||||
&[
|
||||
@@ -168,7 +168,7 @@ impl ComposioProvider for ClickUpProvider {
|
||||
resp.error.unwrap_or_else(|| "provider failure".into())
|
||||
));
|
||||
}
|
||||
sync::extract_workspace_ids(&resp.data)
|
||||
normalization::extract_workspace_ids(&resp.data)
|
||||
}
|
||||
};
|
||||
|
||||
@@ -188,7 +188,7 @@ impl ComposioProvider for ClickUpProvider {
|
||||
resp.error.unwrap_or_else(|| "provider failure".into())
|
||||
));
|
||||
}
|
||||
let id = sync::extract_user_id(&resp.data).ok_or_else(|| {
|
||||
let id = normalization::extract_user_id(&resp.data).ok_or_else(|| {
|
||||
"[composio:clickup] CLICKUP_GET_AUTHORIZED_USER returned no user.id".to_string()
|
||||
})?;
|
||||
vec![id]
|
||||
@@ -227,7 +227,7 @@ impl ComposioProvider for ClickUpProvider {
|
||||
));
|
||||
}
|
||||
|
||||
for task in sync::extract_tasks(&resp.data) {
|
||||
for task in normalization::extract_tasks(&resp.data) {
|
||||
if out.len() >= max {
|
||||
break 'workspaces;
|
||||
}
|
||||
@@ -246,8 +246,8 @@ impl ComposioProvider for ClickUpProvider {
|
||||
/// `None` only when the task has no extractable id (unroutable).
|
||||
fn normalize_clickup_task(task: &serde_json::Value) -> Option<NormalizedTask> {
|
||||
let external_id = pick_str(task, TASK_ID_PATHS)?;
|
||||
let title =
|
||||
sync::extract_task_name(task).unwrap_or_else(|| format!("ClickUp task {external_id}"));
|
||||
let title = normalization::extract_task_name(task)
|
||||
.unwrap_or_else(|| format!("ClickUp task {external_id}"));
|
||||
Some(NormalizedTask {
|
||||
external_id,
|
||||
source_id: String::new(),
|
||||
@@ -265,7 +265,7 @@ fn normalize_clickup_task(task: &serde_json::Value) -> Option<NormalizedTask> {
|
||||
due: pick_str(task, &["due_date", "data.due_date"]),
|
||||
labels: Vec::new(),
|
||||
priority: pick_str(task, &["priority.priority", "data.priority.priority"]),
|
||||
updated_at: sync::extract_task_updated(task),
|
||||
updated_at: normalization::extract_task_updated(task),
|
||||
raw: task.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Unit tests for the ClickUp provider.
|
||||
|
||||
use super::sync::{
|
||||
use super::normalization::{
|
||||
extract_task_name, extract_task_updated, extract_tasks, extract_user_id, extract_workspace_ids,
|
||||
};
|
||||
use super::ClickUpProvider;
|
||||
|
||||
@@ -6,14 +6,14 @@
|
||||
//! re-learning a new shape:
|
||||
//!
|
||||
//! - `provider.rs` — `impl ComposioProvider for GitHubProvider`
|
||||
//! - `sync.rs` — payload-shape helpers (result extraction, title, cursor)
|
||||
//! - `normalization.rs` — payload-shape helpers (result extraction, title, cursor)
|
||||
//! - `tools.rs` — `GITHUB_CURATED` whitelist of Composio actions
|
||||
//! - `tests.rs` — unit tests for the helpers + trait metadata
|
||||
//!
|
||||
//! Issue: #2408.
|
||||
|
||||
mod normalization;
|
||||
mod provider;
|
||||
mod sync;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
pub mod tools;
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
//! GitHub sync helpers — result extraction, identity helpers, and time utilities.
|
||||
//! GitHub host normalization helpers — result extraction, identity helpers, and time utilities.
|
||||
//!
|
||||
//! GitHub's REST API (proxied through Composio) returns search results and
|
||||
//! authenticated-user payloads in a small number of shapes. The functions here
|
||||
@@ -23,7 +23,7 @@ use async_trait::async_trait;
|
||||
use serde_json::{json, Value};
|
||||
use std::time::Duration;
|
||||
|
||||
use super::sync;
|
||||
use super::normalization;
|
||||
use crate::openhuman::memory_sync::composio::providers::{
|
||||
merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider, CuratedTool,
|
||||
GithubFetchMode, NormalizedTask, ProviderContext, ProviderUserProfile, TaskFetchFilter,
|
||||
@@ -94,7 +94,7 @@ impl ComposioProvider for GitHubProvider {
|
||||
}
|
||||
|
||||
let data = &resp.data;
|
||||
let login = sync::extract_user_login(data);
|
||||
let login = normalization::extract_user_login(data);
|
||||
let display_name = pick_str(data, &["name", "data.name"]).or_else(|| login.clone());
|
||||
let email = pick_str(data, &["email", "data.email"]);
|
||||
let avatar_url = pick_str(data, &["avatar_url", "data.avatar_url"]);
|
||||
@@ -157,7 +157,7 @@ impl ComposioProvider for GitHubProvider {
|
||||
};
|
||||
|
||||
let mut out: Vec<NormalizedTask> = Vec::new();
|
||||
for issue in sync::extract_issues(&data) {
|
||||
for issue in normalization::extract_issues(&data) {
|
||||
if out.len() >= max {
|
||||
break;
|
||||
}
|
||||
@@ -481,7 +481,7 @@ pub(super) fn normalize_github_repo_filter(raw: &str) -> String {
|
||||
/// not depend on the fetch query), so even if a `closed` item slips through
|
||||
/// the query bias it is dropped here.
|
||||
pub(super) fn normalize_github_issue(issue: &serde_json::Value) -> Option<NormalizedTask> {
|
||||
let external_id = sync::extract_issue_id(issue)?;
|
||||
let external_id = normalization::extract_issue_id(issue)?;
|
||||
let status = pick_str(issue, &["state", "data.state"]);
|
||||
if status
|
||||
.as_deref()
|
||||
@@ -494,8 +494,8 @@ pub(super) fn normalize_github_issue(issue: &serde_json::Value) -> Option<Normal
|
||||
);
|
||||
return None;
|
||||
}
|
||||
let title =
|
||||
sync::extract_issue_title(issue).unwrap_or_else(|| format!("GitHub issue {external_id}"));
|
||||
let title = normalization::extract_issue_title(issue)
|
||||
.unwrap_or_else(|| format!("GitHub issue {external_id}"));
|
||||
let kind = if is_pull_request(issue) {
|
||||
TaskKind::PullRequest
|
||||
} else {
|
||||
@@ -514,7 +514,7 @@ pub(super) fn normalize_github_issue(issue: &serde_json::Value) -> Option<Normal
|
||||
due: None,
|
||||
labels: extract_github_labels(issue),
|
||||
priority: None,
|
||||
updated_at: sync::extract_issue_updated_at(issue),
|
||||
updated_at: normalization::extract_issue_updated_at(issue),
|
||||
raw: issue.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
//! Unit tests for the GitHub Composio provider.
|
||||
|
||||
use super::normalization::{
|
||||
extract_issue_id, extract_issue_title, extract_issue_updated_at, extract_issues,
|
||||
extract_user_login,
|
||||
};
|
||||
use super::provider::github_env_token;
|
||||
use super::provider::{
|
||||
build_fetch_query, build_search_query, github_search_arg_pairs, normalize_github_issue,
|
||||
normalize_github_repo_filter, ACTION_GET_AUTHENTICATED_USER, ACTION_SEARCH_ISSUES,
|
||||
};
|
||||
use super::sync::{
|
||||
extract_issue_id, extract_issue_title, extract_issue_updated_at, extract_issues,
|
||||
extract_user_login,
|
||||
};
|
||||
use super::tools::GITHUB_CURATED;
|
||||
use super::GitHubProvider;
|
||||
use crate::openhuman::memory_sync::composio::providers::ComposioProvider;
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
mod post_process;
|
||||
mod provider;
|
||||
mod sync;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
pub mod tools;
|
||||
|
||||
@@ -1,242 +0,0 @@
|
||||
//! Gmail sync helpers — message extraction, pagination, cursor
|
||||
//! conversion, and time utilities.
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
/// Walk the Composio response envelope and pull out message objects.
|
||||
pub(crate) fn extract_messages(data: &Value) -> Vec<Value> {
|
||||
let candidates = [
|
||||
data.pointer("/data/messages"),
|
||||
data.pointer("/messages"),
|
||||
data.pointer("/data/data/messages"),
|
||||
data.pointer("/data/items"),
|
||||
data.pointer("/items"),
|
||||
];
|
||||
for cand in candidates.into_iter().flatten() {
|
||||
if let Some(arr) = cand.as_array() {
|
||||
return arr.clone();
|
||||
}
|
||||
}
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
/// Try to extract a pagination token from the API response.
|
||||
pub(crate) fn extract_page_token(data: &Value) -> Option<String> {
|
||||
let candidates = [
|
||||
data.pointer("/data/nextPageToken"),
|
||||
data.pointer("/nextPageToken"),
|
||||
data.pointer("/data/data/nextPageToken"),
|
||||
];
|
||||
for cand in candidates.into_iter().flatten() {
|
||||
if let Some(s) = cand.as_str() {
|
||||
let trimmed = s.trim();
|
||||
if !trimmed.is_empty() {
|
||||
return Some(trimmed.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Convert a cursor value (epoch millis or date string) into a Gmail
|
||||
/// `after:YYYY/MM/DD` filter component. Day-level precision — kept as a
|
||||
/// last-resort filter for non-numeric cursors and for back-compat with
|
||||
/// the older day-cursor write path. New code should prefer
|
||||
/// [`cursor_to_gmail_after_epoch_filter`] which produces a
|
||||
/// second-precision `after:<unix>` filter and so avoids re-fetching
|
||||
/// large same-day windows on every tick.
|
||||
pub(crate) fn cursor_to_gmail_after_filter(cursor: &str) -> Option<String> {
|
||||
let cursor = cursor.trim();
|
||||
// Try parsing as epoch millis first (Gmail's internalDate).
|
||||
if let Ok(millis) = cursor.parse::<i64>() {
|
||||
let secs = millis / 1000;
|
||||
if let Some(dt) = chrono::DateTime::from_timestamp(secs, 0) {
|
||||
return Some(dt.format("%Y/%m/%d").to_string());
|
||||
}
|
||||
}
|
||||
// Try parsing as an ISO date/datetime.
|
||||
if let Ok(dt) = chrono::NaiveDate::parse_from_str(cursor, "%Y-%m-%d") {
|
||||
return Some(dt.format("%Y/%m/%d").to_string());
|
||||
}
|
||||
if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(cursor) {
|
||||
return Some(dt.format("%Y/%m/%d").to_string());
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Convert a cursor value (epoch millis or date string) into a Gmail
|
||||
/// `after:<unix_seconds>` filter component. Gmail's search syntax
|
||||
/// accepts a bare unix-seconds value for `after:` / `before:`, so the
|
||||
/// filter is second-precision rather than the day-level
|
||||
/// `after:YYYY/MM/DD` form. Used by the incremental sync path so a
|
||||
/// same-day re-tick does not re-fetch every message Gmail has filed
|
||||
/// today.
|
||||
///
|
||||
/// Returns `None` when the cursor cannot be parsed; callers should
|
||||
/// fall back to the coarse day filter to avoid sending an unbounded
|
||||
/// query.
|
||||
pub(crate) fn cursor_to_gmail_after_epoch_filter(cursor: &str) -> Option<String> {
|
||||
let secs = parse_cursor_to_epoch_secs(cursor)?;
|
||||
Some(secs.to_string())
|
||||
}
|
||||
|
||||
/// Parse a cursor (epoch millis as string, `YYYY-MM-DD`, or RFC3339)
|
||||
/// into unix-seconds. Shared by the epoch filter and by the adaptive
|
||||
/// page-cap recency check.
|
||||
pub(crate) fn parse_cursor_to_epoch_secs(cursor: &str) -> Option<i64> {
|
||||
let cursor = cursor.trim();
|
||||
if let Ok(millis) = cursor.parse::<i64>() {
|
||||
return Some(millis / 1000);
|
||||
}
|
||||
if let Ok(date) = chrono::NaiveDate::parse_from_str(cursor, "%Y-%m-%d") {
|
||||
return date.and_hms_opt(0, 0, 0).map(|dt| dt.and_utc().timestamp());
|
||||
}
|
||||
if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(cursor) {
|
||||
return Some(dt.timestamp());
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub(crate) fn now_ms() -> u64 {
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as u64)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn extract_messages_from_data_messages() {
|
||||
let data = json!({"data": {"messages": [{"id": "1"}, {"id": "2"}]}});
|
||||
let msgs = extract_messages(&data);
|
||||
assert_eq!(msgs.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_messages_from_top_level() {
|
||||
let data = json!({"messages": [{"id": "1"}]});
|
||||
let msgs = extract_messages(&data);
|
||||
assert_eq!(msgs.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_messages_from_data_items() {
|
||||
let data = json!({"data": {"items": [{"id": "a"}]}});
|
||||
let msgs = extract_messages(&data);
|
||||
assert_eq!(msgs.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_messages_empty_when_no_match() {
|
||||
let data = json!({"foo": "bar"});
|
||||
assert!(extract_messages(&data).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_page_token_from_data() {
|
||||
let data = json!({"data": {"nextPageToken": "abc123"}});
|
||||
assert_eq!(extract_page_token(&data), Some("abc123".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_page_token_from_top_level() {
|
||||
let data = json!({"nextPageToken": "tok"});
|
||||
assert_eq!(extract_page_token(&data), Some("tok".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_page_token_none_when_empty() {
|
||||
let data = json!({"data": {"nextPageToken": " "}});
|
||||
assert_eq!(extract_page_token(&data), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_page_token_none_when_missing() {
|
||||
let data = json!({"data": {}});
|
||||
assert_eq!(extract_page_token(&data), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cursor_to_filter_epoch_millis() {
|
||||
let filter = cursor_to_gmail_after_filter("1700000000000").unwrap();
|
||||
assert!(filter.contains('/'));
|
||||
assert_eq!(filter, "2023/11/14");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cursor_to_filter_iso_date() {
|
||||
let filter = cursor_to_gmail_after_filter("2024-01-15").unwrap();
|
||||
assert_eq!(filter, "2024/01/15");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cursor_to_filter_rfc3339() {
|
||||
let filter = cursor_to_gmail_after_filter("2024-06-01T12:00:00Z").unwrap();
|
||||
assert_eq!(filter, "2024/06/01");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cursor_to_filter_invalid_returns_none() {
|
||||
assert!(cursor_to_gmail_after_filter("not-a-date").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cursor_to_filter_trims_whitespace() {
|
||||
let filter = cursor_to_gmail_after_filter(" 2024-01-15 ").unwrap();
|
||||
assert_eq!(filter, "2024/01/15");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn now_ms_returns_nonzero() {
|
||||
assert!(now_ms() > 0);
|
||||
}
|
||||
|
||||
// ── second-precision cursor ──────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn epoch_filter_emits_unix_seconds_for_internal_date_millis() {
|
||||
let filter = cursor_to_gmail_after_epoch_filter("1700000000000").unwrap();
|
||||
assert_eq!(filter, "1700000000");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn epoch_filter_handles_iso_date() {
|
||||
let filter = cursor_to_gmail_after_epoch_filter("2024-01-15").unwrap();
|
||||
// 2024-01-15 00:00:00 UTC == 1705276800.
|
||||
assert_eq!(filter, "1705276800");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn epoch_filter_handles_rfc3339() {
|
||||
let filter = cursor_to_gmail_after_epoch_filter("2024-06-01T12:00:00Z").unwrap();
|
||||
assert_eq!(filter, "1717243200");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn epoch_filter_returns_none_for_garbage() {
|
||||
assert!(cursor_to_gmail_after_epoch_filter("not-a-date").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn epoch_filter_trims_whitespace() {
|
||||
let filter = cursor_to_gmail_after_epoch_filter(" 2024-01-15 ").unwrap();
|
||||
assert_eq!(filter, "1705276800");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_cursor_round_trip_matches_epoch_filter() {
|
||||
// The adaptive page-cap relies on parse_cursor_to_epoch_secs
|
||||
// agreeing with cursor_to_gmail_after_epoch_filter — both must
|
||||
// emit the same seconds value for any given input.
|
||||
for cursor in ["1700000000000", "2024-01-15", "2024-06-01T12:00:00Z"] {
|
||||
let secs = parse_cursor_to_epoch_secs(cursor).unwrap();
|
||||
let filter = cursor_to_gmail_after_epoch_filter(cursor).unwrap();
|
||||
assert_eq!(filter, secs.to_string(), "cursor `{cursor}`");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,301 +1,46 @@
|
||||
//! Unit tests for the Gmail provider.
|
||||
//! Host-owned Gmail provider surface tests.
|
||||
//!
|
||||
//! Pagination, cursor, envelope parsing, and ingest behavior are owned and
|
||||
//! tested by `tinycortex::memory::sync::GmailSyncPipeline`.
|
||||
|
||||
use super::provider::{BASE_QUERY, SENT_QUERIES};
|
||||
use super::sync::{
|
||||
cursor_to_gmail_after_epoch_filter, cursor_to_gmail_after_filter, extract_messages,
|
||||
extract_page_token, now_ms, parse_cursor_to_epoch_secs,
|
||||
};
|
||||
use super::GmailProvider;
|
||||
use crate::openhuman::memory_sync::composio::providers::ComposioProvider;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn extract_messages_finds_data_messages() {
|
||||
let v = json!({
|
||||
"data": { "messages": [{"id": "m1"}, {"id": "m2"}] },
|
||||
"successful": true,
|
||||
});
|
||||
assert_eq!(extract_messages(&v).len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_messages_finds_top_level_messages() {
|
||||
let v = json!({ "messages": [{"id": "m1"}] });
|
||||
assert_eq!(extract_messages(&v).len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_messages_returns_empty_when_missing() {
|
||||
let v = json!({ "data": { "other": [] } });
|
||||
assert_eq!(extract_messages(&v).len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_page_token_finds_nested() {
|
||||
let v = json!({ "data": { "nextPageToken": "tok123" } });
|
||||
assert_eq!(extract_page_token(&v), Some("tok123".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_page_token_none_when_missing() {
|
||||
let v = json!({ "data": {} });
|
||||
assert_eq!(extract_page_token(&v), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cursor_to_filter_from_epoch_millis() {
|
||||
// 1774915200000 ms = 2026-03-31 UTC
|
||||
let millis = "1774915200000";
|
||||
assert_eq!(
|
||||
cursor_to_gmail_after_filter(millis),
|
||||
Some("2026/03/31".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cursor_to_filter_from_iso_date() {
|
||||
assert_eq!(
|
||||
cursor_to_gmail_after_filter("2026-03-15"),
|
||||
Some("2026/03/15".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cursor_to_filter_from_rfc3339() {
|
||||
let f = cursor_to_gmail_after_filter("2026-03-15T12:00:00Z");
|
||||
assert_eq!(f, Some("2026/03/15".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cursor_to_filter_returns_none_for_garbage() {
|
||||
assert_eq!(cursor_to_gmail_after_filter("not-a-date"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_metadata_is_stable() {
|
||||
let p = GmailProvider::new();
|
||||
assert_eq!(p.toolkit_slug(), "gmail");
|
||||
assert_eq!(p.sync_interval_secs(), Some(15 * 60));
|
||||
let provider = GmailProvider::new();
|
||||
assert_eq!(provider.toolkit_slug(), "gmail");
|
||||
assert_eq!(provider.sync_interval_secs(), Some(15 * 60));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_impl_matches_new() {
|
||||
let _a = GmailProvider::new();
|
||||
let _b = GmailProvider::default();
|
||||
// Both are unit structs — constructing via Default is the cover target.
|
||||
let _new = GmailProvider::new();
|
||||
let _default = GmailProvider::default();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn epoch_filter_is_preferred_over_day_filter_for_typical_internal_date() {
|
||||
// The provider tries `cursor_to_gmail_after_epoch_filter` first
|
||||
// and only falls back to the day filter if the parse fails. Both
|
||||
// helpers must accept the same internalDate (epoch millis) input
|
||||
// so the fallback path is genuinely a fallback, not a divergence.
|
||||
let internal_date = "1774915200000"; // 2026-03-31 UTC
|
||||
let epoch = cursor_to_gmail_after_epoch_filter(internal_date).unwrap();
|
||||
let day = cursor_to_gmail_after_filter(internal_date).unwrap();
|
||||
assert_eq!(epoch, "1774915200");
|
||||
assert_eq!(day, "2026/03/31");
|
||||
// Sanity bound: the epoch filter must be after 2020 and before
|
||||
// year 2100, otherwise we shipped a regression in the cursor
|
||||
// converter that would silently let queries land on year 1970.
|
||||
let secs: i64 = epoch.parse().unwrap();
|
||||
assert!(
|
||||
secs > 1_577_836_800,
|
||||
"epoch filter must be after 2020-01-01"
|
||||
);
|
||||
assert!(
|
||||
secs < 4_102_444_800,
|
||||
"epoch filter must be before 2100-01-01"
|
||||
);
|
||||
}
|
||||
|
||||
// ── Adaptive page cap and early-stop helpers (issue#1404, pr#1474) ──────────
|
||||
//
|
||||
// The full `sync()` path needs a live ComposioClient + MemoryClient, so
|
||||
// we test the helper functions that gate the adaptive cap and early-stop
|
||||
// decisions:
|
||||
//
|
||||
// * `parse_cursor_to_epoch_secs` — used to decide whether `last_sync_at_ms`
|
||||
// falls within `RECENT_SYNC_WINDOW_MS` (5 min) for the adaptive page cap.
|
||||
// * `now_ms` — sanity check: must not return 0 and must be within a plausible
|
||||
// range so the adaptive window comparison never produces pathological results.
|
||||
// * early-stop guard: when `last_seen_id` matches the first page's head id
|
||||
// the sync loop breaks with `stop_reason = "head_unchanged"`. We pin the
|
||||
// helper logic that feeds this decision.
|
||||
|
||||
#[test]
|
||||
fn parse_cursor_to_epoch_secs_handles_epoch_millis() {
|
||||
// Gmail internalDate is epoch milliseconds as a numeric string.
|
||||
// 1774915200000 ms = 1774915200 s (2026-03-31 00:00:00 UTC).
|
||||
assert_eq!(
|
||||
parse_cursor_to_epoch_secs("1774915200000"),
|
||||
Some(1774915200)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_cursor_to_epoch_secs_handles_iso_date() {
|
||||
// YYYY-MM-DD date cursor produced by the older day-cursor write path.
|
||||
let secs = parse_cursor_to_epoch_secs("2024-01-15").unwrap();
|
||||
// 2024-01-15 00:00:00 UTC = 1705276800
|
||||
assert_eq!(secs, 1705276800);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_cursor_to_epoch_secs_handles_rfc3339() {
|
||||
let secs = parse_cursor_to_epoch_secs("2024-01-15T00:00:00Z").unwrap();
|
||||
assert_eq!(secs, 1705276800);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_cursor_to_epoch_secs_returns_none_for_garbage() {
|
||||
assert_eq!(parse_cursor_to_epoch_secs("not-a-timestamp"), None);
|
||||
assert_eq!(parse_cursor_to_epoch_secs(""), None);
|
||||
assert_eq!(parse_cursor_to_epoch_secs(" "), None);
|
||||
}
|
||||
|
||||
/// The adaptive page cap relies on `parse_cursor_to_epoch_secs` and `now_ms`
|
||||
/// agreeing on a common epoch so the "less than 5 min ago" comparison works.
|
||||
/// `now_ms()` must return epoch-milliseconds (not zero, not micros). If it
|
||||
/// returned microseconds, every sync would appear "recent" (delta < 300_000 ms
|
||||
/// vs delta actually being ~ 1e12 µs); if it returned seconds, every sync
|
||||
/// would appear "old" (delta > 300_000 ms trivially).
|
||||
#[test]
|
||||
fn now_ms_is_in_epoch_milliseconds_range() {
|
||||
let ms = now_ms();
|
||||
// Must be strictly positive.
|
||||
assert!(ms > 0, "now_ms must not return zero");
|
||||
// Must be > 2024-01-01 00:00:00 UTC in milliseconds so it's clearly
|
||||
// millisecond-epoch and not seconds-epoch (which would be ~1.7e9, much
|
||||
// smaller than 1.7e12).
|
||||
let jan_2024_ms: u64 = 1_704_067_200_000;
|
||||
assert!(
|
||||
ms > jan_2024_ms,
|
||||
"now_ms ({ms}) must be above 2024-01-01 in epoch-millisecond scale"
|
||||
);
|
||||
// Must be < year 2100 in milliseconds — rules out microseconds/nanoseconds.
|
||||
let year_2100_ms: u64 = 4_102_444_800_000;
|
||||
assert!(
|
||||
ms < year_2100_ms,
|
||||
"now_ms ({ms}) must be below year 2100 in epoch-millisecond scale"
|
||||
);
|
||||
}
|
||||
|
||||
/// The early-stop optimisation fires when `last_seen_id` equals the first
|
||||
/// message id on the first page. We test the helper that extracts message ids
|
||||
/// — `extract_messages` — to verify it correctly surfaces the `id` field so
|
||||
/// the comparison in the sync loop gets the right value.
|
||||
///
|
||||
/// The early-stop check uses `messages.first()` with the `MESSAGE_ID_PATHS`
|
||||
/// extractor. We can't call the private extractor, but we can pin
|
||||
/// `extract_messages` to return messages with their `id` intact so the
|
||||
/// sync loop can compare them to `state.last_seen_id`.
|
||||
#[test]
|
||||
fn extract_messages_preserves_id_field_for_early_stop() {
|
||||
// The early-stop check reads `m["id"]` via `extract_item_id`. Verify
|
||||
// `extract_messages` doesn't strip or transform the field.
|
||||
let v = json!({
|
||||
"data": {
|
||||
"messages": [
|
||||
{"id": "msg_abc", "internalDate": "1774915200000"},
|
||||
{"id": "msg_def", "internalDate": "1774915100000"}
|
||||
]
|
||||
},
|
||||
"successful": true
|
||||
});
|
||||
let msgs = extract_messages(&v);
|
||||
assert_eq!(msgs.len(), 2);
|
||||
assert_eq!(
|
||||
msgs[0]["id"], "msg_abc",
|
||||
"first message id must be preserved"
|
||||
);
|
||||
assert_eq!(
|
||||
msgs[1]["id"], "msg_def",
|
||||
"second message id must be preserved"
|
||||
);
|
||||
}
|
||||
|
||||
/// Variant: messages embedded in `data.data.messages` (deeper nesting
|
||||
/// seen in some Composio provider responses) — the extractor must still
|
||||
/// find them so the early-stop comparison has data to work with.
|
||||
#[test]
|
||||
fn extract_messages_handles_deep_nesting() {
|
||||
let v = json!({
|
||||
"data": {
|
||||
"data": {
|
||||
"messages": [
|
||||
{"id": "deep_msg_1"}
|
||||
]
|
||||
}
|
||||
}
|
||||
});
|
||||
let msgs = extract_messages(&v);
|
||||
assert_eq!(msgs.len(), 1);
|
||||
assert_eq!(msgs[0]["id"], "deep_msg_1");
|
||||
}
|
||||
|
||||
// Note: full `sync` / `fetch_user_profile` / `on_trigger` paths require a
|
||||
// live `ComposioClient` (HTTP) plus the global `MemoryClient` singleton.
|
||||
// Those go through the integration test suite. Here we just lock in
|
||||
// the provider's identity surface and helpers.
|
||||
|
||||
// ── Regression tests for issue #1713: sent-mail retrieval ───────────────────
|
||||
//
|
||||
// Before the fix the sync query was `in:inbox -in:spam -in:trash`, which meant
|
||||
// sent emails (label:SENT) were never fetched or ingested into the memory tree.
|
||||
// The fix removes `in:inbox` so both inbox and sent mail are fetched.
|
||||
|
||||
/// Guard: the provider source must NOT contain the `in:inbox` restriction.
|
||||
/// If this test fails, someone reintroduced the inbox-only query that caused
|
||||
/// issue #1713.
|
||||
#[test]
|
||||
fn provider_source_does_not_restrict_to_inbox() {
|
||||
let source = include_str!("provider.rs");
|
||||
// The old restriction started with `"in:inbox`. The double-quote is part
|
||||
// of the Rust string literal, so this catches the exact regression pattern.
|
||||
assert!(
|
||||
!source.contains("\"in:inbox"),
|
||||
"provider.rs sync query must NOT start with 'in:inbox' — this \
|
||||
restriction prevents sent emails from being ingested (issue #1713). \
|
||||
Use '-in:spam -in:trash' to allow both inbox and sent mail."
|
||||
"provider query must not exclude sent mail"
|
||||
);
|
||||
}
|
||||
|
||||
/// The base sync query must exclude spam and trash but NOT restrict to inbox.
|
||||
/// Asserts against the canonical `BASE_QUERY` constant from provider.rs so
|
||||
/// any change to the production value is caught immediately.
|
||||
#[test]
|
||||
fn sync_base_query_excludes_spam_and_trash_without_inbox_restriction() {
|
||||
assert!(
|
||||
BASE_QUERY.contains("-in:spam"),
|
||||
"BASE_QUERY must exclude spam (got: {BASE_QUERY:?})"
|
||||
);
|
||||
assert!(
|
||||
BASE_QUERY.contains("-in:trash"),
|
||||
"BASE_QUERY must exclude trash (got: {BASE_QUERY:?})"
|
||||
);
|
||||
assert!(
|
||||
!BASE_QUERY.contains("in:inbox"),
|
||||
"BASE_QUERY must NOT restrict to inbox — omitting this allows sent \
|
||||
mail to be fetched and ingested (issue #1713). Got: {BASE_QUERY:?}"
|
||||
);
|
||||
fn base_query_excludes_spam_and_trash_without_inbox_restriction() {
|
||||
assert!(BASE_QUERY.contains("-in:spam"));
|
||||
assert!(BASE_QUERY.contains("-in:trash"));
|
||||
assert!(!BASE_QUERY.contains("in:inbox"));
|
||||
}
|
||||
|
||||
/// Sent-mail query strings must be non-empty and must not restrict to inbox.
|
||||
/// Iterates `SENT_QUERIES` from provider.rs — the canonical list of query
|
||||
/// strings a user or agent can pass to GMAIL_FETCH_EMAILS for sent mail.
|
||||
#[test]
|
||||
fn sent_mail_query_strings_are_well_formed() {
|
||||
assert!(!SENT_QUERIES.is_empty(), "SENT_QUERIES must not be empty");
|
||||
for q in SENT_QUERIES {
|
||||
assert!(
|
||||
!q.is_empty(),
|
||||
"sent-mail query must not be empty, got empty string in SENT_QUERIES"
|
||||
);
|
||||
assert!(
|
||||
!q.starts_with("in:inbox"),
|
||||
"sent-mail query '{q}' must not restrict to inbox"
|
||||
);
|
||||
assert!(!SENT_QUERIES.is_empty());
|
||||
for query in SENT_QUERIES {
|
||||
assert!(!query.is_empty());
|
||||
assert!(!query.starts_with("in:inbox"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
//!
|
||||
//! Issue: #2400.
|
||||
|
||||
mod normalization;
|
||||
mod provider;
|
||||
mod sync;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
pub mod tools;
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
//! Linear sync helpers — result extraction, issue-title extraction,
|
||||
//! Linear host normalization helpers — result extraction, issue-title extraction,
|
||||
//! viewer identity, cursor extraction, and time utilities.
|
||||
//!
|
||||
//! Linear's GraphQL API (and therefore Composio's wrapping of it) returns
|
||||
@@ -21,7 +21,7 @@
|
||||
use async_trait::async_trait;
|
||||
use serde_json::json;
|
||||
|
||||
use super::sync;
|
||||
use super::normalization;
|
||||
use crate::openhuman::memory_sync::composio::providers::{
|
||||
merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider, CuratedTool,
|
||||
NormalizedTask, ProviderContext, ProviderUserProfile, TaskFetchFilter, TaskKind,
|
||||
@@ -86,7 +86,7 @@ impl ComposioProvider for LinearProvider {
|
||||
}
|
||||
|
||||
let data = &resp.data;
|
||||
let viewer = sync::extract_viewer(data);
|
||||
let viewer = normalization::extract_viewer(data);
|
||||
let viewer_ref = viewer.as_ref().unwrap_or(data);
|
||||
|
||||
let display_name = pick_str(viewer_ref, &["name", "data.name", "displayName"]);
|
||||
@@ -143,7 +143,7 @@ impl ComposioProvider for LinearProvider {
|
||||
resp.error.unwrap_or_else(|| "provider failure".into())
|
||||
));
|
||||
}
|
||||
let viewer_id = sync::extract_viewer_id(&resp.data).ok_or_else(|| {
|
||||
let viewer_id = normalization::extract_viewer_id(&resp.data).ok_or_else(|| {
|
||||
"[composio:linear] LINEAR_LIST_LINEAR_USERS returned no viewer id".to_string()
|
||||
})?;
|
||||
args["assigneeId"] = json!(viewer_id);
|
||||
@@ -176,7 +176,7 @@ impl ComposioProvider for LinearProvider {
|
||||
.filter(|s| !s.is_empty());
|
||||
|
||||
let mut out: Vec<NormalizedTask> = Vec::new();
|
||||
for issue in sync::extract_issues(&resp.data) {
|
||||
for issue in normalization::extract_issues(&resp.data) {
|
||||
if out.len() >= max {
|
||||
break;
|
||||
}
|
||||
@@ -203,8 +203,8 @@ impl ComposioProvider for LinearProvider {
|
||||
/// Map a raw Linear issue payload into a [`NormalizedTask`].
|
||||
fn normalize_linear_issue(issue: &serde_json::Value) -> Option<NormalizedTask> {
|
||||
let external_id = pick_str(issue, ISSUE_ID_PATHS)?;
|
||||
let title =
|
||||
sync::extract_issue_title(issue).unwrap_or_else(|| format!("Linear issue {external_id}"));
|
||||
let title = normalization::extract_issue_title(issue)
|
||||
.unwrap_or_else(|| format!("Linear issue {external_id}"));
|
||||
Some(NormalizedTask {
|
||||
external_id,
|
||||
source_id: String::new(),
|
||||
@@ -218,7 +218,7 @@ fn normalize_linear_issue(issue: &serde_json::Value) -> Option<NormalizedTask> {
|
||||
due: pick_str(issue, &["dueDate", "data.dueDate"]),
|
||||
labels: extract_linear_labels(issue),
|
||||
priority: pick_str(issue, &["priorityLabel", "data.priorityLabel"]),
|
||||
updated_at: sync::extract_issue_updated(issue),
|
||||
updated_at: normalization::extract_issue_updated(issue),
|
||||
raw: issue.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Unit tests for the Linear provider.
|
||||
|
||||
use super::sync::{
|
||||
use super::normalization::{
|
||||
extract_issue_title, extract_issue_updated, extract_issues, extract_pagination_cursor,
|
||||
extract_viewer, extract_viewer_id,
|
||||
};
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
mod normalization;
|
||||
mod provider;
|
||||
mod sync;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
pub mod tools;
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
//! Notion sync helpers — result extraction, pagination cursor,
|
||||
//! Notion host normalization helpers — result extraction, pagination cursor,
|
||||
//! page title extraction, and time utilities.
|
||||
|
||||
use serde_json::Value;
|
||||
@@ -18,7 +18,7 @@
|
||||
use async_trait::async_trait;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use super::sync;
|
||||
use super::normalization;
|
||||
use crate::openhuman::memory_sync::composio::providers::{
|
||||
first_array_str, merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider,
|
||||
CuratedTool, NormalizedTask, ProviderContext, ProviderUserProfile, TaskContainer,
|
||||
@@ -194,7 +194,7 @@ impl ComposioProvider for NotionProvider {
|
||||
.filter(|s| !s.is_empty());
|
||||
|
||||
let mut out: Vec<NormalizedTask> = Vec::new();
|
||||
for page in sync::extract_results(&resp.data) {
|
||||
for page in normalization::extract_results(&resp.data) {
|
||||
if out.len() >= max {
|
||||
break;
|
||||
}
|
||||
@@ -302,8 +302,8 @@ impl ComposioProvider for NotionProvider {
|
||||
/// preserved for enrichment.
|
||||
fn normalize_notion_page(page: &serde_json::Value) -> Option<NormalizedTask> {
|
||||
let external_id = pick_str(page, PAGE_ID_PATHS)?;
|
||||
let title =
|
||||
sync::extract_page_title(page).unwrap_or_else(|| format!("Notion page {external_id}"));
|
||||
let title = normalization::extract_page_title(page)
|
||||
.unwrap_or_else(|| format!("Notion page {external_id}"));
|
||||
Some(NormalizedTask {
|
||||
external_id,
|
||||
source_id: String::new(),
|
||||
@@ -358,7 +358,7 @@ fn normalize_notion_page(page: &serde_json::Value) -> Option<NormalizedTask> {
|
||||
/// "keep only object==database" check silently dropped every database.
|
||||
/// Pure (no I/O) so it is unit-testable.
|
||||
pub(super) fn parse_database_results(data: &serde_json::Value) -> Vec<TaskContainer> {
|
||||
let results = sync::extract_results(data);
|
||||
let results = normalization::extract_results(data);
|
||||
let mut kinds: std::collections::BTreeMap<String, usize> = std::collections::BTreeMap::new();
|
||||
let mut out: Vec<TaskContainer> = Vec::new();
|
||||
for item in &results {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Unit tests for the Notion provider.
|
||||
|
||||
use super::sync::{extract_notion_cursor, extract_page_title, extract_results};
|
||||
use super::normalization::{extract_notion_cursor, extract_page_title, extract_results};
|
||||
use super::NotionProvider;
|
||||
use crate::openhuman::memory_sync::composio::providers::ComposioProvider;
|
||||
use serde_json::json;
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
//! | Submodule | Source | Notes |
|
||||
//! | --- | --- | --- |
|
||||
//! | `folder` | Files under a user-added folder memory source | Watch + diff |
|
||||
//! | `harness` | Agent harness turns (memory_archivist's caller side) | Push-based |
|
||||
//! | `harness` | Agent harness turns (TinyCortex archivist caller side) | Push-based |
|
||||
//! | `dictation` | Local audio capture transcripts | Push-based |
|
||||
//!
|
||||
//! ## Status
|
||||
|
||||
@@ -5,16 +5,15 @@ from generic namespace memory and from `learning::tool_tracker` statistics.
|
||||
|
||||
## Namespace convention
|
||||
|
||||
Each tool gets its own namespace `tool-{tool_name}`. Build the string via
|
||||
[`types::tool_memory_namespace`] — never hard-code it.
|
||||
Each tool gets its own namespace `tool-{tool_name}`. Build the string via the
|
||||
`tool_memory_namespace` re-export — never hard-code it.
|
||||
|
||||
## Layout
|
||||
|
||||
| Path | Role |
|
||||
| --- | --- |
|
||||
| [`mod.rs`](mod.rs) | Module root + public re-exports. |
|
||||
| [`types.rs`](types.rs) | **Shim** — re-exports `ToolMemoryRule` / `ToolMemoryPriority` / `ToolMemorySource` / `tool_memory_namespace` from `tinycortex::memory::tool_memory::types` (W7). |
|
||||
| [`store.rs`](store.rs) | **Shim** — re-exports the crate `ToolMemoryStore` (`put_rule`, `get_rule`, `list_rules`, `delete_rule`, `rules_for_prompt`, `list_tool_names`, `record`, `list_rules_json`) + `tool_memory_store(Arc<dyn Memory>)`. The trait and engine are both crate-owned. |
|
||||
| [`mod.rs`](mod.rs) | Re-exports crate types/store and defines `tool_memory_store(Arc<dyn Memory>)`. |
|
||||
| [`capture.rs`](capture.rs) | `ToolMemoryCaptureHook` — `PostTurnHook` impl that captures user edicts and repeated tool failures into the store (host-retained). |
|
||||
| [`prompt.rs`](prompt.rs) | **Shim** — re-exports the crate `ToolMemoryRulesSection` + `render_tool_memory_rules` + `TOOL_MEMORY_HEADING`, and keeps the host `PromptSection` impl that plugs the section into the system-prompt builder. |
|
||||
| [`tools/`](tools/) | Agent-facing read/write tools: `MemoryToolsListTool` (list rules for a tool), `MemoryToolsPutTool` (upsert a rule). |
|
||||
@@ -35,6 +34,5 @@ The agent harness:
|
||||
- No upward dependencies — only `memory::Memory` trait (via `Arc<dyn Memory>`)
|
||||
and project-wide primitives (`tools::traits::Tool`, `serde_json`).
|
||||
- `MockMemory` is `#[cfg(test)]`-only — never available outside test builds.
|
||||
- Re-exports in `mod.rs` are the public surface; the underlying submodules
|
||||
are `pub` so test code can reach in but consumers should go through the
|
||||
re-exports.
|
||||
- Re-exports in `mod.rs` are the public surface. Crate-owned type and store
|
||||
forwarding files were removed; consumers should use the domain root.
|
||||
|
||||
@@ -33,8 +33,7 @@ use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
use super::store::{tool_memory_store, ToolMemoryStore};
|
||||
use super::types::{ToolMemoryPriority, ToolMemorySource};
|
||||
use super::{tool_memory_store, ToolMemoryPriority, ToolMemorySource, ToolMemoryStore};
|
||||
use crate::openhuman::agent::hooks::{PostTurnHook, ToolCallRecord, TurnContext};
|
||||
use crate::openhuman::memory::Memory;
|
||||
|
||||
@@ -291,8 +290,8 @@ fn tool_aliases(tool_name: &str) -> Vec<&'static str> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::openhuman::agent::hooks::ToolCallRecord;
|
||||
use crate::openhuman::memory_tools::store::tool_memory_store;
|
||||
use crate::openhuman::memory_tools::test_helpers::MockMemory;
|
||||
use crate::openhuman::memory_tools::tool_memory_store;
|
||||
|
||||
fn ctx_with(message: &str, tool_calls: Vec<ToolCallRecord>) -> TurnContext {
|
||||
TurnContext {
|
||||
|
||||
@@ -46,9 +46,7 @@ mod tests {
|
||||
use crate::openhuman::agent::prompts::types::{
|
||||
LearnedContextData, PromptContext, ToolCallFormat,
|
||||
};
|
||||
use crate::openhuman::memory_tools::types::{
|
||||
ToolMemoryPriority, ToolMemoryRule, ToolMemorySource,
|
||||
};
|
||||
use crate::openhuman::memory_tools::{ToolMemoryPriority, ToolMemoryRule, ToolMemorySource};
|
||||
|
||||
fn rule(tool: &str, body: &str, priority: ToolMemoryPriority) -> ToolMemoryRule {
|
||||
ToolMemoryRule {
|
||||
|
||||
@@ -1,16 +1,10 @@
|
||||
//! Storage layer for tool-scoped rules — thin host shim over
|
||||
//! `tinycortex::memory::tool_memory::store` (W7).
|
||||
//!
|
||||
//! The store engine (put / get / list / delete / prompt over an
|
||||
//! `Arc<dyn Memory>`) and the `Memory` trait are both owned by tinycortex.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::openhuman::memory::Memory;
|
||||
|
||||
pub use tinycortex::memory::tool_memory::store::{ToolMemoryStore, TOOL_MEMORY_PROMPT_CAP};
|
||||
|
||||
/// Build a crate [`ToolMemoryStore`] over the shared tinycortex memory trait.
|
||||
/// Build the crate-owned store over OpenHuman's shared memory object.
|
||||
pub fn tool_memory_store(memory: Arc<dyn Memory>) -> ToolMemoryStore {
|
||||
ToolMemoryStore::new(memory)
|
||||
}
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
//! Domain types for the tool-scoped memory layer — thin host re-export of
|
||||
//! `tinycortex::memory::tool_memory::types` (W7).
|
||||
//! `tinycortex::memory::tool_memory::types`.
|
||||
//!
|
||||
//! [`ToolMemoryRule`] / [`ToolMemoryPriority`] / [`ToolMemorySource`] and the
|
||||
//! [`tool_memory_namespace`] helper are the crate's (a byte-identical port,
|
||||
//! preserving the serde wire strings + `rule/{id}` storage keys). Host consumers
|
||||
//! keep their `memory_tools::types::*` import paths unchanged.
|
||||
//! This module preserves the public `memory_tools::types::*` import path while
|
||||
//! TinyCortex remains the single implementation and wire-format authority.
|
||||
|
||||
pub use tinycortex::memory::tool_memory::types::{
|
||||
tool_memory_namespace, ToolMemoryPriority, ToolMemoryRule, ToolMemorySource,
|
||||
|
||||
@@ -31,8 +31,7 @@ memory_store::trees (persistence: one Tree table, one schema)
|
||||
| [`tree/`](tree/) | `bucket_seal` (append leaf + cascade seal), `flush` (time-based partial seal), `registry` (kind-parameterized `get_or_create_tree` with UNIQUE-race recovery), `mod.rs` (re-exports + `memory_store::trees` shims for legacy paths). |
|
||||
| [`summarise.rs`](summarise.rs) | One function: produce the next-level summary text for a bucket. Wraps the chat model with a fixed prompt and token budget. |
|
||||
| [`retrieval/`](retrieval/) | Agent-facing tools. Read: `walk` (agentic), `drill_down`, `fetch_leaves`, `query_{source,global,topic}`, `search_entities`. Write: `ingest_document` (orchestrator-facing). |
|
||||
| [`score/`](score/) | Scoring signals, embedding (cloud/ollama/inert), entity extraction (regex/LLM), canonical resolver, entity index store. |
|
||||
| [`tools.rs`](tools.rs) | Re-exports from `memory::query` for backward compatibility. |
|
||||
| [`score/`](score/) | Product adapters over TinyCortex scoring and TinyAgents embedding models, plus entity extraction and the entity index store. |
|
||||
|
||||
## Layer rules
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ pub mod nlp;
|
||||
pub mod retrieval;
|
||||
pub mod score;
|
||||
pub mod summarise;
|
||||
pub mod tools;
|
||||
pub mod tree;
|
||||
pub mod tree_runtime;
|
||||
|
||||
|
||||
@@ -1,18 +1,17 @@
|
||||
# Memory tree — score embed (Phase 4 / #710)
|
||||
# Memory tree embedding bridge
|
||||
|
||||
Vector embedder for chunks and summaries. Produces a fixed-dimension (`EMBEDDING_DIM = 768`) `Vec<f32>` per text so retrieval can rerank candidates by semantic similarity. Default backend is local Ollama running `nomic-embed-text`; tests use the deterministic `InertEmbedder` so no network is required.
|
||||
The memory tree stores fixed 1024-dimensional vectors. Concrete provider
|
||||
transports live in TinyAgents; this directory keeps only memory-tree policy and
|
||||
compatibility:
|
||||
|
||||
## Public surface
|
||||
- `factory.rs`: read/write provider resolution, cloud-session policy, and
|
||||
degraded-state behavior;
|
||||
- `openai_compat.rs`: OpenHuman config/credential and custom-slug resolution;
|
||||
- `inert.rs`: deterministic 1024-element zero vectors for opt-out/tests;
|
||||
- `mod.rs`: the legacy `Embedder` contract, `ProviderEmbedder` bridge, batch
|
||||
fallback/dimension checks, cosine math, and SQLite f32 packing helpers.
|
||||
|
||||
- `pub trait Embedder` — `mod.rs` — `embed(text) -> Vec<f32>` contract; impls must return exactly `EMBEDDING_DIM` floats.
|
||||
- `pub fn build_embedder_from_config` — `factory.rs` — returns `OllamaEmbedder` when configured, otherwise `InertEmbedder` (or bails when `embedding_strict = true`).
|
||||
- `pub struct OllamaEmbedder` — `ollama.rs` — HTTP client posting to `{endpoint}/api/embeddings`.
|
||||
- `pub struct InertEmbedder` — `inert.rs` — zero-vector embedder for tests.
|
||||
- `pub fn cosine_similarity` / `pack_embedding` / `unpack_embedding` / `pack_checked` / `decode_optional_blob` — `mod.rs` — math + SQLite BLOB packing helpers.
|
||||
|
||||
## Files
|
||||
|
||||
- `mod.rs` — trait, `EMBEDDING_DIM`, math + pack/unpack helpers, write-time / read-time semantics.
|
||||
- `factory.rs` — `Config::memory_tree`-driven embedder selection with `embedding_strict` opt-in.
|
||||
- `ollama.rs` — Ollama `/api/embeddings` client; defaults at `http://localhost:11434` / `nomic-embed-text` / 10s timeout.
|
||||
- `inert.rs` — zero-vector embedder; cosine similarity between any two inert vectors is 0.0 (zero-magnitude short-circuit), so retrieval tests that need real reranking should hand-stitch embeddings instead of relying on this path.
|
||||
Ollama uses TinyAgents `OllamaEmbeddingModel` and `/api/embed`, with the shared
|
||||
8192-token context and batch window. Managed cloud uses the host credential and
|
||||
privacy wrapper around TinyAgents `CloudEmbeddingModel`. Do not add provider
|
||||
HTTP clients in this directory.
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
//! Cloud (Voyage-backed) embedder for the memory tree.
|
||||
//!
|
||||
//! Adapts the OpenHuman backend's `POST /openai/v1/embeddings` surface
|
||||
//! (Voyage `voyage-3.5`, 1024 dims) to the memory_tree [`Embedder`] trait
|
||||
//! so Phase 4 ingest / bucket-seal can vectorize chunks without a local
|
||||
//! Ollama install.
|
||||
//!
|
||||
//! The 1024-dim output matches existing on-disk blobs (which were
|
||||
//! produced by `bge-m3`, also 1024-dim), so this is a drop-in replacement
|
||||
//! for the Ollama path — no migration of `mem_tree_chunks.embedding`
|
||||
//! required.
|
||||
//!
|
||||
//! Auth: the cloud embedder resolves the session JWT per call via
|
||||
//! [`OpenHumanCloudEmbedding`], so a session refresh between batches is
|
||||
//! picked up transparently. When the user is unauthenticated the first
|
||||
//! `embed()` returns an error; ingest treats that the same as any other
|
||||
//! embedder failure (don't persist the row, let job retry).
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use async_trait::async_trait;
|
||||
|
||||
use super::{Embedder, EMBEDDING_DIM};
|
||||
use crate::openhuman::config::Config;
|
||||
use crate::openhuman::embeddings::cloud::{
|
||||
OpenHumanCloudEmbedding, DEFAULT_CLOUD_EMBEDDING_DIMENSIONS, DEFAULT_CLOUD_EMBEDDING_MODEL,
|
||||
};
|
||||
use crate::openhuman::embeddings::EmbeddingProvider;
|
||||
|
||||
/// Cloud-backed memory_tree embedder.
|
||||
///
|
||||
/// Wraps [`OpenHumanCloudEmbedding`] (which speaks the OpenAI-compatible
|
||||
/// `/openai/v1/embeddings` shape backed by Voyage on the OpenHuman
|
||||
/// backend) and adapts it to the memory_tree [`Embedder`] trait.
|
||||
pub struct CloudEmbedder {
|
||||
inner: OpenHumanCloudEmbedding,
|
||||
}
|
||||
|
||||
impl CloudEmbedder {
|
||||
/// Build a cloud embedder using the same backend resolution as the
|
||||
/// main embeddings path: `api_url` falls back to
|
||||
/// [`effective_api_url`](crate::api::config::effective_api_url) and
|
||||
/// the workspace dir comes from `config.workspace_dir` so the auth
|
||||
/// service finds the user's session JWT.
|
||||
pub fn new(config: &Config) -> Self {
|
||||
let openhuman_dir = config.config_path.parent().map(std::path::PathBuf::from);
|
||||
Self {
|
||||
inner: OpenHumanCloudEmbedding::new(
|
||||
None,
|
||||
openhuman_dir,
|
||||
config.secrets.encrypt,
|
||||
DEFAULT_CLOUD_EMBEDDING_MODEL,
|
||||
DEFAULT_CLOUD_EMBEDDING_DIMENSIONS,
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Embedder for CloudEmbedder {
|
||||
fn name(&self) -> &'static str {
|
||||
"cloud"
|
||||
}
|
||||
|
||||
async fn embed(&self, text: &str) -> Result<Vec<f32>> {
|
||||
let v = self
|
||||
.inner
|
||||
.embed_one(text)
|
||||
.await
|
||||
.context("cloud embeddings failed")?;
|
||||
if v.len() != EMBEDDING_DIM {
|
||||
anyhow::bail!(
|
||||
"cloud embedder returned {} dims, expected {}",
|
||||
v.len(),
|
||||
EMBEDDING_DIM
|
||||
);
|
||||
}
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
/// Collapse N per-text round-trips into a single batched Voyage request by
|
||||
/// delegating to the inner provider's native batch `embed`. Falls back to
|
||||
/// per-text embedding (preserving per-position error attribution) on a
|
||||
/// whole-batch failure or a length mismatch.
|
||||
async fn embed_batch(&self, texts: &[&str]) -> Vec<Result<Vec<f32>>> {
|
||||
super::embed_batch_via_provider(&self.inner, "cloud", texts).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::openhuman::config::Config;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn test_config() -> (TempDir, Config) {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let mut cfg = Config::default();
|
||||
cfg.workspace_dir = tmp.path().to_path_buf();
|
||||
cfg.config_path = tmp.path().join("config.toml");
|
||||
(tmp, cfg)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn name_is_cloud() {
|
||||
let (_tmp, cfg) = test_config();
|
||||
let e = CloudEmbedder::new(&cfg);
|
||||
assert_eq!(e.name(), "cloud");
|
||||
}
|
||||
}
|
||||
@@ -29,11 +29,14 @@
|
||||
//! - `OPENHUMAN_MEMORY_EMBED_MODEL`
|
||||
//! - `OPENHUMAN_MEMORY_EMBED_TIMEOUT_MS`
|
||||
|
||||
use anyhow::Result;
|
||||
use anyhow::{Context, Result};
|
||||
|
||||
use super::{CloudEmbedder, Embedder, InertEmbedder, OllamaEmbedder};
|
||||
use std::time::Duration;
|
||||
|
||||
use super::{Embedder, InertEmbedder, ProviderEmbedder, EMBEDDING_DIM};
|
||||
use crate::openhuman::config::Config;
|
||||
use crate::openhuman::inference::local::ollama_base_url;
|
||||
use tinyagents::harness::embeddings::{OllamaEmbeddingModel, RECOMMENDED_OLLAMA_CONTEXT_TOKENS};
|
||||
|
||||
/// Cheap heuristic for "is a backend session reachable?" — the cloud
|
||||
/// embedder needs one and bails on first embed call without it. We use
|
||||
@@ -69,7 +72,7 @@ pub fn build_embedder_from_config(config: &Config) -> Result<Box<dyn Embedder>>
|
||||
log::debug!(
|
||||
"[memory_tree::embed::factory] read → Ollama endpoint={endpoint} model={model} timeout_ms={timeout_ms}"
|
||||
);
|
||||
Box::new(OllamaEmbedder::new(endpoint, model, timeout_ms))
|
||||
Box::new(build_ollama_embedder(&endpoint, &model, timeout_ms)?)
|
||||
}
|
||||
EmbedderChoice::OptOut => {
|
||||
log::info!(
|
||||
@@ -90,7 +93,7 @@ pub fn build_embedder_from_config(config: &Config) -> Result<Box<dyn Embedder>>
|
||||
"[memory_tree::embed::factory] read → cloud (Voyage) — flip \
|
||||
'Memory embeddings' in Local AI Settings to switch to local"
|
||||
);
|
||||
Box::new(CloudEmbedder::new(config))
|
||||
Box::new(build_cloud_embedder(config))
|
||||
}
|
||||
EmbedderChoice::NoProvider => {
|
||||
log::warn!(
|
||||
@@ -215,7 +218,9 @@ pub fn build_write_embedder(config: &Config) -> Result<Option<Box<dyn Embedder>>
|
||||
timeout_ms,
|
||||
} => {
|
||||
clear_semantic_recall_degraded();
|
||||
Some(Box::new(OllamaEmbedder::new(endpoint, model, timeout_ms)))
|
||||
Some(Box::new(build_ollama_embedder(
|
||||
&endpoint, &model, timeout_ms,
|
||||
)?))
|
||||
}
|
||||
EmbedderChoice::OptOut => {
|
||||
clear_semantic_recall_degraded();
|
||||
@@ -231,7 +236,7 @@ pub fn build_write_embedder(config: &Config) -> Result<Option<Box<dyn Embedder>>
|
||||
}
|
||||
EmbedderChoice::Cloud => {
|
||||
clear_semantic_recall_degraded();
|
||||
Some(Box::new(CloudEmbedder::new(config)))
|
||||
Some(Box::new(build_cloud_embedder(config)))
|
||||
}
|
||||
EmbedderChoice::NoProvider => {
|
||||
log::warn!(
|
||||
@@ -245,6 +250,36 @@ pub fn build_write_embedder(config: &Config) -> Result<Option<Box<dyn Embedder>>
|
||||
})
|
||||
}
|
||||
|
||||
fn build_ollama_embedder(endpoint: &str, model: &str, timeout_ms: u64) -> Result<ProviderEmbedder> {
|
||||
let timeout = Duration::from_millis(if timeout_ms == 0 { 10_000 } else { timeout_ms });
|
||||
let client = reqwest::Client::builder()
|
||||
.connect_timeout(timeout)
|
||||
.build()
|
||||
.context("build Ollama embeddings HTTP client")?;
|
||||
let model = OllamaEmbeddingModel::try_new(endpoint, model, EMBEDDING_DIM)?
|
||||
.with_context_options(
|
||||
RECOMMENDED_OLLAMA_CONTEXT_TOKENS,
|
||||
RECOMMENDED_OLLAMA_CONTEXT_TOKENS,
|
||||
)
|
||||
.with_client(client);
|
||||
Ok(ProviderEmbedder::new(
|
||||
crate::openhuman::embeddings::TinyAgentsEmbeddingProvider::boxed(model),
|
||||
"ollama",
|
||||
))
|
||||
}
|
||||
|
||||
fn build_cloud_embedder(config: &Config) -> ProviderEmbedder {
|
||||
let openhuman_dir = config.config_path.parent().map(std::path::PathBuf::from);
|
||||
let provider = crate::openhuman::embeddings::cloud::OpenHumanCloudEmbedding::new(
|
||||
None,
|
||||
openhuman_dir,
|
||||
config.secrets.encrypt,
|
||||
crate::openhuman::embeddings::cloud::DEFAULT_CLOUD_EMBEDDING_MODEL,
|
||||
crate::openhuman::embeddings::cloud::DEFAULT_CLOUD_EMBEDDING_DIMENSIONS,
|
||||
);
|
||||
ProviderEmbedder::new(Box::new(provider), "cloud")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -29,16 +29,12 @@
|
||||
use anyhow::{Context, Result};
|
||||
use async_trait::async_trait;
|
||||
|
||||
pub mod cloud;
|
||||
pub mod factory;
|
||||
pub mod inert;
|
||||
pub mod ollama;
|
||||
pub mod openai_compat;
|
||||
|
||||
pub use cloud::CloudEmbedder;
|
||||
pub use factory::{build_embedder_from_config, build_write_embedder};
|
||||
pub use inert::InertEmbedder;
|
||||
pub use ollama::OllamaEmbedder;
|
||||
pub use openai_compat::OpenAiCompatEmbedder;
|
||||
|
||||
/// Embedding dimensionality used across the memory tree.
|
||||
@@ -89,6 +85,43 @@ pub trait Embedder: Send + Sync {
|
||||
}
|
||||
}
|
||||
|
||||
/// Adapts the canonical host embedding-provider contract to the legacy
|
||||
/// memory-tree embedder shape. Concrete network implementations live in
|
||||
/// `tinyagents::harness::embeddings`; this bridge owns only dimension checks
|
||||
/// and the memory tree's per-position batch fallback contract.
|
||||
pub struct ProviderEmbedder {
|
||||
inner: Box<dyn crate::openhuman::embeddings::EmbeddingProvider>,
|
||||
label: &'static str,
|
||||
}
|
||||
|
||||
impl ProviderEmbedder {
|
||||
pub fn new(
|
||||
inner: Box<dyn crate::openhuman::embeddings::EmbeddingProvider>,
|
||||
label: &'static str,
|
||||
) -> Self {
|
||||
Self { inner, label }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Embedder for ProviderEmbedder {
|
||||
fn name(&self) -> &'static str {
|
||||
self.label
|
||||
}
|
||||
|
||||
async fn embed(&self, text: &str) -> Result<Vec<f32>> {
|
||||
self.inner
|
||||
.embed_one(text)
|
||||
.await
|
||||
.with_context(|| format!("{} embeddings failed", self.label))
|
||||
.and_then(|vector| check_embed_dim(vector, self.label))
|
||||
}
|
||||
|
||||
async fn embed_batch(&self, texts: &[&str]) -> Vec<Result<Vec<f32>>> {
|
||||
embed_batch_via_provider(self.inner.as_ref(), self.label, texts).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate that a freshly-produced embedding has exactly [`EMBEDDING_DIM`]
|
||||
/// floats, returning a labelled error otherwise. Shared by the per-text and
|
||||
/// batched provider adapters so the "wrong dims" diagnostic is identical
|
||||
|
||||
@@ -1,384 +0,0 @@
|
||||
//! Ollama-backed embedder for Phase 4 (#710).
|
||||
//!
|
||||
//! Posts `{model, prompt, options: {num_ctx}}` to
|
||||
//! `{endpoint}/api/embeddings` and expects
|
||||
//! `{"embedding": [f32; EMBEDDING_DIM]}` back. Designed for a local
|
||||
//! `ollama serve` hosting `bge-m3`.
|
||||
//!
|
||||
//! This is intentionally a tiny HTTP client — no retry, no pool caching,
|
||||
//! no streaming. Phase 4 wants the simplest thing that works so we can
|
||||
//! land embedding end-to-end and iterate once baseline retrieval quality
|
||||
//! is measurable. Timeouts, parallelism, and caching are explicit
|
||||
//! follow-ups.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{Embedder, EMBEDDING_DIM};
|
||||
|
||||
/// Default Ollama endpoint. Matches the local-install default from the
|
||||
/// `local_ai` subsystem and the Ollama defaults.
|
||||
pub const DEFAULT_ENDPOINT: &str = "http://localhost:11434";
|
||||
|
||||
/// Default embedding model — must output exactly [`EMBEDDING_DIM`]
|
||||
/// (1024) dims. `bge-m3` is a multilingual BERT-family encoder with
|
||||
/// native 8192-token context and 1024-dim output.
|
||||
pub const DEFAULT_MODEL: &str = "bge-m3";
|
||||
|
||||
/// Default request timeout. Ollama's first-use latency is a few hundred
|
||||
/// ms on a warm model; 10s absorbs a cold-model load on commodity
|
||||
/// hardware without stalling ingest on a broken backend.
|
||||
pub const DEFAULT_TIMEOUT_MS: u64 = 10_000;
|
||||
|
||||
/// HTTP client wrapping a single Ollama endpoint + model pair.
|
||||
///
|
||||
/// Cloneable — `reqwest::Client` shares a connection pool under the hood
|
||||
/// so cloning the wrapper stays cheap across seal / ingest call sites.
|
||||
#[derive(Clone)]
|
||||
pub struct OllamaEmbedder {
|
||||
endpoint: String,
|
||||
model: String,
|
||||
timeout: Duration,
|
||||
client: reqwest::Client,
|
||||
}
|
||||
|
||||
impl OllamaEmbedder {
|
||||
/// Build a new embedder. `endpoint` is trimmed of trailing slashes
|
||||
/// so callers don't have to worry about mixing `http://host` and
|
||||
/// `http://host/`. Empty values fall back to the public defaults.
|
||||
pub fn new(endpoint: String, model: String, timeout_ms: u64) -> Self {
|
||||
let endpoint = if endpoint.trim().is_empty() {
|
||||
DEFAULT_ENDPOINT.to_string()
|
||||
} else {
|
||||
endpoint.trim().trim_end_matches('/').to_string()
|
||||
};
|
||||
let model = if model.trim().is_empty() {
|
||||
DEFAULT_MODEL.to_string()
|
||||
} else {
|
||||
model.trim().to_string()
|
||||
};
|
||||
let timeout_ms = if timeout_ms == 0 {
|
||||
DEFAULT_TIMEOUT_MS
|
||||
} else {
|
||||
timeout_ms
|
||||
};
|
||||
let timeout = Duration::from_millis(timeout_ms);
|
||||
// No body-read timeout. Ollama is local — slow responses mean
|
||||
// the model is genuinely processing, not that the network
|
||||
// broke. A body-read timeout here would cancel mid-stream and
|
||||
// force retries against the same slow model. `timeout` is
|
||||
// repurposed as the TCP connect timeout (fast-fail when
|
||||
// Ollama is actually down).
|
||||
let client = reqwest::Client::builder()
|
||||
.connect_timeout(timeout)
|
||||
.build()
|
||||
.unwrap_or_else(|e| {
|
||||
log::warn!(
|
||||
"[memory_tree::embed::ollama] failed to build client \
|
||||
with connect_timeout — using default: {e}"
|
||||
);
|
||||
reqwest::Client::new()
|
||||
});
|
||||
log::debug!(
|
||||
"[memory_tree::embed::ollama] created endpoint={endpoint} \
|
||||
model={model} timeout_ms={timeout_ms}"
|
||||
);
|
||||
Self {
|
||||
endpoint,
|
||||
model,
|
||||
timeout,
|
||||
client,
|
||||
}
|
||||
}
|
||||
|
||||
/// Convenience constructor using all defaults ([`DEFAULT_ENDPOINT`],
|
||||
/// [`DEFAULT_MODEL`], [`DEFAULT_TIMEOUT_MS`]).
|
||||
pub fn default_new() -> Self {
|
||||
Self::new(String::new(), String::new(), 0)
|
||||
}
|
||||
|
||||
fn embed_url(&self) -> String {
|
||||
format!("{}/api/embeddings", self.endpoint)
|
||||
}
|
||||
}
|
||||
|
||||
fn is_missing_model_response(status: reqwest::StatusCode, body: &str) -> bool {
|
||||
if status != reqwest::StatusCode::NOT_FOUND {
|
||||
return false;
|
||||
}
|
||||
|
||||
let normalized = body.to_ascii_lowercase();
|
||||
normalized.contains("model") && normalized.contains("not found")
|
||||
}
|
||||
|
||||
fn format_embedding_status_error(
|
||||
status: reqwest::StatusCode,
|
||||
body: &str,
|
||||
endpoint: &str,
|
||||
model: &str,
|
||||
) -> String {
|
||||
let trimmed_body = body.trim();
|
||||
|
||||
if is_missing_model_response(status, trimmed_body) {
|
||||
return format!(
|
||||
"Ollama embedding model `{model}` is not installed at {endpoint}. \
|
||||
Run `ollama pull {model}` or choose an installed embedding model in \
|
||||
Connections → API keys → Embeddings. status={status} body={trimmed_body}"
|
||||
);
|
||||
}
|
||||
|
||||
format!("ollama embeddings failed status={status} body={trimmed_body}")
|
||||
}
|
||||
|
||||
/// Override Ollama's per-model `num_ctx` default. Ollama loads
|
||||
/// embedding models with `num_ctx = 4096` (or whatever default the
|
||||
/// model's modelfile carries) unless the request explicitly asks for
|
||||
/// more. `bge-m3` natively supports 8192 tokens, and chunker-token
|
||||
/// counts undercount BERT-WordPiece tokens by ~1.5-2× for HTML-derived
|
||||
/// markdown — so a 1500-chunker-token chunk routinely produces 2500+
|
||||
/// real tokens at embed time. Asking for 8192 unconditionally avoids
|
||||
/// silent prompt truncation; on models that natively support less,
|
||||
/// Ollama clamps `num_ctx` to the model's actual maximum, so this is
|
||||
/// safe to over-request.
|
||||
///
|
||||
/// This is also the **single source of truth for the memory layer's
|
||||
/// minimum model context window**: a local model whose native context is
|
||||
/// below this floor silently truncates chunks/summaries and corrupts
|
||||
/// recall. `inference::local::model_requirements::MIN_CONTEXT_TOKENS`
|
||||
/// re-exports this value so the model-acceptance gate can never drift
|
||||
/// from what the embedder actually requests.
|
||||
pub(crate) const EMBED_NUM_CTX: u32 = 8192;
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct EmbedRequest<'a> {
|
||||
model: &'a str,
|
||||
prompt: &'a str,
|
||||
options: EmbedOptions,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct EmbedOptions {
|
||||
num_ctx: u32,
|
||||
// Ollama/llama.cpp evaluates an embedding prompt in a SINGLE batch and
|
||||
// rejects (HTTP 500 "input length exceeds the context length") any prompt
|
||||
// longer than `n_batch` — whose default is 2048, far below `num_ctx`.
|
||||
// Raising `num_ctx` alone does NOT lift this; `num_batch` must match, or
|
||||
// chunks between ~2k and 8k real tokens fail to embed even though they fit
|
||||
// the context window. Set equal to `num_ctx` so the batch limit never bites
|
||||
// before the real context limit.
|
||||
num_batch: u32,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct EmbedResponse {
|
||||
#[serde(default)]
|
||||
embedding: Vec<f32>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Embedder for OllamaEmbedder {
|
||||
fn name(&self) -> &'static str {
|
||||
"ollama"
|
||||
}
|
||||
|
||||
async fn embed(&self, text: &str) -> Result<Vec<f32>> {
|
||||
log::debug!(
|
||||
"[memory_tree::embed::ollama] embed endpoint={} model={} bytes={}",
|
||||
self.endpoint,
|
||||
self.model,
|
||||
text.len()
|
||||
);
|
||||
let req = EmbedRequest {
|
||||
model: &self.model,
|
||||
prompt: text,
|
||||
options: EmbedOptions {
|
||||
num_ctx: EMBED_NUM_CTX,
|
||||
num_batch: EMBED_NUM_CTX,
|
||||
},
|
||||
};
|
||||
let resp = self
|
||||
.client
|
||||
// No per-request body-read timeout — see `OllamaEmbedder::new`
|
||||
// for rationale. The Client's `connect_timeout` still applies
|
||||
// and fails fast if Ollama isn't reachable.
|
||||
.post(self.embed_url())
|
||||
.json(&req)
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| {
|
||||
format!(
|
||||
"ollama embeddings request failed (is Ollama running at {}?)",
|
||||
self.endpoint
|
||||
)
|
||||
})?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
let msg = format_embedding_status_error(status, &body, &self.endpoint, &self.model);
|
||||
// Log at WARN so missing-model failures surface in traces without
|
||||
// requiring debug-level logging to be enabled. Missing-model 404s
|
||||
// include the `ollama pull` remediation hint from
|
||||
// `format_embedding_status_error`.
|
||||
log::warn!("[embeddings] {msg}");
|
||||
anyhow::bail!(msg);
|
||||
}
|
||||
|
||||
let payload: EmbedResponse = resp
|
||||
.json()
|
||||
.await
|
||||
.context("ollama embeddings response parse failed")?;
|
||||
|
||||
if payload.embedding.len() != EMBEDDING_DIM {
|
||||
anyhow::bail!(
|
||||
"ollama embeddings returned {} dims, expected {EMBEDDING_DIM} \
|
||||
(check model={})",
|
||||
payload.embedding.len(),
|
||||
self.model
|
||||
);
|
||||
}
|
||||
|
||||
Ok(payload.embedding)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use axum::{extract::Json, http::StatusCode, routing::post, Router};
|
||||
use std::net::SocketAddr;
|
||||
|
||||
/// Spin up a local axum server and return its base URL.
|
||||
async fn start_mock(app: Router) -> String {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr: SocketAddr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
format!("http://127.0.0.1:{}", addr.port())
|
||||
}
|
||||
|
||||
fn fixed_vec(val: f32) -> Vec<f32> {
|
||||
vec![val; EMBEDDING_DIM]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn defaults_applied_on_empty_input() {
|
||||
let e = OllamaEmbedder::new(String::new(), String::new(), 0);
|
||||
assert_eq!(e.endpoint, DEFAULT_ENDPOINT);
|
||||
assert_eq!(e.model, DEFAULT_MODEL);
|
||||
assert_eq!(e.timeout, Duration::from_millis(DEFAULT_TIMEOUT_MS));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trailing_slash_trimmed() {
|
||||
let e = OllamaEmbedder::new("http://host:1234/".into(), "m".into(), 1000);
|
||||
assert_eq!(e.endpoint, "http://host:1234");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn embed_url_format() {
|
||||
let e = OllamaEmbedder::default_new();
|
||||
assert_eq!(e.embed_url(), "http://localhost:11434/api/embeddings");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn name_is_ollama() {
|
||||
assert_eq!(OllamaEmbedder::default_new().name(), "ollama");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn happy_path_returns_embedding() {
|
||||
let v = fixed_vec(0.25);
|
||||
let v_clone = v.clone();
|
||||
let app = Router::new().route(
|
||||
"/api/embeddings",
|
||||
post(move |Json(body): Json<serde_json::Value>| {
|
||||
let v = v_clone.clone();
|
||||
async move {
|
||||
assert_eq!(body["model"], "bge-m3");
|
||||
assert_eq!(body["prompt"], "hello world");
|
||||
assert_eq!(body["options"]["num_ctx"], 8192);
|
||||
assert_eq!(body["options"]["num_batch"], 8192);
|
||||
Json(serde_json::json!({ "embedding": v }))
|
||||
}
|
||||
}),
|
||||
);
|
||||
let url = start_mock(app).await;
|
||||
let e = OllamaEmbedder::new(url, String::new(), 0);
|
||||
let out = e.embed("hello world").await.unwrap();
|
||||
assert_eq!(out.len(), EMBEDDING_DIM);
|
||||
assert!((out[0] - 0.25).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn server_error_bubbles_up() {
|
||||
let app = Router::new().route(
|
||||
"/api/embeddings",
|
||||
post(|| async { (StatusCode::INTERNAL_SERVER_ERROR, "model crashed") }),
|
||||
);
|
||||
let url = start_mock(app).await;
|
||||
let e = OllamaEmbedder::new(url, String::new(), 0);
|
||||
let err = e.embed("hello").await.unwrap_err().to_string();
|
||||
assert!(err.contains("500"), "msg: {err}");
|
||||
assert!(err.contains("model crashed"), "msg: {err}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_model_404_mentions_pull_command() {
|
||||
let app = Router::new().route(
|
||||
"/api/embeddings",
|
||||
post(|| async { (StatusCode::NOT_FOUND, "{\"error\":\"model not found\"}") }),
|
||||
);
|
||||
let url = start_mock(app).await;
|
||||
let e = OllamaEmbedder::new(url, "custom-embed".into(), 0);
|
||||
let err = e.embed("hello").await.unwrap_err().to_string();
|
||||
|
||||
assert!(
|
||||
err.contains("embedding model `custom-embed` is not installed"),
|
||||
"msg: {err}"
|
||||
);
|
||||
assert!(err.contains("ollama pull custom-embed"), "msg: {err}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dim_mismatch_rejected() {
|
||||
let app = Router::new().route(
|
||||
"/api/embeddings",
|
||||
post(|| async {
|
||||
// Return a 3-dim vector — must fail validation.
|
||||
Json(serde_json::json!({ "embedding": [0.1, 0.2, 0.3] }))
|
||||
}),
|
||||
);
|
||||
let url = start_mock(app).await;
|
||||
let e = OllamaEmbedder::new(url, String::new(), 0);
|
||||
let err = e.embed("hi").await.unwrap_err().to_string();
|
||||
assert!(err.contains("3 dims"), "msg: {err}");
|
||||
assert!(err.contains("expected 1024"), "msg: {err}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn malformed_json_response_rejected() {
|
||||
let app = Router::new().route(
|
||||
"/api/embeddings",
|
||||
post(|| async { (StatusCode::OK, "not even json") }),
|
||||
);
|
||||
let url = start_mock(app).await;
|
||||
let e = OllamaEmbedder::new(url, String::new(), 0);
|
||||
let err = e.embed("hi").await.unwrap_err().to_string();
|
||||
assert!(err.contains("parse failed"), "msg: {err}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn request_failure_is_descriptive() {
|
||||
let e = OllamaEmbedder::new("http://%".into(), String::new(), 500);
|
||||
let err = e.embed("hi").await.unwrap_err().to_string();
|
||||
assert!(
|
||||
err.contains("is Ollama running"),
|
||||
"should mention Ollama: {err}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
//! Compatibility shim: high-level memory query tools now live in
|
||||
//! `crate::openhuman::memory::query`.
|
||||
|
||||
pub use crate::openhuman::memory::query::*;
|
||||
@@ -84,7 +84,6 @@ pub mod medulla_local;
|
||||
pub mod meet;
|
||||
pub mod meet_agent;
|
||||
pub mod memory;
|
||||
pub mod memory_archivist;
|
||||
pub mod memory_conversations;
|
||||
pub mod memory_diff;
|
||||
pub mod memory_goals;
|
||||
|
||||
@@ -24,7 +24,7 @@ use crate::openhuman::scheduler_gate::signals::Signals;
|
||||
/// simultaneous Ollama requests have crashed the user's laptop twice.
|
||||
///
|
||||
/// Cloud-backend LLM calls bypass this semaphore at the worker layer
|
||||
/// (see `memory::jobs::worker::run_once`) because they're
|
||||
/// (see `memory_queue::worker::run_once`) because they're
|
||||
/// bandwidth-bound, not RAM-bound, and the worker pool itself bounds
|
||||
/// concurrency upstream. Keeping this at 1 preserves the laptop-RAM
|
||||
/// contract regardless of backend.
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
//! chunks / tree / retrieval / queue / ingest / score + the long tail). This
|
||||
//! module is the **adapter seam**, mirroring `src/openhuman/tinyagents/`: it
|
||||
//! implements the crate's engine traits over OpenHuman services and derives the
|
||||
//! engine's [`MemoryConfig`] from the host [`Config`]. Nothing here contains
|
||||
//! engine's [`tinycortex::memory::MemoryConfig`] from the host [`Config`]. Nothing here contains
|
||||
//! engine logic — that lives in the crate.
|
||||
//!
|
||||
//! ## Ownership boundary (the seam contract)
|
||||
@@ -16,19 +16,20 @@
|
||||
//! archivist/tool-memory/conversations long tail.
|
||||
//!
|
||||
//! **Product (host, stays in OpenHuman):** JSON-RPC schemas/ops/read_rpc, agent
|
||||
//! tools + `SecurityPolicy` gating, live sync (`memory_sync`), the event bus,
|
||||
//! preferences, `source_scope` per-turn allowlist, redaction, the global
|
||||
//! singleton + background queue worker, embedding/LLM **compute**, and the
|
||||
//! tools + `SecurityPolicy` gating, sync scheduling/credentials/events, the
|
||||
//! event bus, preferences, `source_scope` per-turn allowlist, redaction, the
|
||||
//! global singleton + background queue worker, embedding/LLM **compute**, and the
|
||||
//! host-retained `UnifiedMemory` namespace-document tier (episodic/event/
|
||||
//! segment/doc/graph/profile tables) plus the `wiki_git`/`obsidian` content
|
||||
//! surfaces the crate deliberately excludes.
|
||||
//!
|
||||
//! The crate never makes a network call or owns a worker pool: LLM/embedding
|
||||
//! compute is injected through `EmbeddingBackend`, `ChatProvider`, `Summariser`,
|
||||
//! and `EntityExtractor`; the job queue is driven by the host worker loop via
|
||||
//! Network-capable sync providers are feature-gated in the crate; their
|
||||
//! credentials and product policy stay in the host. LLM/embedding compute is
|
||||
//! injected through `EmbeddingBackend`, `ChatProvider`, `Summariser`, and
|
||||
//! `EntityExtractor`; the job queue is driven by the host worker loop via
|
||||
//! `queue::run_once` / `drain_until_idle`. Those adapters live beside this file
|
||||
//! (`embeddings.rs`, `chat.rs`, `queue_driver.rs`, `sinks.rs`, `bus.rs`) and are
|
||||
//! added workstream by workstream.
|
||||
//! (`embeddings.rs`, `chat.rs`, `queue_driver.rs`, `ingest.rs`, `seal.rs`, and
|
||||
//! `sync.rs`).
|
||||
//!
|
||||
//! See `docs/tinycortex-migration-spec.md` for the full ownership split,
|
||||
//! drift/gap/parity ledgers, and the workstream order.
|
||||
@@ -66,12 +67,3 @@ pub use sync::{
|
||||
run_gmail_backfill, run_slack_search_backfill, run_source_pipeline, sync_context,
|
||||
HostSyncAdapter, SourcePipelineFailure, HOST_SYNC_STATE_NAMESPACE,
|
||||
};
|
||||
// Facade re-exports — the rest of the host imports memory-engine types through
|
||||
// this one seam so consumer import paths stay stable as the internals flip to
|
||||
// the crate (the type-unification decision, spec §0.5). `MemoryTaint` is the
|
||||
// security-critical provenance type; it is proven byte-identical to the host's
|
||||
// (fail-closed to `ExternalSync`) before re-exporting.
|
||||
pub use tinycortex::memory::{
|
||||
MemoryCategory, MemoryConfig, MemoryEntry, MemoryTaint, NamespaceSummary, RecallOpts,
|
||||
WeightProfile,
|
||||
};
|
||||
|
||||
@@ -18,12 +18,10 @@
|
||||
//! reproduces it exactly. It is a pure function so the policy is unit-tested
|
||||
//! without spinning a live loop.
|
||||
//!
|
||||
//! Still to land in W4 (tracked in the migration spec): the real
|
||||
//! `HostQueueDelegates` that bridges the 8 delegate methods to the host
|
||||
//! `memory_tree` engine (paired with W5), the tokio worker-loop + scheduler that
|
||||
//! applies this policy while driving `run_once`, re-pointing `memory/global.rs`
|
||||
//! and the enqueue call sites onto `queue::store`, and deleting the legacy
|
||||
//! `memory_queue` engine. This brick is additive: nothing is flipped yet.
|
||||
//! The host worker is flipped to `tinycortex::memory::queue::run_once` through
|
||||
//! `HostQueueDelegates`. The adapter preserves host-owned scheduling, health
|
||||
//! reporting, event-bus publishing, and product-policy hooks while the crate
|
||||
//! owns claim/dispatch/settle.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
use chrono::{TimeZone, Utc};
|
||||
use openhuman_core::openhuman::config::Config;
|
||||
use openhuman_core::openhuman::memory::ingest_pipeline::{ingest_chat, ingest_email};
|
||||
use openhuman_core::openhuman::memory::jobs::drain_until_idle;
|
||||
use openhuman_core::openhuman::memory_queue::drain_until_idle;
|
||||
use openhuman_core::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage};
|
||||
use openhuman_core::openhuman::memory_sync::canonicalize::email::{EmailMessage, EmailThread};
|
||||
use openhuman_core::openhuman::tools::{
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
//! 1. the **crate-owned substrate** the `tinycortex` chunk DB creates
|
||||
//! (`init_db` → `chunks/schema.rs`), and
|
||||
//! 2. the **host-retained `UnifiedMemory` namespace-document tier**
|
||||
//! (`memory_store/unified/*`),
|
||||
//! (`memory_store/namespace_store/*`),
|
||||
//!
|
||||
//! coexisting without collision (parity checklist P3/P5/P11/P12 — the W3 gate).
|
||||
//! A store/tree cutover that reshaped, renamed, or dropped a table would strand
|
||||
@@ -105,7 +105,7 @@ const CRATE_CHUNK_SCHEMA_TABLES: &[&str] = &[
|
||||
];
|
||||
|
||||
/// The host-retained `UnifiedMemory` namespace-document tier
|
||||
/// (`memory_store/unified/*`) — stays host, coexists in the shared workspace.
|
||||
/// (`memory_store/namespace_store/*`) — stays host, coexists in the shared workspace.
|
||||
const HOST_UNIFIED_TABLES: &[&str] = &[
|
||||
"memory_docs",
|
||||
"graph_global",
|
||||
|
||||
@@ -308,7 +308,7 @@ async fn round15_composio_agent_tools_backend_cache_and_trigger_history_edges()
|
||||
assert_eq!(action_tool.name(), "GMAIL_FETCH_EMAILS");
|
||||
assert_eq!(action_tool.category().to_string(), "skill");
|
||||
let contract_result = action_tool
|
||||
.execute(json!({ "query": "from:me" }))
|
||||
.execute(json!({ "invented_filter": "from:me" }))
|
||||
.await
|
||||
.expect("per-action contract gate");
|
||||
assert!(contract_result.is_error);
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user