From 12ef69b89fd15acb6973d38928fa77235b827f64 Mon Sep 17 00:00:00 2001 From: sanil-23 Date: Sun, 10 May 2026 22:15:40 +0530 Subject: [PATCH] feat(learning): ambient personalization cache for #566 (#1460) --- docs/AGENT_SELF_LEARNING.md | 368 ++++++++ src/core/event_bus/events.rs | 54 +- src/core/event_bus/events_tests.rs | 23 + src/openhuman/agent/harness/archivist.rs | 186 ++++ .../agent/harness/session/builder.rs | 31 + src/openhuman/channels/runtime/startup.rs | 74 ++ src/openhuman/composio/providers/profile.rs | 29 +- .../composio/providers/profile_md.rs | 410 +++++++-- src/openhuman/config/schema/learning.rs | 27 + src/openhuman/learning/cache.rs | 126 +++ src/openhuman/learning/cache_tests.rs | 250 ++++++ src/openhuman/learning/candidate.rs | 380 ++++++++ src/openhuman/learning/extract/heuristics.rs | 614 +++++++++++++ src/openhuman/learning/extract/mod.rs | 15 + src/openhuman/learning/extract/signature.rs | 643 +++++++++++++ .../learning/extract/summary_facets.rs | 343 +++++++ src/openhuman/learning/mod.rs | 33 +- src/openhuman/learning/profile_md_renderer.rs | 514 +++++++++++ src/openhuman/learning/prompt_sections.rs | 282 ++++++ src/openhuman/learning/reflection.rs | 131 ++- src/openhuman/learning/reflection_tests.rs | 78 ++ src/openhuman/learning/scheduler.rs | 164 ++++ src/openhuman/learning/schemas.rs | 819 ++++++++++++++++- src/openhuman/learning/stability_detector.rs | 846 ++++++++++++++++++ src/openhuman/memory/store/unified/init.rs | 17 + src/openhuman/memory/store/unified/profile.rs | 489 +++++++++- .../memory/store/unified/profile_tests.rs | 263 ++++++ src/openhuman/memory/tree/ingest.rs | 46 +- .../memory/tree/tree_source/summariser/llm.rs | 232 ++++- .../memory/tree/tree_source/summariser/mod.rs | 5 +- tests/learning_phase4_integration_test.rs | 310 +++++++ 31 files changed, 7656 insertions(+), 146 deletions(-) create mode 100644 docs/AGENT_SELF_LEARNING.md create mode 100644 src/openhuman/learning/cache.rs create mode 100644 src/openhuman/learning/cache_tests.rs create mode 100644 src/openhuman/learning/candidate.rs create mode 100644 src/openhuman/learning/extract/heuristics.rs create mode 100644 src/openhuman/learning/extract/mod.rs create mode 100644 src/openhuman/learning/extract/signature.rs create mode 100644 src/openhuman/learning/extract/summary_facets.rs create mode 100644 src/openhuman/learning/profile_md_renderer.rs create mode 100644 src/openhuman/learning/scheduler.rs create mode 100644 src/openhuman/learning/stability_detector.rs create mode 100644 tests/learning_phase4_integration_test.rs diff --git a/docs/AGENT_SELF_LEARNING.md b/docs/AGENT_SELF_LEARNING.md new file mode 100644 index 000000000..d06c1eed6 --- /dev/null +++ b/docs/AGENT_SELF_LEARNING.md @@ -0,0 +1,368 @@ +# Agent self-learning + +OpenHuman learns user preferences continuously and surfaces them as ambient defaults in every system prompt. The mechanism is a small **personalization cache** materialized from multiple deterministic + LLM-driven producers, scored by stability, rendered into a user-editable `PROFILE.md`, and injected into prompts through the existing prompt-section pipeline. + +This document covers how preferences are captured, scored, persisted, and surfaced. For the originating issue, see [#566](https://github.com/tinyhumansai/openhuman/issues/566). + +--- + +## What gets learned + +Six classes, encoded in the `key` prefix and stored in the existing `user_profile_facets` table: + +| Class | Example facets | +|---|---| +| `style/*` | `verbosity=terse`, `format=bullets`, `preamble=skip`, `language=english`, `emoji=skip` | +| `identity/*` | `name=Sanil`, `timezone=PST`, `role=engineer`, `employer=vezures` | +| `tooling/*` | `package_manager=pnpm`, `lang=rust`, `framework=astro`, `runtime=bun` | +| `veto/*` | `tool=jest → banned`, `format=nested-bullets → banned` | +| `goal/*` | free-form goal sentences (slugified key) | +| `channel/preference` | `primary=desktop-chat` | + +Recurring topics, recurring entities, and prior threads are **not** in the cache. They live in the memory tree and are retrieved per-turn by `memory_recall` under the prompt-bias instruction described below. + +--- + +## Architecture + +Four stages: **capture → identify → score → materialize**. + +``` +inputs substrate candidates cache +───── ───────── ────────── ───── +chat turns ──→ episodic_log ──→ Buffer (push) ──┐ + + tree │ +skill syncs ──→ tree (sources) ──→ Buffer (push) ──┤ + ├─→ user_profile_facets +channel inbound ──→ tree ──→ Buffer (push) ──┤ (state, stability, + │ user_state, evidence) +documents ──→ tree ──→ Buffer (push) ──┘ │ + ▼ + stability_detector::rebuild CacheRebuilt event + (every 30 min + event-driven) │ + ▼ + ProfileMdRenderer + │ + ▼ + PROFILE.md + (managed blocks) + │ + ▼ + UserFilesSection + + UserProfileSection + + MemoryAccessSection + │ + ▼ + agent system prompt +``` + +Chat turns flow into the tree as `source_id="conversations:agent"` (with `tool_calls_json` stripped at canonicalize), so the same `tree_source::summariser` that already runs over Slack/Gmail/Notion now also produces summaries — and structured facet candidates — over chat content. + +--- + +## Producers + +Five producers write `LearningCandidate` values into `learning::candidate::global()`: + +| # | Producer | Emits | LLM? | +|---|---|---|---| +| 1 | `composio::providers::profile::persist_provider_profile` (existing) | Identity from Gmail/Slack/Notion account fields | No | +| 2 | `learning::extract::signature::EmailSignatureSubscriber` | Identity from email signatures (last 8 lines) | No | +| 3 | `learning::extract::heuristics` (`LengthRatioDetector`, `EditWindowDetector`, `CorrectionRepeatDetector`) | Style + Veto from per-turn rolling state | No | +| 4 | `learning::ReflectionHook` (rerouted) | Goal + Style from heuristic cues and LLM-extracted reflections | Optional | +| 5 | `tree_source::summariser::llm` (extended schema) | All classes — long-tail extraction over rolling per-source summaries | Yes (existing call, extended output) | + +Producer 5 is the long-tail backbone. Its prompt now asks the model to emit `{ summary, facets[] }`, and `learning::extract::summary_facets` validates each `ParsedFacet` (canonical class, mandatory `evidence_chunks`) before pushing to the buffer. **No new LLM calls** are introduced — the change extends an existing summarization round-trip with ~150–400 output tokens. + +Explicitly **not** producing: + +- Hand-curated regex catalogs for style/identity/tooling — Producer 5 covers them and generalizes to vocabulary the catalog wouldn't predict. +- Hand-curated manifest parsers for tooling — same reasoning; Astro, Bun, Deno, uv, mise, etc. all surface through the LLM summarizer. +- Free-text NER on chat — recurring entities live in the tree's graph (from structured provider metadata) and surface contextually via `memory_recall`. +- `ToolTrackerHook` candidate emission — failure-rate is not a clean preference signal; it stays as substrate for debugging. + +--- + +## Identification → candidate buffer + +Every emission carries provenance: + +```rust +pub struct LearningCandidate { + pub class: FacetClass, // Style | Identity | Tooling | Veto | Goal | Channel + pub key: String, // canonical slug, e.g. "verbosity" + pub value: String, // canonical value, e.g. "terse" + pub cue_family: CueFamily, // Explicit | Structural | Behavioral | Recurrence + pub evidence: EvidenceRef, // pointer back into substrate + pub initial_confidence: f64, // 0..=1, source-provided hint + pub observed_at: f64, // epoch seconds +} +``` + +`EvidenceRef` variants cover every substrate origin (`Episodic`, `EpisodicWindow`, `SourceSummary`, `TreeTopic`, `DocumentChunk`, `EmailMessage`, `Provider`, `ToolCall`, `TreeSourceWeight`). The buffer is a thread-safe bounded ring (default capacity 1024). The stability detector drains it every rebuild. + +--- + +## Stability scoring + +``` +stability(class, key, value) = base × cue × user_state + +base = Σ over evidence: + cue_family_weight × exp(-Δt / half_life_for_class) × log(1 + evidence_count_for_family) + +cue = 2.0 if any evidence is Explicit, else 1.0 +user = ∞ if Pinned, 0 if Forgotten, 1 otherwise +``` + +**Cue-family weights**: + +| Family | Weight | Rationale | +|---|---|---| +| `Explicit` | 1.0 | direct user statement — declaration is intent | +| `Structural` | 0.9 | provider data / manifest content / signature — data doesn't lie | +| `Behavioral` | 0.7 | heuristics + summary mining — must accumulate | +| `Recurrence` | 0.6 | tree statistics — emerging | + +**Class half-lives** (the time over which evidence weight halves): + +| Class | Half-life | +|---|---| +| `identity` | 90 days | +| `veto` | 60 days | +| `tooling` | 30 days | +| `goal` | 30 days | +| `style` | 14 days | +| `channel` | 7 days | + +**Conflict resolution per `(class, key)`**: active value = `argmax(stability)` over candidate values. Losing values are dropped from the cache; if they re-emerge they reinforce naturally through the same path. + +--- + +## States and qualification + +Two state columns govern visibility: + +`state` (driven by stability): + +| State | Range | Effect | +|---|---|---| +| `Active` | `stability ≥ τ_promote` (1.5) | Renders in `PROFILE.md`, injected into prompt | +| `Provisional` | `0.7 ≤ stability < 1.5` | Stored, not rendered | +| `Candidate` | `0.4 ≤ stability < 0.7` | Stays in buffer, not yet in cache | +| `Dropped` | `stability < 0.4` | Removed | + +`user_state` (overrides scoring): + +| State | Effect | +|---|---| +| `Auto` | Default; `state` follows scoring | +| `Pinned` | Locks `Active`; resists decay | +| `Forgotten` | Locks `Dropped`; blocks re-promotion | + +**Class budgets** (top-N selection per class with a shared overflow pool): + +``` +style 4 · identity 4 · tooling 5 · veto 3 · goal 3 · channel 1 · overflow 5 +total cap ≈ 25 +``` + +Per-class budgets prevent a tooling-heavy user from drowning out style and identity; the overflow pool lets one class take spare capacity from another when underused. + +--- + +## Storage + +Single SQLite table — `user_profile_facets` (existing, extended by an idempotent migration in `memory::store::unified::profile::migrate_profile_schema`): + +```sql +CREATE TABLE user_profile ( + facet_id TEXT PRIMARY KEY, + facet_type TEXT NOT NULL, -- legacy enum + key TEXT NOT NULL, + value TEXT NOT NULL, + confidence REAL NOT NULL DEFAULT 0.5, + evidence_count INTEGER NOT NULL DEFAULT 1, + source_segment_ids TEXT, + first_seen_at REAL NOT NULL, + last_seen_at REAL NOT NULL, + -- learning-cache columns (added by migrate_profile_schema): + state TEXT NOT NULL DEFAULT 'active', + stability REAL NOT NULL DEFAULT 0.0, + user_state TEXT NOT NULL DEFAULT 'auto', + evidence_refs_json TEXT, + class TEXT, + cue_families_json TEXT, + UNIQUE(facet_type, key) +); +CREATE INDEX idx_profile_state ON user_profile(state); +CREATE INDEX idx_profile_class ON user_profile(class); +``` + +The provider-profile path (`composio::providers::profile::persist_provider_profile`) continues to write its existing rows untouched; class is auto-derived for backward-compatibility (provider keys like `skill:gmail:default:email` map to `class=tooling`/`identity`). + +--- + +## Surface — `PROFILE.md` + +`{workspace_dir}/PROFILE.md` carries multiple managed blocks. Each block is owned by automation; content outside the markers is user-authored and preserved across rebuilds. + +```markdown +# User Profile + + +## Style + +- **verbosity**: terse +- **preamble**: skip *(pinned)* + + + + +## Identity + +- **name**: Sanil +- **timezone**: PST + + + + +## Tooling + +- **lang**: rust +- **package_manager**: pnpm + + + + +## Vetoes + +- **tool=jest**: banned + + + + +## Goals + +- ship #566 before #686 + + + + +## Connected Accounts + +- gmail (sanil@vezures.xyz) + + +``` + +User-authored content between blocks (free-form notes, hand-edited details) is preserved verbatim across rebuilds. + +`ProfileMdRenderer` subscribes to `DomainEvent::CacheRebuilt` and rewrites the cache-derived blocks (`style`, `identity`, `tooling`, `vetoes`, `goals`). The `connected-accounts` block remains owned by the provider sync path. + +--- + +## Surface — prompt sections + +Three sections in `agent/prompts/` cooperate: + +- **`UserFilesSection`** — already-existing; injects `PROFILE.md` verbatim into the system prompt every turn. +- **`UserProfileSection`** — repointed in Phase 4 to read `FacetCache::list_active()` (gated by `LearningConfig::use_cache_for_user_profile_section`, default true). Renders `- **{class}/{key}**: {value}` bullets. +- **`MemoryAccessSection`** — new in Phase 4; static instruction biasing the agent to call `memory_recall` for entities, threads, prior decisions, recurring topics. Adds ~80 tokens to the system prompt. + +The `MemoryAccessSection` covers the **contextual** half of personalization that the cache structurally cannot address: things that need to be retrieved when relevant (recurring people, prior threads) rather than always rendered. + +--- + +## Surface — RPC controllers + +Wired through the controller registry (`learning::schemas::all_learning_controller_schemas`): + +| RPC | Purpose | +|---|---| +| `learning.list_facets { class }` | Enumerate the cache, optionally filtered by class. Returns active + provisional entries with provenance. | +| `learning.get_facet { class, key }` | Single-entry lookup. | +| `learning.update_facet { class, key, value }` | Overwrite the active value; auto-pins to prevent rebuild from clobbering. | +| `learning.pin_facet { class, key }` | `user_state = Pinned`. Locks `Active` regardless of stability score. | +| `learning.unpin_facet { class, key }` | `user_state = Auto`. | +| `learning.forget_facet { class, key }` | `user_state = Forgotten`. Locks `Dropped`, blocks re-promotion. | +| `learning.reset_cache {}` | Clears all `Auto` rows; preserves `Pinned`. Next rebuild repopulates from substrate. | +| `learning.rebuild_cache {}` | Manual trigger for the stability rebuild. | +| `learning.cache_stats {}` | `{ total, active, provisional, candidate, dropped, by_class }`. | + +The agent itself acts as a conversational user-control surface: asked "what do you know about me?" it can call `list_facets` and cite `evidence_refs_json` for each entry; "forget that I prefer terse" calls `forget_facet("style", "verbosity")`. + +--- + +## Configuration + +`LearningConfig` in `src/openhuman/config/schema/learning.rs`: + +```rust +pub struct LearningConfig { + pub enabled: bool, // master switch + pub reflection_enabled: bool, + pub user_profile_enabled: bool, // legacy hook + pub tool_tracking_enabled: bool, // substrate-only + pub reflection_source: ReflectionSource, // Local | Cloud + pub max_reflections_per_session: usize, + pub min_turn_complexity: usize, + pub chat_to_tree_enabled: bool, // pipe agent chat into tree + pub stability_detector_enabled: bool, + pub rebuild_interval_secs: u64, // default 1800 (30 min) + pub use_cache_for_user_profile_section: bool, // route prompt-section reads through cache +} +``` + +The summarizer-side facet emission is gated by `LlmSummariserConfig::structured_facet_extraction` (default `true`) so production deployments can disable structured extraction independently of the rest of the learning subsystem. + +--- + +## Observability + +`DomainEvent::CacheRebuilt { added, evicted, kept, total_size, rebuilt_at }` is published after every successful rebuild. Subscribers can wire personalization metrics — for example, facet-in-prompt × positive-acceptance rate — to satisfy #566's "measurable personalization improvements" criterion. + +`learning.cache_stats` returns the breakdown by state and class for ad-hoc inspection. + +Tracing prefixes used by new flows (filter your log stream with these): + +- `[learning::candidate]` +- `[learning::extract::signature]` +- `[learning::extract::heuristics]` +- `[learning::extract::summary_facets]` +- `[learning::stability_detector]` +- `[learning::cache]` +- `[learning::profile_md]` +- `[archivist]` (chat-into-tree path) +- `[memory::tree::ingest]` (`DocumentCanonicalized` emission) + +--- + +## Testing + +Unit tests live next to their modules (`*_tests.rs` siblings; consistent with the rest of the codebase). End-to-end coverage in `tests/learning_phase4_integration_test.rs`: + +- Push candidates → `stability_detector.rebuild()` → expected `CacheRebuilt` event published +- `ProfileMdRenderer` writes the expected managed blocks into `PROFILE.md` +- `learning.pin_facet` keeps an entry `Active` despite weak evidence +- `learning.forget_facet` removes an entry from `PROFILE.md` and blocks re-promotion +- `learning.list_facets` returns the expected shape and class filter + +Run targeted suites: + +```bash +cargo test --manifest-path Cargo.toml --lib learning:: +cargo test --manifest-path Cargo.toml --lib memory::store::unified::profile +cargo test --manifest-path Cargo.toml --test learning_phase4_integration_test +``` + +--- + +## Future work + +- **Validation loop** — track facet-in-prompt × positive-acceptance to calibrate cue-family weights from outcome signal rather than fixed defaults. +- **Confirmation prompts** — for high-impact, low-confidence facets (Identity, Veto), surface to the user via the agent or subconscious before promoting to `Active`. +- **Cross-corpus topic stitching** — once chat-as-tree-source has accumulated, surface unified hot topics back into the prompt as contextual hints alongside the cache. +- **Per-channel persona facets** — tooling and style differ by channel; split when signals diverge meaningfully (e.g., terser in Slack DMs, more structured in email). +- **Tool-suggestion telemetry** — distinct from the existing failure stats; needs new `ToolSuggestionAccepted`/`ToolSuggestionRejected` events before it can produce clean veto signal. +- **Soft user confirmation in PROFILE.md** — render `Provisional` rows under a quieter sub-heading so the user can promote them by editing without waiting for accumulation. diff --git a/src/core/event_bus/events.rs b/src/core/event_bus/events.rs index 06258e759..27e5d4b42 100644 --- a/src/core/event_bus/events.rs +++ b/src/core/event_bus/events.rs @@ -353,6 +353,55 @@ pub enum DomainEvent { routed: bool, }, + // ── Memory tree ───────────────────────────────────────────────────── + /// A document (chat batch, email thread, or standalone document) was + /// fully canonicalised and its chunks written to the memory tree. + /// + /// Emitted by `memory::tree::ingest::persist()` after the chunk upsert + /// and extract-job enqueue complete. Subscribers (Phase 2 producers such + /// as the email-signature parser) react to this to inspect the + /// canonicalised content. + DocumentCanonicalized { + /// The source identifier passed to the ingest call (e.g. `"gmail:abc"`, + /// `"conversations:agent"`). + source_id: String, + /// Kind of content — `"chat"`, `"email"`, `"document"`. + source_kind: String, + /// Number of chunks written to `vector_chunks` in this ingest. + chunks_written: usize, + /// IDs of the chunks that were written. + chunk_ids: Vec, + /// Wall-clock seconds since epoch when canonicalisation completed. + canonicalized_at: f64, + /// Last ≤ 2 048 characters of the canonicalised markdown body. + /// + /// Populated for `email` and `document` sources so that lightweight + /// subscribers (e.g. the email-signature parser) can inspect trailing + /// content without hitting disk. `None` for `chat` sources where the + /// content is conversational and doesn't contain signature-style structure. + body_preview: Option, + }, + + // ── Learning ───────────────────────────────────────────────────────── + /// The stability detector finished a full cache rebuild cycle. + /// + /// Emitted by `learning::stability_detector` (Phase 3) after writing + /// the new snapshot to `user_profile_facets`. Subscribers (Phase 4 + /// `profile_md_renderer`) react to re-render the `PROFILE.md` managed + /// blocks. + CacheRebuilt { + /// Number of facets added in this cycle. + added: usize, + /// Number of facets evicted (below τ_evict threshold) in this cycle. + evicted: usize, + /// Number of facets unchanged / carried over. + kept: usize, + /// Total facets in the cache after the rebuild. + total_size: usize, + /// Wall-clock seconds since epoch when the rebuild completed. + rebuilt_at: f64, + }, + // ── System lifecycle ──────────────────────────────────────────────── /// A system component started up. SystemStartup { component: String }, @@ -389,7 +438,10 @@ impl DomainEvent { | Self::MemoryRecalled { .. } | Self::MemorySyncRequested { .. } | Self::MemoryIngestionStarted { .. } - | Self::MemoryIngestionCompleted { .. } => "memory", + | Self::MemoryIngestionCompleted { .. } + | Self::DocumentCanonicalized { .. } => "memory", + + Self::CacheRebuilt { .. } => "learning", Self::ChannelInboundMessage { .. } | Self::ChannelMessageReceived { .. } diff --git a/src/core/event_bus/events_tests.rs b/src/core/event_bus/events_tests.rs index 73a325115..8629c3aef 100644 --- a/src/core/event_bus/events_tests.rs +++ b/src/core/event_bus/events_tests.rs @@ -415,6 +415,29 @@ fn all_variants_have_correct_domain() { }, "system", ), + // Memory tree + ( + DomainEvent::DocumentCanonicalized { + source_id: "gmail:abc".into(), + source_kind: "email".into(), + chunks_written: 3, + chunk_ids: vec!["c1".into(), "c2".into(), "c3".into()], + canonicalized_at: 1_700_000_000.0, + body_preview: Some("Thanks,\nAlice".into()), + }, + "memory", + ), + // Learning + ( + DomainEvent::CacheRebuilt { + added: 2, + evicted: 1, + kept: 5, + total_size: 7, + rebuilt_at: 1_700_000_000.0, + }, + "learning", + ), ]; for (event, expected_domain) in cases { diff --git a/src/openhuman/agent/harness/archivist.rs b/src/openhuman/agent/harness/archivist.rs index b80280e10..7e49ddc51 100644 --- a/src/openhuman/agent/harness/archivist.rs +++ b/src/openhuman/agent/harness/archivist.rs @@ -6,14 +6,19 @@ //! 2. Manages conversation segments (boundary detection + lifecycle). //! 3. On segment close: extracts events (heuristic) and updates user profile. //! 4. Extracts simple lessons from tool failures. +//! 5. (Phase 1 / #566) Pipes the turn into the memory tree as `conversations:agent` +//! when `config.learning.chat_to_tree_enabled` is true. use crate::openhuman::agent::hooks::{PostTurnHook, TurnContext}; +use crate::openhuman::config::Config; use crate::openhuman::memory::store::events::{self, EventRecord, EventType}; use crate::openhuman::memory::store::fts5::{self, EpisodicEntry}; use crate::openhuman::memory::store::profile::{self, FacetType}; use crate::openhuman::memory::store::segments::{ self, BoundaryConfig, BoundaryDecision, ConversationSegment, }; +use crate::openhuman::memory::tree::canonicalize::chat::{ChatBatch, ChatMessage}; +use crate::openhuman::memory::tree::ingest; use async_trait::async_trait; use parking_lot::Mutex; use rusqlite::Connection; @@ -31,24 +36,43 @@ pub struct ArchivistHook { enabled: bool, /// Boundary detection configuration. boundary_config: BoundaryConfig, + /// Optional runtime config — used to gate the tree-ingest path. + /// + /// When `None`, the tree-ingest path is skipped. Set via + /// [`ArchivistHook::with_config`] on the production path. + config: Option, } impl ArchivistHook { /// Create an Archivist hook with a shared SQLite connection. + /// + /// Tree-ingest is disabled by default; call [`Self::with_config`] to + /// enable it on the production path. pub fn new(conn: Arc>, enabled: bool) -> Self { Self { conn: Some(conn), enabled, boundary_config: BoundaryConfig::default(), + config: None, } } + /// Attach runtime config so the archivist can gate the tree-ingest path. + /// + /// When `config.learning.chat_to_tree_enabled` is `true`, each completed + /// turn is also piped into the memory tree as `source="conversations:agent"`. + pub fn with_config(mut self, config: Config) -> Self { + self.config = Some(config); + self + } + /// Create a disabled/no-op Archivist (when FTS5 is not enabled). pub fn disabled() -> Self { Self { conn: None, enabled: false, boundary_config: BoundaryConfig::default(), + config: None, } } @@ -365,11 +389,173 @@ impl PostTurnHook for ArchivistHook { current_episodic_id, ); + // ── Phase 1 / #566: pipe turn into the memory tree ─────────────────── + // Gate: only when config is attached and chat_to_tree_enabled is true. + // Non-fatal: if tree-ingest fails, the episodic write already succeeded + // and the turn result is not affected. + if let Some(ref cfg) = self.config { + if cfg.learning.chat_to_tree_enabled { + tracing::debug!( + "[archivist] piping turn into tree as conversations:agent session={}", + session_id + ); + self.pipe_turn_to_tree(cfg, ctx, session_id, timestamp) + .await; + } + } + tracing::debug!("[archivist] turn indexed successfully"); Ok(()) } } +impl ArchivistHook { + /// Pipe the completed turn into the memory tree as `source="conversations:agent"`. + /// + /// Tool-call JSON is stripped from the assistant text before ingest — only + /// the assistant's prose response flows into the tree (memory ingestion + /// policy: tool outputs must not reach memory). + /// + /// Failures are logged and swallowed; the episodic write is the source of + /// truth. + async fn pipe_turn_to_tree( + &self, + config: &Config, + ctx: &TurnContext, + session_id: &str, + timestamp: f64, + ) { + use chrono::{TimeZone, Utc}; + + // Build turn timestamps. The assistant message is offset by 1ms as in + // the episodic write so ordering is stable. + let user_ts = Utc + .timestamp_opt( + timestamp as i64, + ((timestamp.fract() * 1e9) as u32).min(999_999_999), + ) + .single() + .unwrap_or_else(Utc::now); + let asst_ts = Utc + .timestamp_opt( + (timestamp + 0.001) as i64, + (((timestamp + 0.001).fract() * 1e9) as u32).min(999_999_999), + ) + .single() + .unwrap_or(user_ts); + + // Strip tool-call JSON from the assistant response. + // Per memory ingestion policy, structured tool-call payloads must not + // flow into the tree — only the prose response is ingested. + let assistant_prose = strip_tool_calls_from_response(&ctx.assistant_response); + + let batch = ChatBatch { + platform: "agent".into(), + channel_label: session_id.to_string(), + messages: vec![ + ChatMessage { + author: "user".into(), + timestamp: user_ts, + text: ctx.user_message.clone(), + source_ref: Some(format!("agent://session/{session_id}")), + }, + ChatMessage { + author: "assistant".into(), + timestamp: asst_ts, + text: assistant_prose, + source_ref: Some(format!("agent://session/{session_id}")), + }, + ], + }; + + // Use the session_id as the owner / identity tag. + let source_id = "conversations:agent"; + let owner = session_id; + let tags = vec!["agent_chat".to_string()]; + + match ingest::ingest_chat(config, source_id, owner, tags, batch).await { + Ok(result) => { + tracing::debug!( + "[archivist] tree ingest ok: source_id={} chunks_written={} session={}", + source_id, + result.chunks_written, + session_id + ); + } + Err(e) => { + tracing::warn!( + "[archivist] tree ingest failed (non-fatal): source_id={} session={} error={e}", + source_id, + session_id + ); + } + } + } +} + +/// Strip tool-call JSON blocks from an assistant response, leaving only the +/// prose text. +/// +/// The archivist stores the full response (including `tool_calls_json`) in +/// the episodic log for diagnostic purposes. However, per the memory +/// ingestion policy, structured tool-call payloads must not reach the memory +/// tree — only the assistant's natural-language prose is ingested. +/// +/// This function applies a lightweight heuristic: it removes any contiguous +/// spans of text that look like `…` XML/JSON blocks or +/// raw JSON objects that begin with `{"tool_calls":`. The output may be empty +/// if the entire response was tool-call markup — callers should handle that +/// case (empty text → no-op ingest). +fn strip_tool_calls_from_response(response: &str) -> String { + // Fast path: if the response contains no obvious tool-call markers, return + // it unchanged to avoid unnecessary allocation. + if !response.contains("") + && !response.contains("{\"tool_calls\"") + && !response.contains("\"tool_use\"") + { + return response.to_string(); + } + + // Remove XML-style tool-call blocks. + let mut cleaned = response.to_string(); + + // Strip … spans (may span multiple lines). + while let Some(start) = cleaned.find("") { + if let Some(end) = cleaned[start..].find("") { + cleaned.drain(start..start + end + "".len()); + } else { + // Unclosed tag — remove from the tag to end of string. + cleaned.truncate(start); + break; + } + } + + // Trim and collapse runs of blank lines left by block removal. + let trimmed = cleaned + .lines() + .map(str::trim_end) + .collect::>() + .join("\n"); + + // Collapse more than two consecutive newlines to two. + let mut result = String::with_capacity(trimmed.len()); + let mut blank_run = 0usize; + for line in trimmed.lines() { + if line.is_empty() { + blank_run += 1; + if blank_run <= 2 { + result.push('\n'); + } + } else { + blank_run = 0; + result.push_str(line); + result.push('\n'); + } + } + + result.trim().to_string() +} + /// Extract simple lessons from tool call outcomes (no LLM needed). fn extract_lesson_from_tools( tool_calls: &[crate::openhuman::agent::hooks::ToolCallRecord], diff --git a/src/openhuman/agent/harness/session/builder.rs b/src/openhuman/agent/harness/session/builder.rs index 28aa88a9d..e3c20fe9b 100644 --- a/src/openhuman/agent/harness/session/builder.rs +++ b/src/openhuman/agent/harness/session/builder.rs @@ -762,6 +762,8 @@ impl Agent { .add_section(Box::new( crate::openhuman::learning::UserProfileSection::new(memory.clone()), )); + // NOTE: MemoryAccessSection is added after tool-filtering so we can + // gate it on retrieval-tool visibility — see below. log::info!( "[learning] prompt sections registered (user_reflections, learned_context, user_profile)" ); @@ -928,6 +930,35 @@ impl Agent { .map(|t| t.name().to_string()) .collect(), }; + + // Phase 4 (#566): add the MemoryAccessSection bias instruction only + // when at least one retrieval tool is actually loaded AND survives + // filtering. We require both because: + // - the tool may be filtered out by the agent's scope config + // - the tool may not be registered at all on this agent (tool + // listing is build-time configurable) + // An empty `visible` set means "no filter" (wildcard / orchestrator + // path); in that case any registered retrieval tool is reachable. + if config.learning.enabled { + let recall_tools = ["memory_recall", "memory_search"]; + let has_retrieval = recall_tools.iter().any(|name| { + let registered = tools.iter().any(|t| t.name() == *name) + || delegation_tools.iter().any(|t| t.name() == *name); + let allowed_by_filter = visible.is_empty() || visible.contains(*name); + registered && allowed_by_filter + }); + if has_retrieval { + prompt_builder = prompt_builder + .add_section(Box::new(crate::openhuman::learning::MemoryAccessSection)); + log::debug!("[learning] memory_access prompt section registered"); + } else { + log::debug!( + "[learning] skipping MemoryAccessSection — neither memory_recall nor \ + memory_search is registered+visible for agent={agent_id}" + ); + } + } + // De-duplicate: some synthesised tool names may collide with // already-registered tools (unlikely for `delegate_*` names but // cheap to guard against). diff --git a/src/openhuman/channels/runtime/startup.rs b/src/openhuman/channels/runtime/startup.rs index 5d2c79495..39c5c94b7 100644 --- a/src/openhuman/channels/runtime/startup.rs +++ b/src/openhuman/channels/runtime/startup.rs @@ -64,6 +64,80 @@ pub async fn start_channels(config: Config) -> Result<()> { // channel dispatch calls `request_native_global("agent.run_turn", …)` // for every inbound message. crate::openhuman::agent::bus::register_agent_handlers(); + // Phase 2 learning producers: email-signature subscriber reacts to + // DocumentCanonicalized events and emits Identity candidates into the buffer. + // The handle is intentionally leaked into a static so the subscription stays + // alive for the lifetime of the process (same pattern as TracingSubscriber). + { + use crate::core::event_bus::SubscriptionHandle; + use std::sync::OnceLock; + static EMAIL_SIG_HANDLE: OnceLock> = OnceLock::new(); + EMAIL_SIG_HANDLE.get_or_init(|| { + crate::openhuman::learning::extract::signature::register_email_signature_subscriber() + }); + } + + // Phase 3 learning: register the event-driven rebuild trigger. + // The stability detector is wired up only when the global memory client is + // already initialised (it may not be in the channel runtime path — the + // client is initialised later in `start_channels`). + { + use crate::core::event_bus::SubscriptionHandle; + use std::sync::OnceLock; + static REBUILD_TRIGGER_HANDLE: OnceLock> = OnceLock::new(); + REBUILD_TRIGGER_HANDLE.get_or_init(|| { + if let Some(client) = crate::openhuman::memory::global::client_if_ready() { + use crate::openhuman::learning::cache::FacetCache; + use crate::openhuman::learning::scheduler::register_event_trigger; + use crate::openhuman::learning::StabilityDetector; + use std::sync::Arc; + let cache = FacetCache::new(client.profile_conn()); + let detector = Arc::new(StabilityDetector::new(cache)); + // Also spawn the periodic rebuild loop (30-minute cadence). + let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); + // Leak the sender so the loop never receives a shutdown signal + // until the process exits. This matches the pattern used by + // other always-on background tasks. + Box::leak(Box::new(shutdown_tx)); + crate::openhuman::learning::scheduler::spawn_rebuild_loop( + Arc::clone(&detector), + crate::openhuman::learning::scheduler::DEFAULT_REBUILD_INTERVAL, + shutdown_rx, + ); + register_event_trigger(detector) + } else { + tracing::debug!("[learning::scheduler] memory client not ready at channel startup, skipping event-trigger registration"); + None + } + }); + } + + // Phase 4 learning: register the ProfileMdRenderer subscriber. + // Subscribes to CacheRebuilt events and re-renders the five cache-derived + // PROFILE.md blocks (style, identity, tooling, vetoes, goals). + { + use crate::core::event_bus::SubscriptionHandle; + use std::sync::OnceLock; + static PROFILE_MD_RENDERER_HANDLE: OnceLock> = OnceLock::new(); + PROFILE_MD_RENDERER_HANDLE.get_or_init(|| { + if let Some(client) = crate::openhuman::memory::global::client_if_ready() { + use crate::openhuman::learning::cache::FacetCache; + use crate::openhuman::learning::ProfileMdRenderer; + use std::sync::Arc; + let cache = Arc::new(FacetCache::new(client.profile_conn())); + let renderer = + Arc::new(ProfileMdRenderer::new(cache, config.workspace_dir.clone())); + ProfileMdRenderer::subscribe(renderer) + } else { + tracing::debug!( + "[learning::profile_md_renderer] memory client not ready at startup, \ + skipping ProfileMdRenderer registration" + ); + None + } + }); + } + tracing::debug!("[event_bus] global singleton initialized in start_channels"); // Initialise the sub-agent definition registry from this workspace. diff --git a/src/openhuman/composio/providers/profile.rs b/src/openhuman/composio/providers/profile.rs index 396f8149d..4bb37dc54 100644 --- a/src/openhuman/composio/providers/profile.rs +++ b/src/openhuman/composio/providers/profile.rs @@ -18,7 +18,10 @@ //! RPC ops. use super::ProviderUserProfile; -use crate::openhuman::memory::store::profile::{self, FacetType}; +use crate::openhuman::learning::candidate::{ + self as learning_candidate, CueFamily, EvidenceRef, FacetClass, LearningCandidate, +}; +use crate::openhuman::memory::store::profile::{self, FacetState, FacetType, UserState}; use rusqlite::params; use serde_json::Value; use std::collections::BTreeMap; @@ -170,6 +173,28 @@ pub fn persist_provider_profile(profile: &ProviderUserProfile) -> usize { ); continue; } + + // Phase 3 (#566): also emit a LearningCandidate so the stability detector + // can score provider data alongside other evidence on the next rebuild. + // We use the `identity/` key prefix for provider identity fields. + if kind.is_matchable() { + let identity_key = format!("{}:{}", normalize_token(&toolkit), kind.as_str()); + let candidate = LearningCandidate { + class: FacetClass::Identity, + key: identity_key, + value: value.clone(), + cue_family: CueFamily::Structural, + evidence: EvidenceRef::Provider { + toolkit: toolkit.clone(), + connection_id: identifier.clone(), + field: kind.as_str().to_string(), + }, + initial_confidence: kind.confidence(), + observed_at: now, + }; + learning_candidate::global().push(candidate); + } + written += 1; } @@ -178,7 +203,7 @@ pub fn persist_provider_profile(profile: &ProviderUserProfile) -> usize { toolkit = %toolkit, identifier = %identifier, rows_written = written, - "[composio:profile] persisted identity rows" + "[composio:profile] persisted identity rows (+ emitted Identity candidates)" ); } written diff --git a/src/openhuman/composio/providers/profile_md.rs b/src/openhuman/composio/providers/profile_md.rs index bd511e958..f2c471d82 100644 --- a/src/openhuman/composio/providers/profile_md.rs +++ b/src/openhuman/composio/providers/profile_md.rs @@ -1,23 +1,27 @@ -//! `PROFILE.md` markdown bridge — mirrors the per-toolkit identity -//! fragments we already persist into the `user_profile` facet table -//! into a managed block inside `{workspace_dir}/PROFILE.md` so the -//! agent prompt loader (`agent/prompts/mod.rs::UserFilesSection`) -//! picks them up on the next turn. +//! `PROFILE.md` markdown bridge — mirrors managed facet blocks into +//! `{workspace_dir}/PROFILE.md` so the agent prompt loader +//! (`agent/prompts/mod.rs::UserFilesSection`) picks them up on the next +//! turn. //! -//! The block lives between the markers +//! ## Block convention +//! +//! Each managed section lives between a pair of HTML comment markers: //! //! ```md -//! -//! ... -//! +//! +//! ##
+//! +//! +//! +//! //! ``` //! -//! Anything outside the markers is left untouched, so a profile authored -//! by the LinkedIn onboarding pipeline or hand-edited by the user is -//! preserved across reconnects. +//! Anything outside the markers is left untouched, so user-authored prose +//! or hand-edited bullets are preserved across provider reconnects or +//! cache rebuilds. //! -//! All operations are best-effort and log on failure rather than -//! propagating, matching the existing PII-discipline pattern in +//! All operations are best-effort — errors are logged rather than +//! propagated, matching the PII-discipline pattern used in //! `on_connection_created`. use super::ProviderUserProfile; @@ -25,17 +29,31 @@ use std::fs; use std::io; use std::path::Path; -const BLOCK_START: &str = ""; -const BLOCK_END: &str = ""; -const SECTION_HEADING: &str = "## Connected Accounts"; +// ── Legacy connected-accounts constants (kept for internal helpers) ─────────── + +const CA_BLOCK: &str = "connected-accounts"; +const CA_HEADING: &str = "## Connected Accounts"; const FILE_HEADER: &str = "# User Profile\n"; +/// All managed block names, in the order they are appended when a new +/// `PROFILE.md` is created. +pub const BLOCKS: &[&str] = &[ + "connected-accounts", // written by provider path (merge_provider_into_profile_md) + "style", + "identity", + "tooling", + "vetoes", + "goals", +]; + +// ── Public API ──────────────────────────────────────────────────────────────── + /// Upsert the per-toolkit bullet for `profile` inside the managed -/// Connected Accounts block of `{workspace_dir}/PROFILE.md`. +/// `connected-accounts` block of `{workspace_dir}/PROFILE.md`. /// -/// Creates the file with a `# User Profile` header if it does not -/// exist. Idempotent — re-connecting the same toolkit replaces the -/// existing bullet rather than duplicating it. +/// Creates the file with a `# User Profile` header if it does not exist. +/// Idempotent — re-connecting the same toolkit replaces the existing +/// bullet rather than duplicating it. pub fn merge_provider_into_profile_md( workspace_dir: &Path, profile: &ProviderUserProfile, @@ -45,9 +63,7 @@ pub fn merge_provider_into_profile_md( return Ok(()); } // Require a real connection_id so the bullet keys match what the - // disconnect path (`composio_delete_connection`) will look up. A - // synthetic "default" fallback would orphan bullets when the - // connection is removed. + // disconnect path (`composio_delete_connection`) will look up. let identifier = profile .connection_id .as_deref() @@ -64,9 +80,8 @@ pub fn merge_provider_into_profile_md( } }; - let bullet = match render_bullet(&toolkit, &identifier, profile) { + let bullet = match render_provider_bullet(&toolkit, &identifier, profile) { Some(b) => b, - // No non-empty fields — nothing worth writing. None => return Ok(()), }; @@ -80,7 +95,7 @@ pub fn merge_provider_into_profile_md( Err(e) => return Err(e), }; - let updated = upsert_bullet(&existing, &toolkit, &identifier, &bullet); + let updated = upsert_provider_bullet(&existing, &toolkit, &identifier, &bullet); fs::write(&path, updated)?; tracing::debug!( target_file = "PROFILE.md", @@ -92,9 +107,8 @@ pub fn merge_provider_into_profile_md( } /// Remove the per-toolkit bullet for `(source, identifier)` from the -/// managed Connected Accounts block. If the block becomes empty as a -/// result, the whole block is dropped. Missing file or missing block -/// are no-ops. +/// managed Connected Accounts block. If the block becomes empty the whole +/// block is dropped. Missing file or missing block are no-ops. pub fn remove_provider_from_profile_md( workspace_dir: &Path, source: &str, @@ -111,7 +125,7 @@ pub fn remove_provider_from_profile_md( if toolkit.is_empty() || identifier.is_empty() { return Ok(()); } - let updated = remove_bullet(&existing, &toolkit, &identifier); + let updated = remove_provider_bullet(&existing, &toolkit, &identifier); if updated != existing { fs::write(&path, updated)?; tracing::debug!( @@ -124,11 +138,56 @@ pub fn remove_provider_from_profile_md( Ok(()) } -// ── Internals ──────────────────────────────────────────────────────── +/// Upsert a generic managed block. +/// +/// * `block_name` — one of [`BLOCKS`] (e.g. `"style"`, `"identity"`). +/// * `section_heading` — heading rendered inside the block (e.g. `"## Style"`). +/// * `body_markdown` — pre-rendered content (bullets, prose). Must not +/// contain the block markers themselves. +/// +/// Creates `PROFILE.md` if it does not exist. If the block is absent it is +/// appended at the end of the file. If the block exists its body is replaced +/// in-place — content outside the markers is left byte-for-byte untouched. +/// +/// An empty `body_markdown` renders a `*(no entries yet)*` placeholder +/// instead of deleting the block; this preserves the block's position for the +/// next write. +/// +/// Idempotent: calling with the same inputs twice produces the same file. +pub fn replace_managed_block( + workspace_dir: &Path, + block_name: &str, + section_heading: &str, + body_markdown: String, +) -> io::Result<()> { + let path = workspace_dir.join("PROFILE.md"); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } -/// Build the markdown bullet for one provider connection. Returns -/// `None` if the profile carries no usable fields. -fn render_bullet(toolkit: &str, identifier: &str, profile: &ProviderUserProfile) -> Option { + let existing = match fs::read_to_string(&path) { + Ok(s) => s, + Err(e) if e.kind() == io::ErrorKind::NotFound => String::new(), + Err(e) => return Err(e), + }; + + let updated = upsert_block(&existing, block_name, section_heading, &body_markdown); + fs::write(&path, updated)?; + tracing::debug!( + block_name = %block_name, + "[composio:profile_md] replaced managed block '{}' in PROFILE.md", + block_name + ); + Ok(()) +} + +// ── Connected-accounts internals ────────────────────────────────────────────── + +fn render_provider_bullet( + toolkit: &str, + identifier: &str, + profile: &ProviderUserProfile, +) -> Option { let mut fields: Vec = Vec::new(); if let Some(v) = profile.display_name.as_deref().map(sanitize) { if !v.is_empty() { @@ -153,8 +212,6 @@ fn render_bullet(toolkit: &str, identifier: &str, profile: &ProviderUserProfile) if fields.is_empty() { return None; } - // Stable per-(toolkit,identifier) marker so we can locate this - // bullet on later upserts even if the rendered text changes. let marker = bullet_marker(toolkit, identifier); Some(format!( "- {marker} **{title}** ({identifier}): {fields}", @@ -168,10 +225,12 @@ fn bullet_marker(toolkit: &str, identifier: &str) -> String { format!("") } -/// Insert or replace `bullet` inside the managed block. -fn upsert_bullet(existing: &str, toolkit: &str, identifier: &str, bullet: &str) -> String { +/// Insert or replace `bullet` inside the connected-accounts managed block. +fn upsert_provider_bullet(existing: &str, toolkit: &str, identifier: &str, bullet: &str) -> String { let marker = bullet_marker(toolkit, identifier); - let (prefix, block_body, suffix) = split_block(existing); + let start_tag = block_start(CA_BLOCK); + let end_tag = block_end(CA_BLOCK); + let (prefix, block_body, suffix) = split_any_block(existing, &start_tag, &end_tag); let mut lines: Vec = block_body .lines() @@ -187,20 +246,21 @@ fn upsert_bullet(existing: &str, toolkit: &str, identifier: &str, bullet: &str) bullets.sort(); let block = format!( - "{BLOCK_START}\n{SECTION_HEADING}\n\n{body}\n{BLOCK_END}", + "{start_tag}\n{CA_HEADING}\n\n{body}\n{end_tag}", body = bullets.join("\n") ); assemble(&prefix, &block, &suffix) } -/// Remove the bullet matching `(toolkit, identifier)` from the managed -/// block. Drops the block entirely if no bullets remain. -fn remove_bullet(existing: &str, toolkit: &str, identifier: &str) -> String { +/// Remove the bullet matching `(toolkit, identifier)` from the connected- +/// accounts managed block. Drops the block entirely if no bullets remain. +fn remove_provider_bullet(existing: &str, toolkit: &str, identifier: &str) -> String { let marker = bullet_marker(toolkit, identifier); - let (prefix, block_body, suffix) = split_block(existing); + let start_tag = block_start(CA_BLOCK); + let end_tag = block_end(CA_BLOCK); + let (prefix, block_body, suffix) = split_any_block(existing, &start_tag, &end_tag); if block_body.is_empty() && prefix == existing { - // No managed block present. return existing.to_string(); } let bullets: Vec = block_body @@ -209,27 +269,70 @@ fn remove_bullet(existing: &str, toolkit: &str, identifier: &str) -> String { .map(|l| l.to_string()) .collect(); if bullets.is_empty() { - // Drop the entire block. return assemble(&prefix, "", &suffix); } let block = format!( - "{BLOCK_START}\n{SECTION_HEADING}\n\n{body}\n{BLOCK_END}", + "{start_tag}\n{CA_HEADING}\n\n{body}\n{end_tag}", body = bullets.join("\n") ); assemble(&prefix, &block, &suffix) } -/// Split the file into `(prefix, block_body, suffix)` around the -/// managed block. Bytes outside the markers are returned verbatim so -/// the caller can preserve user-authored whitespace, indentation, and -/// trailing newlines exactly. If no block is present, `prefix` is the -/// full file and `block_body` / `suffix` are empty. -fn split_block(existing: &str) -> (String, String, String) { - if let (Some(start), Some(end)) = (existing.find(BLOCK_START), existing.find(BLOCK_END)) { +// ── Generic managed-block helpers ───────────────────────────────────────────── + +/// Build the start marker for `block_name`. +pub fn block_start(block_name: &str) -> String { + format!("") +} + +/// Build the end marker for `block_name`. +pub fn block_end(block_name: &str) -> String { + format!("") +} + +/// Insert or replace a generic managed block in `existing`. +/// +/// If the block is absent it is appended. If it exists its body (between the +/// markers) is replaced. Content outside the markers is returned unchanged. +fn upsert_block( + existing: &str, + block_name: &str, + section_heading: &str, + body_markdown: &str, +) -> String { + let start_tag = block_start(block_name); + let end_tag = block_end(block_name); + + let body = if body_markdown.trim().is_empty() { + "*(no entries yet)*".to_string() + } else { + body_markdown.to_string() + }; + + let block = format!("{start_tag}\n{section_heading}\n\n{body}\n\n{end_tag}"); + + let (prefix, _old_body, suffix) = split_any_block(existing, &start_tag, &end_tag); + + if prefix == existing { + // Block was absent — append. + assemble(existing, &block, "") + } else { + assemble(&prefix, &block, &suffix) + } +} + +/// Split `existing` around the markers `[start_tag, end_tag]`. +/// +/// Returns `(prefix, block_body, suffix)`. If no block is present, +/// `prefix` is the full string and `block_body` / `suffix` are empty. +/// `block_body` is the content *between* the markers (excluding the +/// markers themselves). +fn split_any_block(existing: &str, start_tag: &str, end_tag: &str) -> (String, String, String) { + if let (Some(start), Some(end)) = (existing.find(start_tag), existing.find(end_tag)) { if end > start { let prefix = existing[..start].to_string(); - let body = existing[start + BLOCK_START.len()..end].to_string(); - let suffix_start = end + BLOCK_END.len(); + let body = existing[start + start_tag.len()..end].to_string(); + let suffix_start = end + end_tag.len(); let suffix = existing[suffix_start..].to_string(); return (prefix, body, suffix); } @@ -237,25 +340,19 @@ fn split_block(existing: &str) -> (String, String, String) { (existing.to_string(), String::new(), String::new()) } -/// Assemble `prefix + block + suffix`, preserving the user-authored -/// bytes in `prefix` and `suffix` verbatim. We only normalize the -/// newline separators *immediately adjacent* to the managed block — -/// the bytes we own — to keep one blank line on each boundary. +/// Assemble `prefix + block + suffix`, normalising the newlines immediately +/// adjacent to the managed block while leaving the user's bytes elsewhere +/// untouched. fn assemble(prefix: &str, block: &str, suffix: &str) -> String { if block.is_empty() { - // Removing the block entirely. Strip the newlines we previously - // added on each side of the block, but leave the rest of the - // user's content untouched. + // Removing the block entirely. let p = prefix.trim_end_matches('\n'); let s = suffix.trim_start_matches('\n'); let mut out = String::with_capacity(p.len() + s.len() + 2); out.push_str(p); if !p.is_empty() { - // Keep one trailing newline on the prefix. out.push('\n'); if !s.is_empty() { - // Plus a blank-line separator before whatever the user - // had after the block. out.push('\n'); } } @@ -268,34 +365,35 @@ fn assemble(prefix: &str, block: &str, suffix: &str) -> String { let mut out = String::new(); if prefix.trim().is_empty() { - // Empty / whitespace-only file → seed with a friendly header so - // the agent prompt loader has a sensible top of the file. + // Seed with a header on first creation. out.push_str(FILE_HEADER); out.push('\n'); } else { - // Preserve user prefix bytes verbatim, then ensure exactly one - // blank line before the block. let p = prefix.trim_end_matches('\n'); out.push_str(p); out.push_str("\n\n"); } out.push_str(block); - // The block string we emit doesn't include a trailing newline. if suffix.is_empty() { out.push('\n'); } else { - // Drop any newlines we previously inserted between block and - // suffix; preserve the rest of the user's bytes. let s = suffix.trim_start_matches('\n'); - out.push_str("\n\n"); - out.push_str(s); - if !out.ends_with('\n') { + if s.is_empty() { + // Suffix was only whitespace — end with single newline, no blank line. out.push('\n'); + } else { + out.push_str("\n\n"); + out.push_str(s); + if !out.ends_with('\n') { + out.push('\n'); + } } } out } +// ── Token / string helpers ──────────────────────────────────────────────────── + fn normalize_token(raw: &str) -> String { let mut out = String::with_capacity(raw.len()); for ch in raw.chars() { @@ -322,11 +420,15 @@ fn sanitize(raw: &str) -> String { replaced.split_whitespace().collect::>().join(" ") } +// ── Tests ───────────────────────────────────────────────────────────────────── + #[cfg(test)] mod tests { use super::*; use tempfile::TempDir; + // ── merge_provider_into_profile_md (legacy API, unchanged) ─────────────── + fn sample(toolkit: &str, conn: &str) -> ProviderUserProfile { ProviderUserProfile { toolkit: toolkit.into(), @@ -346,12 +448,14 @@ mod tests { merge_provider_into_profile_md(tmp.path(), &sample("gmail", "c-1")).unwrap(); let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); assert!(body.starts_with("# User Profile"), "body was:\n{body}"); - assert!(body.contains(BLOCK_START)); - assert!(body.contains(SECTION_HEADING)); + let start = block_start(CA_BLOCK); + let end = block_end(CA_BLOCK); + assert!(body.contains(&start)); + assert!(body.contains(CA_HEADING)); assert!(body.contains("**Gmail** (c-1):")); assert!(body.contains("jane@example.com")); assert!(body.contains("@janedoe")); - assert!(body.contains(BLOCK_END)); + assert!(body.contains(&end)); } #[test] @@ -376,8 +480,10 @@ mod tests { let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); assert!(body.contains("acct:gmail:c-1")); assert!(body.contains("acct:twitter:c-2")); - assert_eq!(body.matches(BLOCK_START).count(), 1); - assert_eq!(body.matches(BLOCK_END).count(), 1); + let start = block_start(CA_BLOCK); + let end = block_end(CA_BLOCK); + assert_eq!(body.matches(&start).count(), 1); + assert_eq!(body.matches(&end).count(), 1); } #[test] @@ -431,8 +537,10 @@ mod tests { merge_provider_into_profile_md(tmp.path(), &sample("gmail", "c-1")).unwrap(); remove_provider_from_profile_md(tmp.path(), "gmail", "c-1").unwrap(); let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); - assert!(!body.contains(BLOCK_START), "block remained:\n{body}"); - assert!(!body.contains(BLOCK_END)); + let start = block_start(CA_BLOCK); + let end = block_end(CA_BLOCK); + assert!(!body.contains(&start), "block remained:\n{body}"); + assert!(!body.contains(&end)); assert!(body.starts_with("# User Profile")); } @@ -457,8 +565,6 @@ mod tests { extras: serde_json::Value::Null, }; merge_provider_into_profile_md(tmp.path(), &p).unwrap(); - // No file written — without a connection_id we'd orphan the - // bullet at disconnect time. assert!(!tmp.path().join("PROFILE.md").exists()); } @@ -466,23 +572,20 @@ mod tests { fn preserves_indentation_and_blank_lines_around_block() { let tmp = TempDir::new().unwrap(); let path = tmp.path().join("PROFILE.md"); - // User-authored content on both sides of where the block will - // land, with intentional blank lines and trailing whitespace. let original = "# User Profile\n\n indented bio line\n\n## Notes\n- alpha\n- beta\n\n"; fs::write(&path, original).unwrap(); merge_provider_into_profile_md(tmp.path(), &sample("gmail", "c-1")).unwrap(); let body = fs::read_to_string(&path).unwrap(); - // User content unchanged byte-for-byte. assert!(body.contains(" indented bio line")); assert!(body.contains("## Notes\n- alpha\n- beta")); - // Block landed somewhere. - assert!(body.contains(BLOCK_START) && body.contains(BLOCK_END)); - // Now remove and verify the user content is still intact. + let start = block_start(CA_BLOCK); + let end = block_end(CA_BLOCK); + assert!(body.contains(&start) && body.contains(&end)); remove_provider_from_profile_md(tmp.path(), "gmail", "c-1").unwrap(); let after = fs::read_to_string(&path).unwrap(); assert!(after.contains(" indented bio line")); assert!(after.contains("## Notes\n- alpha\n- beta")); - assert!(!after.contains(BLOCK_START)); + assert!(!after.contains(&start)); } #[test] @@ -491,4 +594,125 @@ mod tests { assert_eq!(sanitize("a | b"), "a / b"); assert_eq!(sanitize(" multi space "), "multi space"); } + + // ── replace_managed_block ───────────────────────────────────────────────── + + #[test] + fn replace_managed_block_creates_file_if_missing() { + let tmp = TempDir::new().unwrap(); + replace_managed_block( + tmp.path(), + "style", + "## Style", + "- **verbosity**: terse".into(), + ) + .unwrap(); + let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); + assert!(body.contains("# User Profile"), "missing header:\n{body}"); + assert!(body.contains(&block_start("style"))); + assert!(body.contains("## Style")); + assert!(body.contains("- **verbosity**: terse")); + assert!(body.contains(&block_end("style"))); + } + + #[test] + fn replace_managed_block_appends_block_when_absent() { + let tmp = TempDir::new().unwrap(); + let path = tmp.path().join("PROFILE.md"); + fs::write(&path, "# User Profile\n\nSome existing text.\n").unwrap(); + replace_managed_block( + tmp.path(), + "identity", + "## Identity", + "- **name**: Alice".into(), + ) + .unwrap(); + let body = fs::read_to_string(&path).unwrap(); + // Existing content preserved. + assert!(body.contains("Some existing text.")); + // New block appended. + assert!(body.contains(&block_start("identity"))); + assert!(body.contains("## Identity")); + assert!(body.contains("- **name**: Alice")); + } + + #[test] + fn replace_managed_block_replaces_body_in_place() { + let tmp = TempDir::new().unwrap(); + replace_managed_block( + tmp.path(), + "style", + "## Style", + "- **verbosity**: verbose".into(), + ) + .unwrap(); + replace_managed_block( + tmp.path(), + "style", + "## Style", + "- **verbosity**: terse".into(), + ) + .unwrap(); + let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); + assert!(body.contains("terse")); + assert!(!body.contains("verbose")); + // Only one start marker. + assert_eq!(body.matches(&block_start("style")).count(), 1); + } + + #[test] + fn replace_managed_block_preserves_other_blocks_and_user_text() { + let tmp = TempDir::new().unwrap(); + // Write two blocks. + replace_managed_block( + tmp.path(), + "style", + "## Style", + "- **verbosity**: terse".into(), + ) + .unwrap(); + replace_managed_block( + tmp.path(), + "identity", + "## Identity", + "- **name**: Bob".into(), + ) + .unwrap(); + // Update only style. + replace_managed_block( + tmp.path(), + "style", + "## Style", + "- **verbosity**: verbose".into(), + ) + .unwrap(); + let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); + // Identity block untouched. + assert!(body.contains("- **name**: Bob")); + // Style updated. + assert!(body.contains("verbose")); + assert!(!body.contains("terse")); + } + + #[test] + fn replace_managed_block_empty_body_renders_placeholder() { + let tmp = TempDir::new().unwrap(); + replace_managed_block(tmp.path(), "goals", "## Goals", String::new()).unwrap(); + let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); + assert!(body.contains("*(no entries yet)*")); + // Block markers still present. + assert!(body.contains(&block_start("goals"))); + assert!(body.contains(&block_end("goals"))); + } + + #[test] + fn replace_managed_block_idempotent_on_repeat_invocation() { + let tmp = TempDir::new().unwrap(); + let content = "- **verbosity**: terse".to_string(); + replace_managed_block(tmp.path(), "style", "## Style", content.clone()).unwrap(); + let body1 = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); + replace_managed_block(tmp.path(), "style", "## Style", content).unwrap(); + let body2 = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); + assert_eq!(body1, body2, "second write should be idempotent"); + } } diff --git a/src/openhuman/config/schema/learning.rs b/src/openhuman/config/schema/learning.rs index 54960ff16..0d4f28689 100644 --- a/src/openhuman/config/schema/learning.rs +++ b/src/openhuman/config/schema/learning.rs @@ -46,6 +46,30 @@ pub struct LearningConfig { /// Minimum tool calls in a turn to trigger reflection. Default: 1. #[serde(default = "default_min_turn_complexity")] pub min_turn_complexity: usize, + + /// Pipe agent chat turns into the memory tree as `source="conversations:agent"`. + /// + /// When enabled, [`ArchivistHook`] calls `tree::ingest::ingest_chat` with a + /// two-message [`ChatBatch`] (user + assistant) after every completed turn. + /// Tool-call JSON is stripped from the assistant message before ingest — + /// only the prose response reaches the tree. + /// + /// Default: true. Disable to stop agent chat from flowing into the tree + /// without affecting the episodic-log write path. + #[serde(default = "default_true")] + pub chat_to_tree_enabled: bool, + + /// Enable the stability detector rebuild cycle. Default: true. + #[serde(default = "default_true")] + pub stability_detector_enabled: bool, + + /// How often the periodic rebuild loop runs in seconds. Default: 1800 (30 minutes). + #[serde(default = "default_rebuild_interval_secs")] + pub rebuild_interval_secs: u64, +} + +fn default_rebuild_interval_secs() -> u64 { + 1800 } fn default_true() -> bool { @@ -70,6 +94,9 @@ impl Default for LearningConfig { reflection_source: ReflectionSource::default(), max_reflections_per_session: default_max_reflections(), min_turn_complexity: default_min_turn_complexity(), + chat_to_tree_enabled: default_true(), + stability_detector_enabled: default_true(), + rebuild_interval_secs: default_rebuild_interval_secs(), } } } diff --git a/src/openhuman/learning/cache.rs b/src/openhuman/learning/cache.rs new file mode 100644 index 000000000..fa4c067a8 --- /dev/null +++ b/src/openhuman/learning/cache.rs @@ -0,0 +1,126 @@ +//! `FacetCache` — thin wrapper over `user_profile_facets` for Phase 3. +//! +//! Provides typed read/write access to the facet table with class-aware helpers. +//! The stability detector uses this to persist the result of each rebuild cycle. +//! Prompt sections use [`FacetCache::list_active`] to read the ambient cache. + +use parking_lot::Mutex; +use rusqlite::Connection; +use std::sync::Arc; + +use crate::openhuman::learning::candidate::FacetClass; +use crate::openhuman::memory::store::profile::{self, FacetState, ProfileFacet, UserState}; + +/// Thin wrapper around the `user_profile` table. +/// +/// All methods delegate to the standalone helpers in +/// `memory::store::unified::profile`. This type exists so callers +/// (stability detector, prompt sections, RPCs) share a single typed +/// entry-point that can be constructed from any `Arc>`. +pub struct FacetCache { + conn: Arc>, +} + +impl FacetCache { + pub fn new(conn: Arc>) -> Self { + Self { conn } + } + + /// List all facets with `state = 'active'`, ordered by stability descending. + pub fn list_active(&self) -> anyhow::Result> { + profile::profile_select_active(&self.conn) + } + + /// List all facets (all states), ordered by stability descending. + pub fn list_all(&self) -> anyhow::Result> { + profile::profile_select_all(&self.conn) + } + + /// List active facets belonging to a specific class. + /// + /// Class is determined by the `key` prefix before the first `/`. + pub fn list_by_class(&self, class: FacetClass) -> anyhow::Result> { + let prefix = format!("{}/", class_prefix(class)); + let all = self.list_active()?; + Ok(all + .into_iter() + .filter(|f| f.key.starts_with(&prefix)) + .collect()) + } + + /// Fetch a single facet by its full key (e.g. `"style/verbosity"`). + pub fn get(&self, key: &str) -> anyhow::Result> { + profile::profile_get_by_key(&self.conn, key) + } + + /// Upsert a fully-formed facet row (rebuild path). + pub fn upsert(&self, facet: &ProfileFacet) -> anyhow::Result<()> { + profile::profile_upsert_full(&self.conn, facet) + } + + /// Override the `user_state` of a facet. + /// + /// Returns `Ok(true)` if a row was found and updated. + pub fn set_user_state(&self, key: &str, user_state: UserState) -> anyhow::Result { + profile::profile_set_user_state(&self.conn, key, user_state) + } + + /// Delete a facet by key. Returns `true` if a row was removed. + pub fn delete(&self, key: &str) -> anyhow::Result { + profile::profile_delete_by_key(&self.conn, key) + } + + /// Delete all `Dropped`-state facets whose stability is below `threshold`. + /// + /// Pinned facets are never deleted. Returns the number of rows removed. + pub fn drop_below_threshold(&self, threshold: f64) -> anyhow::Result { + profile::profile_delete_below_threshold(&self.conn, threshold) + } +} + +// ── Class ↔ key utilities ───────────────────────────────────────────────────── + +/// Extract the [`FacetClass`] from a full key string (e.g. `"style/verbosity"` → `Style`). +/// +/// Returns `None` for keys that don't have a recognised class prefix. +pub fn class_from_key(key: &str) -> Option { + let prefix = key.split('/').next()?; + match prefix { + "style" => Some(FacetClass::Style), + "identity" => Some(FacetClass::Identity), + "tooling" => Some(FacetClass::Tooling), + "veto" => Some(FacetClass::Veto), + "goal" => Some(FacetClass::Goal), + "channel" => Some(FacetClass::Channel), + _ => None, + } +} + +/// Build a full key from a class and a suffix (e.g. `(Style, "verbosity")` → `"style/verbosity"`). +pub fn key_with_class(class: FacetClass, suffix: &str) -> String { + format!("{}/{suffix}", class_prefix(class)) +} + +/// Return the canonical key prefix for a [`FacetClass`]. +pub fn class_prefix(class: FacetClass) -> &'static str { + match class { + FacetClass::Style => "style", + FacetClass::Identity => "identity", + FacetClass::Tooling => "tooling", + FacetClass::Veto => "veto", + FacetClass::Goal => "goal", + FacetClass::Channel => "channel", + } +} + +// ── Facet state enum re-export (convenience for callers of this module) ─────── + +pub use crate::openhuman::memory::store::profile::{ + FacetState as CacheFacetState, UserState as CacheUserState, +}; + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +#[path = "cache_tests.rs"] +mod tests; diff --git a/src/openhuman/learning/cache_tests.rs b/src/openhuman/learning/cache_tests.rs new file mode 100644 index 000000000..618fa7d75 --- /dev/null +++ b/src/openhuman/learning/cache_tests.rs @@ -0,0 +1,250 @@ +//! Tests for `learning::cache::FacetCache`. + +use parking_lot::Mutex; +use rusqlite::Connection; +use std::sync::Arc; + +use super::*; +use crate::openhuman::learning::candidate::{EvidenceRef, FacetClass}; +use crate::openhuman::memory::store::profile::{ + FacetState, FacetType, ProfileFacet, UserState, PROFILE_INIT_SQL, +}; + +fn make_cache() -> FacetCache { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(PROFILE_INIT_SQL).unwrap(); + FacetCache::new(Arc::new(Mutex::new(conn))) +} + +fn stub_facet(id: &str, key: &str, value: &str, state: FacetState, stability: f64) -> ProfileFacet { + ProfileFacet { + facet_id: id.into(), + facet_type: FacetType::Preference, + key: key.into(), + value: value.into(), + confidence: 0.8, + evidence_count: 2, + source_segment_ids: None, + first_seen_at: 1000.0, + last_seen_at: 1200.0, + state, + stability, + user_state: UserState::Auto, + evidence_refs: vec![], + class: None, + cue_families: None, + } +} + +// ── upsert_then_list_active ─────────────────────────────────────────────────── + +#[test] +fn upsert_then_list_active() { + let cache = make_cache(); + + cache + .upsert(&stub_facet( + "f1", + "style/verbosity", + "terse", + FacetState::Active, + 1.8, + )) + .unwrap(); + cache + .upsert(&stub_facet( + "f2", + "style/tone", + "formal", + FacetState::Provisional, + 0.8, + )) + .unwrap(); + + let active = cache.list_active().unwrap(); + assert_eq!(active.len(), 1, "only Active state should be listed"); + assert_eq!(active[0].key, "style/verbosity"); +} + +// ── class_from_key_parses_known_classes ─────────────────────────────────────── + +#[test] +fn class_from_key_parses_known_classes() { + assert_eq!(class_from_key("style/verbosity"), Some(FacetClass::Style)); + assert_eq!(class_from_key("identity/name"), Some(FacetClass::Identity)); + assert_eq!( + class_from_key("tooling/package_manager"), + Some(FacetClass::Tooling) + ); + assert_eq!( + class_from_key("veto/no_sports_updates"), + Some(FacetClass::Veto) + ); + assert_eq!(class_from_key("goal/learn_rust"), Some(FacetClass::Goal)); + assert_eq!(class_from_key("channel/slack"), Some(FacetClass::Channel)); + assert_eq!(class_from_key("unknown/foo"), None); + assert_eq!(class_from_key("no_slash"), None); +} + +// ── set_user_state_pinned_persists ──────────────────────────────────────────── + +#[test] +fn set_user_state_pinned_persists() { + let cache = make_cache(); + + cache + .upsert(&stub_facet( + "f-pin", + "identity/name", + "Alice", + FacetState::Active, + 2.0, + )) + .unwrap(); + + let updated = cache + .set_user_state("identity/name", UserState::Pinned) + .unwrap(); + assert!(updated, "row should exist and be updated"); + + let f = cache.get("identity/name").unwrap().unwrap(); + assert_eq!(f.user_state, UserState::Pinned); +} + +// ── drop_below_threshold_removes_facets ─────────────────────────────────────── + +#[test] +fn drop_below_threshold_removes_facets() { + let cache = make_cache(); + + cache + .upsert(&stub_facet( + "f-low", + "style/dropped_one", + "x", + FacetState::Dropped, + 0.1, + )) + .unwrap(); + cache + .upsert(&stub_facet( + "f-keep", + "style/active_one", + "y", + FacetState::Active, + 0.1, // low stability but Active state — should NOT be deleted + )) + .unwrap(); + cache + .upsert(&stub_facet( + "f-pinned-drop", + "style/pinned_one", + "z", + FacetState::Dropped, + 0.1, + )) + .and_then(|_| cache.set_user_state("style/pinned_one", UserState::Pinned)) + .unwrap(); + + let removed = cache.drop_below_threshold(0.3).unwrap(); + assert_eq!( + removed, 1, + "only the non-pinned Dropped row should be removed" + ); + + // Active and Pinned rows survive. + let all = cache.list_all().unwrap(); + assert_eq!(all.len(), 2); +} + +// ── list_by_class_filters_correctly ─────────────────────────────────────────── + +#[test] +fn list_by_class_filters_correctly() { + let cache = make_cache(); + + for (id, key, val) in [ + ("f-s1", "style/verbosity", "terse"), + ("f-s2", "style/tone", "formal"), + ("f-i1", "identity/name", "Alice"), + ] { + cache + .upsert(&stub_facet(id, key, val, FacetState::Active, 1.6)) + .unwrap(); + } + + let style = cache.list_by_class(FacetClass::Style).unwrap(); + assert_eq!(style.len(), 2); + assert!(style.iter().all(|f| f.key.starts_with("style/"))); + + let identity = cache.list_by_class(FacetClass::Identity).unwrap(); + assert_eq!(identity.len(), 1); + assert_eq!(identity[0].key, "identity/name"); + + let tooling = cache.list_by_class(FacetClass::Tooling).unwrap(); + assert!(tooling.is_empty()); +} + +// ── key_with_class helper ───────────────────────────────────────────────────── + +#[test] +fn key_with_class_produces_prefixed_key() { + assert_eq!( + key_with_class(FacetClass::Style, "verbosity"), + "style/verbosity" + ); + assert_eq!( + key_with_class(FacetClass::Tooling, "package_manager"), + "tooling/package_manager" + ); +} + +// ── Evidence refs round-trip ────────────────────────────────────────────────── + +#[test] +fn evidence_refs_survive_upsert_round_trip() { + let cache = make_cache(); + let mut f = stub_facet("f-ev", "identity/email", "a@b.com", FacetState::Active, 2.0); + f.evidence_refs = vec![ + EvidenceRef::Provider { + toolkit: "gmail".into(), + connection_id: "c-1".into(), + field: "email".into(), + }, + EvidenceRef::Episodic { episodic_id: 7 }, + ]; + cache.upsert(&f).unwrap(); + + let loaded = cache.get("identity/email").unwrap().unwrap(); + assert_eq!(loaded.evidence_refs.len(), 2); + assert_eq!( + loaded.evidence_refs[0], + EvidenceRef::Provider { + toolkit: "gmail".into(), + connection_id: "c-1".into(), + field: "email".into(), + } + ); +} + +// ── delete helper ───────────────────────────────────────────────────────────── + +#[test] +fn delete_removes_facet_by_key() { + let cache = make_cache(); + cache + .upsert(&stub_facet( + "f-del", + "goal/learn_rust", + "learn Rust", + FacetState::Active, + 1.5, + )) + .unwrap(); + + let deleted = cache.delete("goal/learn_rust").unwrap(); + assert!(deleted); + + let loaded = cache.get("goal/learn_rust").unwrap(); + assert!(loaded.is_none()); +} diff --git a/src/openhuman/learning/candidate.rs b/src/openhuman/learning/candidate.rs new file mode 100644 index 000000000..95285df25 --- /dev/null +++ b/src/openhuman/learning/candidate.rs @@ -0,0 +1,380 @@ +//! Learning candidate buffer — Phase 1 of issue #566. +//! +//! Defines the taxonomy types ([`FacetClass`], [`CueFamily`], [`EvidenceRef`]), +//! the unit-of-work [`LearningCandidate`], and a thread-safe ring-buffer +//! [`Buffer`] that collects candidates emitted by producers (Phase 2) before +//! they are consumed by the stability detector (Phase 3). +//! +//! The buffer is bounded: when full it evicts the oldest entry (FIFO overflow). +//! A global singleton is exposed via [`global()`]; individual tests may +//! construct their own [`Buffer`] with `Buffer::new(capacity)`. + +use std::collections::VecDeque; +use std::sync::OnceLock; + +use parking_lot::Mutex; +use serde::{Deserialize, Serialize}; + +// ── Taxonomy ──────────────────────────────────────────────────────────────── + +/// Six-class taxonomy of what the cache can hold. +/// +/// Keys are stored with a class prefix, e.g. `style/verbosity` or +/// `tooling/package_manager`. The class determines the half-life and +/// class budget used by the stability detector (Phase 3). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FacetClass { + /// Communication style preferences — verbosity, formality, code format. + Style, + /// Stable biographical facts — timezone, name, language, role. + Identity, + /// Developer toolchain preferences — package manager, editor, OS, language. + Tooling, + /// Hard user vetoes — things the user has explicitly rejected or forbidden. + Veto, + /// Active user goals or ongoing projects. + Goal, + /// Preferred communication channel or platform. + Channel, +} + +/// How a candidate signal was produced — determines the weight multiplier +/// applied in the stability formula. +/// +/// Higher-weight families contribute more strongly per evidence item. +/// The weights here are the canonical values from the Phase 1 plan: +/// `Explicit=1.0`, `Structural=0.9`, `Behavioral=0.7`, `Recurrence=0.6`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CueFamily { + /// Direct declaration of intent by the user (highest weight — 1.0). + /// + /// Examples: "I prefer pnpm", "my timezone is PST", "always use terse replies". + Explicit, + /// Inferred from structured file or provider metadata (weight 0.9). + /// + /// Examples: `package.json#packageManager`, Gmail display name, Slack workspace. + Structural, + /// Inferred by heuristics or LLM from observed behaviour (weight 0.7). + /// + /// Examples: rolling edit-window ratio, correction-repeat signal, reflection hook output. + Behavioral, + /// Materialized from recurrence statistics in the memory tree (weight 0.6). + /// + /// Examples: tree-topic hotness, source_weight per channel. + Recurrence, +} + +impl CueFamily { + /// Weight multiplier for this cue family in the stability formula. + /// + /// Phase 1 canonical values (matches the plan): + /// `Explicit=1.0`, `Structural=0.9`, `Behavioral=0.7`, `Recurrence=0.6`. + pub fn weight(self) -> f64 { + match self { + CueFamily::Explicit => 1.0, + CueFamily::Structural => 0.9, + CueFamily::Behavioral => 0.7, + CueFamily::Recurrence => 0.6, + } + } +} + +// ── Evidence reference ─────────────────────────────────────────────────────── + +/// A typed pointer back into the memory substrate from which a candidate was +/// derived. Used for provenance tracking, citation, and the `evidence_ids` +/// column in `user_profile_facets` (Phase 3+). +/// +/// Serialised with a `"type"` discriminator in snake_case so the JSON is +/// human-readable: `{"type":"episodic","episodic_id":42}`. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum EvidenceRef { + /// A single row in `episodic_log`. + Episodic { episodic_id: i64 }, + /// A contiguous window of rows in `episodic_log`. + EpisodicWindow { from_id: i64, to_id: i64 }, + /// A row in the tree-source summary table. + SourceSummary { summary_id: String }, + /// A node in `tree_topic`. + TreeTopic { topic_id: String }, + /// A chunk in `vector_chunks` associated with a document source. + DocumentChunk { source_id: String, chunk_id: String }, + /// A specific message in an email source. + EmailMessage { + source_id: String, + message_id: String, + }, + /// A field value from a connected provider (Composio toolkit). + Provider { + toolkit: String, + connection_id: String, + field: String, + }, + /// A tool call record within an episodic entry. + ToolCall { tool_name: String, episodic_id: i64 }, + /// A per-window weight from `tree_source`. + TreeSourceWeight { window_label: String }, +} + +// ── Learning candidate ─────────────────────────────────────────────────────── + +/// A single unit of learning evidence emitted by a producer and queued in the +/// [`Buffer`]. +/// +/// Each candidate asserts a specific `(class, key, value)` triple alongside +/// the evidence that backs it. The stability detector (Phase 3) aggregates +/// competing candidates for the same `(class, key)` pair and resolves them +/// into a single cache entry. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LearningCandidate { + /// Which facet class this evidence touches. + pub class: FacetClass, + /// Canonical slug key within the class, e.g. `"verbosity"`, `"package_manager"`. + /// + /// Convention: `snake_case`, lowercase, no class prefix (the class carries that). + pub key: String, + /// Canonical value string, e.g. `"terse"`, `"pnpm"`, `"UTC+5:30"`. + pub value: String, + /// How this candidate was produced. + pub cue_family: CueFamily, + /// Pointer to the backing evidence in the memory substrate. + pub evidence: EvidenceRef, + /// Source-provided confidence hint, `0.0..=1.0`. + /// + /// This is an initial hint; the stability detector will reweight it using + /// the cue-family weight and recency decay. + pub initial_confidence: f64, + /// When this candidate was observed, as seconds since the Unix epoch. + pub observed_at: f64, +} + +// ── Buffer ─────────────────────────────────────────────────────────────────── + +/// Thread-safe, bounded ring-buffer of [`LearningCandidate`] items. +/// +/// Backed by a `parking_lot::Mutex>`. When full +/// the oldest entry is evicted to make room (FIFO overflow). This keeps +/// memory bounded and naturally prioritises recent evidence. +/// +/// The global singleton has a default capacity of 1024. Tests should +/// construct their own buffer via [`Buffer::new`]. +pub struct Buffer { + inner: Mutex>, + capacity: usize, +} + +impl Buffer { + /// Create a new buffer with the given capacity. + /// + /// `capacity` must be ≥ 1. A capacity of zero would make every `push` + /// a no-op; callers should use a non-zero value. + pub fn new(capacity: usize) -> Self { + let cap = capacity.max(1); + Self { + inner: Mutex::new(VecDeque::with_capacity(cap)), + capacity: cap, + } + } + + /// Push a candidate onto the buffer. + /// + /// If the buffer is already at capacity, the oldest entry is evicted first + /// (FIFO overflow). This ensures the buffer always reflects the most recent + /// evidence. + pub fn push(&self, candidate: LearningCandidate) { + let mut guard = self.inner.lock(); + if guard.len() >= self.capacity { + guard.pop_front(); // evict oldest + } + guard.push_back(candidate); + } + + /// Drain all candidates from the buffer and return them in FIFO order. + /// + /// After this call the buffer is empty. + pub fn drain(&self) -> Vec { + let mut guard = self.inner.lock(); + guard.drain(..).collect() + } + + /// Clone all candidates without removing them. + /// + /// Useful for inspection or debugging. + pub fn peek(&self) -> Vec { + let guard = self.inner.lock(); + guard.iter().cloned().collect() + } + + /// Current number of candidates in the buffer. + pub fn len(&self) -> usize { + self.inner.lock().len() + } + + /// Returns `true` when the buffer holds no candidates. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Maximum number of candidates the buffer will hold. + pub fn capacity(&self) -> usize { + self.capacity + } +} + +// ── Global singleton ───────────────────────────────────────────────────────── + +static GLOBAL_BUFFER: OnceLock = OnceLock::new(); + +/// Return the global [`Buffer`] singleton. +/// +/// Initialised on first call with a default capacity of 1024. All producers +/// push into this buffer; the stability detector drains it. +pub fn global() -> &'static Buffer { + GLOBAL_BUFFER.get_or_init(|| Buffer::new(1024)) +} + +// ── Tests ──────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn now_secs() -> f64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64() + } + + fn make_candidate(value: &str) -> LearningCandidate { + LearningCandidate { + class: FacetClass::Style, + key: "verbosity".into(), + value: value.into(), + cue_family: CueFamily::Explicit, + evidence: EvidenceRef::Episodic { episodic_id: 1 }, + initial_confidence: 0.8, + observed_at: now_secs(), + } + } + + #[test] + fn push_then_drain_preserves_fifo_order() { + let buf = Buffer::new(10); + buf.push(make_candidate("a")); + buf.push(make_candidate("b")); + buf.push(make_candidate("c")); + + let drained = buf.drain(); + assert_eq!(drained.len(), 3); + assert_eq!(drained[0].value, "a"); + assert_eq!(drained[1].value, "b"); + assert_eq!(drained[2].value, "c"); + } + + #[test] + fn drain_empties_the_buffer() { + let buf = Buffer::new(10); + buf.push(make_candidate("x")); + buf.push(make_candidate("y")); + assert_eq!(buf.len(), 2); + + let _ = buf.drain(); + assert_eq!(buf.len(), 0); + assert!(buf.is_empty()); + } + + #[test] + fn bounded_capacity_evicts_oldest() { + let buf = Buffer::new(3); + buf.push(make_candidate("first")); + buf.push(make_candidate("second")); + buf.push(make_candidate("third")); + // Buffer is full — next push evicts "first" + buf.push(make_candidate("fourth")); + + assert_eq!(buf.len(), 3); + let items = buf.drain(); + assert_eq!(items[0].value, "second"); + assert_eq!(items[1].value, "third"); + assert_eq!(items[2].value, "fourth"); + } + + #[test] + fn peek_does_not_remove() { + let buf = Buffer::new(10); + buf.push(make_candidate("p")); + buf.push(make_candidate("q")); + + let peeked = buf.peek(); + assert_eq!(peeked.len(), 2); + // Buffer still holds the items + assert_eq!(buf.len(), 2); + + let drained = buf.drain(); + assert_eq!(drained[0].value, "p"); + assert_eq!(drained[1].value, "q"); + } + + #[test] + fn cue_family_weight_values() { + assert_eq!(CueFamily::Explicit.weight(), 1.0); + assert_eq!(CueFamily::Structural.weight(), 0.9); + assert_eq!(CueFamily::Behavioral.weight(), 0.7); + assert_eq!(CueFamily::Recurrence.weight(), 0.6); + } + + #[test] + fn roundtrip_serde_evidence_ref() { + let cases: Vec = vec![ + EvidenceRef::Episodic { episodic_id: 42 }, + EvidenceRef::EpisodicWindow { + from_id: 10, + to_id: 20, + }, + EvidenceRef::SourceSummary { + summary_id: "sum-abc".into(), + }, + EvidenceRef::TreeTopic { + topic_id: "topic-xyz".into(), + }, + EvidenceRef::DocumentChunk { + source_id: "notion:page1".into(), + chunk_id: "chunk-001".into(), + }, + EvidenceRef::EmailMessage { + source_id: "gmail:user@example.com".into(), + message_id: "".into(), + }, + EvidenceRef::Provider { + toolkit: "gmail".into(), + connection_id: "conn-1".into(), + field: "display_name".into(), + }, + EvidenceRef::ToolCall { + tool_name: "write_file".into(), + episodic_id: 99, + }, + EvidenceRef::TreeSourceWeight { + window_label: "2026-W18".into(), + }, + ]; + + for ev in &cases { + let json = serde_json::to_string(ev).expect("serialize failed"); + let back: EvidenceRef = serde_json::from_str(&json).expect("deserialize failed"); + assert_eq!(ev, &back, "round-trip failed for variant: {json}"); + } + } + + #[test] + fn global_returns_same_instance_across_calls() { + let a = global() as *const Buffer; + let b = global() as *const Buffer; + assert_eq!(a, b, "global() must return the same static instance"); + } +} diff --git a/src/openhuman/learning/extract/heuristics.rs b/src/openhuman/learning/extract/heuristics.rs new file mode 100644 index 000000000..5ac8a7955 --- /dev/null +++ b/src/openhuman/learning/extract/heuristics.rs @@ -0,0 +1,614 @@ +//! Deterministic heuristic detectors for Style and Veto candidates. +//! +//! Three detectors feed [`crate::openhuman::learning::candidate::global()`]: +//! +//! - **`LengthRatioDetector`** — emits `Style/verbosity` when the rolling +//! ratio of user-to-agent message lengths shifts significantly over ≥ 30 turns. +//! - **`EditWindowDetector`** — emits `Style/*` candidates when the user sends +//! a correction within 30 s of the previous agent reply. +//! - **`CorrectionRepeatDetector`** — promotes repeated (≥ 3×) correction cues to +//! `Veto/*` candidates. +//! +//! All three detectors advance via a single call to [`record_turn`], which is +//! intended to be called from [`crate::openhuman::learning::ReflectionHook`] or a +//! sibling post-turn hook. +//! +//! ## State lifetime +//! +//! State is per-session and bounded to 100 turns per session. A global `RwLock`- +//! guarded map keeps per-session [`RollingState`]. Sessions that exceed +//! `MAX_SESSION_TURNS` have their oldest turn evicted (FIFO) to stay bounded. + +use std::collections::{HashMap, HashSet, VecDeque}; +use std::sync::OnceLock; +use std::time::{SystemTime, UNIX_EPOCH}; + +use parking_lot::RwLock; + +use crate::openhuman::learning::candidate::{ + self, CueFamily, EvidenceRef, FacetClass, LearningCandidate, +}; + +// ── Constants ──────────────────────────────────────────────────────────────── + +/// Per-session turn capacity before FIFO eviction. +const MAX_SESSION_TURNS: usize = 100; + +/// Minimum number of turns before the length-ratio detector fires. +const LENGTH_RATIO_MIN_TURNS: usize = 30; + +/// Number of turns in each of the two comparison windows. +const LENGTH_RATIO_WINDOW: usize = 15; + +/// User-to-agent message length ratio below which we classify as "compressed". +const RATIO_COMPRESSED: f64 = 0.3; + +/// User-to-agent message length ratio above which we classify as "balanced". +const RATIO_BALANCED: f64 = 0.8; + +/// Ratio of the "earlier" window required to cross before we treat the shift as meaningful. +const RATIO_EARLIER_THRESHOLD: f64 = 0.5; + +/// Edit-window threshold: correction sent within 30 s of agent reply is "quick". +const EDIT_WINDOW_SECS: f64 = 30.0; + +/// Cooldown for EditWindowDetector per (key, value): 5 minutes. +const EDIT_COOLDOWN_SECS: f64 = 300.0; + +/// Number of repeated corrections before promoting to Veto. +const VETO_PROMOTION_THRESHOLD: usize = 3; + +// ── Per-turn state ──────────────────────────────────────────────────────────── + +/// A single turn entry in the rolling window. +#[derive(Clone, Debug)] +pub struct TurnEntry { + pub turn_id: String, + pub user_msg_len: usize, + pub agent_msg_len: usize, + /// Wall-clock seconds (epoch) when the user message arrived. + pub user_timestamp: f64, + /// Wall-clock seconds (epoch) when the agent finished replying. + pub agent_timestamp: f64, +} + +/// Per-session state for all three detectors. +#[derive(Default)] +pub struct RollingState { + pub turns: VecDeque, + /// Last emission per (key, value) to enforce cooldown / dedupe for length-ratio. + pub length_ratio_emitted: HashSet<(String, String)>, + /// Per (key, value) last emission timestamp for edit-window cooldown. + pub edit_cooldown: HashMap<(String, String), f64>, + /// Per correction-cue counter across the session. + pub correction_counts: HashMap, + /// Cues that have already been promoted to Veto (prevent double-emit). + pub veto_promoted: HashSet, +} + +// ── Global state map ───────────────────────────────────────────────────────── + +static SESSION_STATE: OnceLock>> = OnceLock::new(); + +fn session_state() -> &'static RwLock> { + SESSION_STATE.get_or_init(|| RwLock::new(HashMap::new())) +} + +// ── Public entry point ──────────────────────────────────────────────────────── + +/// Advance all three detectors for a completed turn and push any emitted +/// candidates to [`candidate::global()`]. +/// +/// # Arguments +/// +/// * `session_id` — stable session identifier used as the state map key. +/// * `turn_id` — stable identifier for the current turn (e.g. an episodic id +/// encoded as a string). +/// * `episodic_id` — numeric episodic log id used to build `EvidenceRef`. +/// * `user_message` — user's message text for edit-window scanning. +/// * `user_msg_len` — byte length of the user message. +/// * `agent_msg_len` — byte length of the assistant response. +/// * `user_timestamp` — epoch seconds when the user message arrived. +/// * `agent_timestamp` — epoch seconds when the agent replied. +/// * `prev_agent_at` — epoch seconds of the **previous** agent reply, used by +/// the edit-window detector to check whether the user responded within +/// `EDIT_WINDOW_SECS`. Pass `None` on the first turn of a session. +#[allow(clippy::too_many_arguments)] +pub fn record_turn( + session_id: &str, + turn_id: &str, + episodic_id: i64, + user_message: &str, + user_msg_len: usize, + agent_msg_len: usize, + user_timestamp: f64, + agent_timestamp: f64, + prev_agent_at: Option, +) { + let entry = TurnEntry { + turn_id: turn_id.to_string(), + user_msg_len, + agent_msg_len, + user_timestamp, + agent_timestamp, + }; + + let mut candidates: Vec = Vec::new(); + + { + let mut map = session_state().write(); + let state = map.entry(session_id.to_string()).or_default(); + + // FIFO eviction. + if state.turns.len() >= MAX_SESSION_TURNS { + state.turns.pop_front(); + } + state.turns.push_back(entry.clone()); + + // ── A. Length-ratio detector ────────────────────────────────────── + if state.turns.len() >= LENGTH_RATIO_MIN_TURNS { + let turns_slice: Vec<&TurnEntry> = state.turns.iter().collect(); + let n = turns_slice.len(); + let recent: &[&TurnEntry] = &turns_slice[n - LENGTH_RATIO_WINDOW..]; + let earlier: &[&TurnEntry] = &turns_slice + [n - LENGTH_RATIO_MIN_TURNS..n - LENGTH_RATIO_MIN_TURNS + LENGTH_RATIO_WINDOW]; + + let recent_ratio = mean_ratio(recent); + let earlier_ratio = mean_ratio(earlier); + + // Compressed: user messages shrunk relative to earlier. + if recent_ratio < RATIO_COMPRESSED + && earlier_ratio > RATIO_EARLIER_THRESHOLD + && !state + .length_ratio_emitted + .contains(&("verbosity".to_string(), "compressed".to_string())) + { + let first_id = state.turns.front().map(|t| t.turn_id.clone()); + let last_id = state.turns.back().map(|t| t.turn_id.clone()); + let from_id = first_id + .and_then(|s| s.parse::().ok()) + .unwrap_or(episodic_id); + let to_id = last_id + .and_then(|s| s.parse::().ok()) + .unwrap_or(episodic_id); + + tracing::debug!( + "[learning::extract::heuristics] length_ratio compressed session={} \ + recent_ratio={:.2} earlier_ratio={:.2}", + session_id, + recent_ratio, + earlier_ratio + ); + candidates.push(LearningCandidate { + class: FacetClass::Style, + key: "verbosity".to_string(), + value: "compressed".to_string(), + cue_family: CueFamily::Behavioral, + evidence: EvidenceRef::EpisodicWindow { from_id, to_id }, + initial_confidence: 0.65, + observed_at: now_secs(), + }); + state + .length_ratio_emitted + .insert(("verbosity".to_string(), "compressed".to_string())); + } + + // Balanced: user message length roughly matches agent. + if recent_ratio > RATIO_BALANCED + && !state + .length_ratio_emitted + .contains(&("verbosity".to_string(), "balanced".to_string())) + { + let first_id = state.turns.front().map(|t| t.turn_id.clone()); + let last_id = state.turns.back().map(|t| t.turn_id.clone()); + let from_id = first_id + .and_then(|s| s.parse::().ok()) + .unwrap_or(episodic_id); + let to_id = last_id + .and_then(|s| s.parse::().ok()) + .unwrap_or(episodic_id); + + tracing::debug!( + "[learning::extract::heuristics] length_ratio balanced session={} \ + recent_ratio={:.2}", + session_id, + recent_ratio + ); + candidates.push(LearningCandidate { + class: FacetClass::Style, + key: "verbosity".to_string(), + value: "balanced".to_string(), + cue_family: CueFamily::Behavioral, + evidence: EvidenceRef::EpisodicWindow { from_id, to_id }, + initial_confidence: 0.60, + observed_at: now_secs(), + }); + state + .length_ratio_emitted + .insert(("verbosity".to_string(), "balanced".to_string())); + } + } + + // ── B. Edit-window detector ─────────────────────────────────────── + if let Some(prev_at) = prev_agent_at { + let gap = user_timestamp - prev_at; + if gap >= 0.0 && gap < EDIT_WINDOW_SECS { + let lower = user_message.to_ascii_lowercase(); + // Pattern → (key, value) pairs. + let patterns: &[(&str, &str, &str)] = &[ + ("shorter", "verbosity", "terse"), + ("too long", "verbosity", "terse"), + (" less ", "verbosity", "terse"), + ("just code", "format", "code-only"), + ("not bullets", "format", "prose"), + ("more detail", "verbosity", "detailed"), + ("more verbose", "verbosity", "detailed"), + ]; + for (pattern, key, value) in patterns { + if lower.contains(pattern) { + let now = now_secs(); + + // ── C. Correction-repeat detector ───────────────────── + // The counter advances on EVERY in-window correction, regardless + // of the edit-window cooldown. This lets 3× corrections promote + // to a Veto even if the cooldown suppressed intermediate emissions. + let cue_key = format!("{key}={value}"); + let count = state.correction_counts.entry(cue_key.clone()).or_insert(0); + *count += 1; + let current_count = *count; + + if current_count >= VETO_PROMOTION_THRESHOLD + && !state.veto_promoted.contains(&cue_key) + { + tracing::debug!( + "[learning::extract::heuristics] correction_repeat veto \ + cue={:?} count={} session={}", + cue_key, + current_count, + session_id + ); + let (veto_key, veto_value, veto_confidence) = + correction_to_veto(key, value); + candidates.push(LearningCandidate { + class: FacetClass::Veto, + key: veto_key, + value: veto_value, + cue_family: CueFamily::Behavioral, + evidence: EvidenceRef::Episodic { episodic_id }, + initial_confidence: veto_confidence, + observed_at: now, + }); + state.veto_promoted.insert(cue_key); + } + + // Style candidate emission respects the cooldown. + let cooldown_key = (key.to_string(), value.to_string()); + let last_emit = state + .edit_cooldown + .get(&cooldown_key) + .copied() + .unwrap_or(0.0); + if now - last_emit >= EDIT_COOLDOWN_SECS { + tracing::debug!( + "[learning::extract::heuristics] edit_window matched \ + pattern={:?} key={} value={} session={}", + pattern, + key, + value, + session_id + ); + candidates.push(LearningCandidate { + class: FacetClass::Style, + key: key.to_string(), + value: value.to_string(), + cue_family: CueFamily::Behavioral, + evidence: EvidenceRef::Episodic { episodic_id }, + initial_confidence: 0.70, + observed_at: now, + }); + state.edit_cooldown.insert(cooldown_key, now); + } + } + } + } + } + } + + // Push outside the lock. + let buf = candidate::global(); + let count = candidates.len(); + for c in candidates { + buf.push(c); + } + if count > 0 { + tracing::debug!( + "[learning::extract::heuristics] record_turn session={} pushed {} candidate(s)", + session_id, + count + ); + } +} + +/// Map a (key, value) Style correction to a Veto triple: (veto_key, veto_value, confidence). +fn correction_to_veto(key: &str, value: &str) -> (String, String, f64) { + match (key, value) { + ("format", "prose") => ("format".to_string(), "nested-bullets".to_string(), 0.75), + ("verbosity", "terse") => ("style".to_string(), "long-replies".to_string(), 0.70), + _ => (format!("{key}-{value}"), "banned".to_string(), 0.65), + } +} + +/// Compute mean user/agent length ratio for a slice of turns. +/// Returns 0.0 for empty slices or all-zero agent lengths. +fn mean_ratio(turns: &[&TurnEntry]) -> f64 { + if turns.is_empty() { + return 0.0; + } + let sum: f64 = turns + .iter() + .map(|t| { + if t.agent_msg_len == 0 { + 0.0 + } else { + t.user_msg_len as f64 / t.agent_msg_len as f64 + } + }) + .sum(); + sum / turns.len() as f64 +} + +fn now_secs() -> f64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64() +} + +// ── Tests ──────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod heuristics_tests { + use super::*; + use crate::openhuman::learning::candidate::{Buffer, FacetClass}; + + fn fresh_session_id() -> String { + format!( + "test-session-{}-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(), + rand_id() + ) + } + + /// Cheap random suffix so parallel tests don't collide on session keys. + fn rand_id() -> u64 { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + let mut h = DefaultHasher::new(); + std::thread::current().id().hash(&mut h); + h.finish() + } + + /// Record N turns with decreasing user_msg_len into the heuristics module. + fn push_turns(session_id: &str, n: usize, user_len: usize, agent_len: usize, buf: &Buffer) { + let _ = buf; // We push into the global; this is just for clarity. + let now = now_secs(); + for i in 0..n { + record_turn( + session_id, + &i.to_string(), + i as i64, + "neutral message", + user_len, + agent_len, + now + i as f64, + now + i as f64 + 1.0, + None, // no edit window + ); + } + } + + #[test] + fn length_ratio_emits_compressed_when_user_msgs_shrink() { + let session = fresh_session_id(); + let buf = Buffer::new(1024); + + // First 15 turns: high ratio (user talks a lot). + let now = now_secs(); + for i in 0..15 { + record_turn( + &session, + &i.to_string(), + i as i64, + "long long long message", + 200, + 100, + now + i as f64, + now + i as f64 + 1.0, + None, + ); + } + // Next 15 turns: low ratio (user became terse). + for i in 15..30 { + record_turn( + &session, + &i.to_string(), + i as i64, + "ok", + 5, + 100, + now + i as f64, + now + i as f64 + 1.0, + None, + ); + } + + // Should have emitted the "compressed" candidate into the global buffer. + let all = candidate::global().peek(); + let compressed = all + .iter() + .filter(|c| { + c.key == "verbosity" + && c.value == "compressed" + && matches!(&c.evidence, EvidenceRef::EpisodicWindow { .. }) + }) + .count(); + assert!( + compressed >= 1, + "expected at least one compressed verbosity candidate, got 0" + ); + } + + #[test] + fn length_ratio_does_not_emit_with_short_window() { + let session = fresh_session_id(); + // Only 10 turns — below the 30-turn minimum. + push_turns(&session, 10, 5, 500, &Buffer::new(32)); + // Peek the global buffer for this session's compressed candidates. + // We can't isolate per-session here, but at least ensure no crash. + // (Functional assertion is in the positive test above.) + } + + #[test] + fn length_ratio_cooldown_prevents_repeated_emission() { + let session = fresh_session_id(); + // Trigger the compressed detection twice. + let now = now_secs(); + for i in 0..30 { + record_turn( + &session, + &i.to_string(), + i as i64, + "msg", + if i < 15 { 200 } else { 5 }, + 100, + now + i as f64, + now + i as f64 + 1.0, + None, + ); + } + // Trigger it again — should be suppressed by the cooldown set. + for i in 30..60 { + record_turn( + &session, + &i.to_string(), + i as i64, + "msg", + 5, + 100, + now + i as f64, + now + i as f64 + 1.0, + None, + ); + } + + // Check that compressed was only emitted once for this session. + let map = session_state().read(); + let st = map.get(&session).expect("state for session"); + assert!( + st.length_ratio_emitted + .contains(&("verbosity".to_string(), "compressed".to_string())), + "emitted set must record the compression emission" + ); + } + + #[test] + fn edit_window_emits_terse_on_shorter_correction() { + let session = fresh_session_id(); + let now = now_secs(); + + // Record a turn where user sends "shorter" within 10s of the agent. + record_turn( + &session, + "1", + 1, + "shorter please", + 14, + 200, + now, + now + 1.0, + Some(now - 10.0), // agent replied 10s ago + ); + + let all = candidate::global().peek(); + let terse = all.iter().any(|c| { + c.key == "verbosity" + && c.value == "terse" + && c.class == FacetClass::Style + && matches!(&c.evidence, EvidenceRef::Episodic { episodic_id } if *episodic_id == 1) + }); + assert!( + terse, + "expected a terse verbosity candidate after 'shorter' correction" + ); + } + + #[test] + fn edit_window_ignores_late_messages() { + let session = fresh_session_id(); + let now = now_secs(); + + // User sends "shorter" but 60s after the agent reply — outside window. + let before_count = candidate::global().len(); + record_turn( + &session, + "late-1", + 999, + "shorter please", + 14, + 200, + now, + now + 1.0, + Some(now - 60.0), + ); + let after_count = candidate::global().len(); + + // No new terse candidate should have been added for this episode. + // We can only check the delta is zero from our (outside the lock) vantage. + // The correction pattern might still be stored internally but not emitted. + // Confirm: global buffer didn't grow with a terse candidate for episodic_id=999. + let all = candidate::global().peek(); + let late_terse = all.iter().any(|c| { + c.key == "verbosity" + && c.value == "terse" + && matches!(&c.evidence, EvidenceRef::Episodic { episodic_id } if *episodic_id == 999) + }); + assert!(!late_terse, "late message must not emit terse candidate"); + let _ = (before_count, after_count); + } + + #[test] + fn correction_repeat_promotes_to_veto_after_3() { + let session = fresh_session_id(); + let now = now_secs(); + + // Three "not bullets" corrections within the edit window. + for i in 0..3usize { + // Each turn: agent replied just before, user corrects quickly. + let prev_agent_at = now + i as f64 * 100.0 - 5.0; + let user_at = now + i as f64 * 100.0; + record_turn( + &session, + &format!("veto-{i}"), + (100 + i) as i64, + "not bullets please", + 18, + 300, + user_at, + user_at + 2.0, + Some(prev_agent_at), + ); + } + + let all = candidate::global().peek(); + let veto = all.iter().any(|c| { + c.class == FacetClass::Veto && c.key == "format" && c.value == "nested-bullets" + }); + assert!( + veto, + "3× 'not bullets' correction must promote to Veto/format=nested-bullets" + ); + } +} diff --git a/src/openhuman/learning/extract/mod.rs b/src/openhuman/learning/extract/mod.rs new file mode 100644 index 000000000..b2d0ddecf --- /dev/null +++ b/src/openhuman/learning/extract/mod.rs @@ -0,0 +1,15 @@ +//! Phase 2 producer modules for the ambient personalization cache. +//! +//! Each submodule is a distinct producer that writes [`LearningCandidate`]s +//! into [`crate::openhuman::learning::candidate::global()`]. Phase 3 +//! (stability detector) drains and aggregates those candidates. +//! +//! | Module | Trigger | Signal class | +//! |--------|---------|-------------| +//! | [`signature`] | `DomainEvent::DocumentCanonicalized` (email) | Identity | +//! | [`heuristics`] | Post-turn hook | Style + Veto | +//! | [`summary_facets`] | LLM summariser output parsing | All classes | + +pub mod heuristics; +pub mod signature; +pub mod summary_facets; diff --git a/src/openhuman/learning/extract/signature.rs b/src/openhuman/learning/extract/signature.rs new file mode 100644 index 000000000..95d469a52 --- /dev/null +++ b/src/openhuman/learning/extract/signature.rs @@ -0,0 +1,643 @@ +//! Email signature parser — Phase 2 producer for Identity candidates. +//! +//! Subscribes to [`DomainEvent::DocumentCanonicalized`] events whose +//! `source_kind == "email"` and parses the trailing signature region of the +//! email body for identity facets (name, role, timezone, employer, location). +//! +//! ## Design notes +//! +//! The parser is intentionally conservative: false positives would pollute the +//! identity class with noisy data and erode user trust. Each detection rule +//! requires a concrete structural signal (capitalised name pattern, role keyword +//! anchor, timezone abbreviation, etc.) rather than scoring free text. +//! +//! ## Registration +//! +//! Call [`register_email_signature_subscriber`] once at startup (alongside the +//! `TracingSubscriber` and other domain subscribers). The returned +//! `SubscriptionHandle` must be kept alive for the subscriber to remain active. + +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +use async_trait::async_trait; + +use crate::core::event_bus::{subscribe_global, DomainEvent, EventHandler, SubscriptionHandle}; +use crate::openhuman::learning::candidate::{ + self, CueFamily, EvidenceRef, FacetClass, LearningCandidate, +}; + +// ── Constants ──────────────────────────────────────────────────────────────── + +/// Number of non-empty lines to scan from the bottom of the body. +const SIG_WINDOW_LINES: usize = 8; + +/// Initial confidence values per detection kind. +const CONF_NAME: f64 = 0.85; +const CONF_ROLE: f64 = 0.80; +const CONF_TIMEZONE: f64 = 0.90; +const CONF_EMPLOYER: f64 = 0.70; +const CONF_LOCATION: f64 = 0.60; + +// ── Public parse function ──────────────────────────────────────────────────── + +/// Parse the trailing signature region of `email_body` for identity facets. +/// +/// Returns a (possibly empty) list of [`LearningCandidate`]s, one per +/// detected signal. Candidates with `initial_confidence` < 0.6 are +/// dropped before returning. +/// +/// # Arguments +/// +/// * `email_body` — the canonical markdown body of the email message +/// * `source_id` — the ingest source id (e.g. `"gmail:abc"`) +/// * `message_id` — provider message identifier for provenance +pub fn parse_signature( + email_body: &str, + source_id: &str, + message_id: &str, +) -> Vec { + let now = now_secs(); + + // Take the last SIG_WINDOW_LINES non-empty lines in original order. + let all_non_empty: Vec<&str> = email_body + .lines() + .filter(|l| !l.trim().is_empty()) + .collect(); + let start = all_non_empty.len().saturating_sub(SIG_WINDOW_LINES); + let sig_lines = &all_non_empty[start..]; + + if sig_lines.is_empty() { + return Vec::new(); + } + + let evidence = || EvidenceRef::EmailMessage { + source_id: source_id.to_string(), + message_id: message_id.to_string(), + }; + + let mut candidates: Vec = Vec::new(); + + // Track whether we found any strong signature signal so we can gate + // low-confidence detections (location). + let mut strong_signal_found = false; + + // --- Name detection --- + let mut name_line_idx: Option = None; + for (idx, line) in sig_lines.iter().enumerate() { + let t = line.trim(); + if is_likely_name(t) { + name_line_idx = Some(idx); + candidates.push(LearningCandidate { + class: FacetClass::Identity, + key: "name".to_string(), + value: t.to_string(), + cue_family: CueFamily::Structural, + evidence: evidence(), + initial_confidence: CONF_NAME, + observed_at: now, + }); + strong_signal_found = true; + tracing::debug!( + "[learning::extract::signature] name detected: {:?} source_id={}", + t, + source_id + ); + break; + } + } + + // --- Role detection --- + let mut role_line_idx: Option = None; + for (idx, line) in sig_lines.iter().enumerate() { + let t = line.trim(); + if let Some(role) = extract_role(t) { + role_line_idx = Some(idx); + candidates.push(LearningCandidate { + class: FacetClass::Identity, + key: "role".to_string(), + value: role.to_string(), + cue_family: CueFamily::Structural, + evidence: evidence(), + initial_confidence: CONF_ROLE, + observed_at: now, + }); + strong_signal_found = true; + tracing::debug!( + "[learning::extract::signature] role detected: {:?} source_id={}", + role, + source_id + ); + break; + } + } + + // --- Timezone detection --- + for line in sig_lines { + let t = line.trim(); + if let Some(tz) = extract_timezone(t) { + candidates.push(LearningCandidate { + class: FacetClass::Identity, + key: "timezone".to_string(), + value: tz.to_string(), + cue_family: CueFamily::Structural, + evidence: evidence(), + initial_confidence: CONF_TIMEZONE, + observed_at: now, + }); + strong_signal_found = true; + tracing::debug!( + "[learning::extract::signature] timezone detected: {:?} source_id={}", + tz, + source_id + ); + break; + } + } + + // --- Employer detection --- + // Strategy 1: line immediately following the role line. + if let Some(role_idx) = role_line_idx { + let employer_idx = role_idx + 1; + if employer_idx < sig_lines.len() { + let t = sig_lines[employer_idx].trim(); + if is_plausible_employer(t) { + candidates.push(LearningCandidate { + class: FacetClass::Identity, + key: "employer".to_string(), + value: clean_employer(t), + cue_family: CueFamily::Structural, + evidence: evidence(), + initial_confidence: CONF_EMPLOYER, + observed_at: now, + }); + strong_signal_found = true; + tracing::debug!( + "[learning::extract::signature] employer (post-role) detected: {:?} source_id={}", + t, + source_id + ); + } + } + } + // Strategy 2: "@ Company" or "Company, Inc" pattern on any sig line. + if !candidates.iter().any(|c| c.key == "employer") { + for line in sig_lines { + let t = line.trim(); + if let Some(emp) = extract_employer_pattern(t) { + candidates.push(LearningCandidate { + class: FacetClass::Identity, + key: "employer".to_string(), + value: emp, + cue_family: CueFamily::Structural, + evidence: evidence(), + initial_confidence: CONF_EMPLOYER, + observed_at: now, + }); + strong_signal_found = true; + tracing::debug!( + "[learning::extract::signature] employer (pattern) detected source_id={}", + source_id + ); + break; + } + } + } + + // --- Location detection (low-confidence, only when strong signals present) --- + if strong_signal_found { + // Look at the line right after the name line. + if let Some(name_idx) = name_line_idx { + let loc_idx = name_idx + 1; + if loc_idx < sig_lines.len() { + let t = sig_lines[loc_idx].trim(); + // Skip if already matched as role or employer. + let already_used = role_line_idx.map_or(false, |ri| ri == loc_idx); + if !already_used { + if let Some(loc) = extract_location(t) { + candidates.push(LearningCandidate { + class: FacetClass::Identity, + key: "location".to_string(), + value: loc, + cue_family: CueFamily::Structural, + evidence: evidence(), + initial_confidence: CONF_LOCATION, + observed_at: now, + }); + tracing::debug!( + "[learning::extract::signature] location detected source_id={}", + source_id + ); + } + } + } + } + } + + // Drop low-confidence noise. All CONF_* constants are currently ≥ 0.6, + // so this is mostly a guard against future regressions and codifies the + // contract advertised in the doc comment. + let kept = candidates.len(); + candidates.retain(|c| c.initial_confidence >= 0.6); + let dropped = kept - candidates.len(); + + tracing::debug!( + "[learning::extract::signature] parse_signature source_id={} candidates={} dropped={}", + source_id, + candidates.len(), + dropped, + ); + + candidates +} + +// ── Detection helpers ───────────────────────────────────────────────────────── + +/// Returns `true` if `s` looks like a person's name: 1–4 capitalised words, +/// no `@`, no URL fragments, mostly alphabetic, no trailing punctuation like commas. +fn is_likely_name(s: &str) -> bool { + if s.contains('@') || s.contains("://") || s.contains("www.") { + return false; + } + // Lines ending with punctuation like "Thanks," or "Best regards," are not names. + if s.ends_with(',') || s.ends_with(':') || s.ends_with(';') { + return false; + } + // Lines that are clearly greetings or sign-offs. + let lower = s.to_ascii_lowercase(); + const EXCLUSIONS: &[&str] = &[ + "thanks", + "regards", + "best", + "cheers", + "sincerely", + "cordially", + "hi", + "hello", + "dear", + "hey", + ]; + if EXCLUSIONS.iter().any(|e| lower == *e) { + return false; + } + + let words: Vec<&str> = s.split_whitespace().collect(); + if words.is_empty() || words.len() > 4 { + return false; + } + // Every word must start with an uppercase letter and be mostly alpha. + for w in &words { + let first = w.chars().next().unwrap_or(' '); + if !first.is_uppercase() { + return false; + } + // Strip trailing punctuation for the ratio check. + let clean: String = w.chars().filter(|c| c.is_alphabetic()).collect(); + if clean.is_empty() { + return false; + } + let alpha_ratio = clean.len() as f64 / w.len() as f64; + if alpha_ratio < 0.7 { + return false; + } + } + // Must contain at least two words or exactly one that's clearly a name + // (more than 3 chars). Single-word uppercase abbreviations like "PST" + // would be false positives without this guard. + words.len() >= 2 || (words.len() == 1 && words[0].len() > 3) +} + +/// Extracts a role string from `s` if it contains a known role keyword. +/// Returns the trimmed line (or the keyword match) as the role value. +fn extract_role(s: &str) -> Option<&str> { + const ROLE_KEYWORDS: &[&str] = &[ + "engineer", + "developer", + "designer", + "manager", + "director", + "founder", + "cto", + "ceo", + "coo", + "cpo", + "cfo", + "product manager", + "consultant", + "lead", + "head of", + "vp", + "vice president", + "principal", + "staff ", + "senior ", + "architect", + "researcher", + "analyst", + "recruiter", + "sales", + "marketing", + "intern", + ]; + let lower = s.to_ascii_lowercase(); + for kw in ROLE_KEYWORDS { + if lower.contains(kw) { + return Some(s); + } + } + None +} + +/// Extracts a timezone abbreviation from `s`. +fn extract_timezone(s: &str) -> Option<&str> { + // Scan for known timezone patterns. We return the matched slice. + const STATIC_TZ: &[&str] = &[ + "PST", "PDT", "EST", "EDT", "MST", "MDT", "CST", "CDT", "UTC", "GMT", + ]; + for tz in STATIC_TZ { + if let Some(pos) = s.find(tz) { + // Confirm the match is a whole word (preceded/followed by non-alpha). + let before_ok = pos == 0 || !s[..pos].ends_with(|c: char| c.is_alphabetic()); + let after = &s[pos + tz.len()..]; + let after_ok = after.is_empty() + || after.starts_with(|c: char| !c.is_alphabetic()) + || after.starts_with(|c: char| c == '+' || c == '-'); + if before_ok && after_ok { + // Grab "UTC+5:30" or "GMT-7" style suffix. + if tz.starts_with("UTC") || tz.starts_with("GMT") { + let end = s[pos + tz.len()..] + .find(|c: char| !c.is_ascii_digit() && c != '+' && c != '-' && c != ':') + .map(|off| pos + tz.len() + off) + .unwrap_or(s.len()); + return Some(&s[pos..end]); + } + return Some(&s[pos..pos + tz.len()]); + } + } + } + None +} + +/// Returns `true` if `s` is plausible as a company / employer name line +/// (not an email, not a URL, contains at least one capital letter, not pure punctuation). +fn is_plausible_employer(s: &str) -> bool { + if s.is_empty() || s.len() > 80 { + return false; + } + if s.contains('@') || s.contains("://") || s.contains("www.") { + return false; + } + // Must have at least one uppercase letter. + if !s.chars().any(|c| c.is_uppercase()) { + return false; + } + let alpha_count = s.chars().filter(|c| c.is_alphabetic()).count(); + alpha_count >= 2 +} + +/// Cleans a raw employer line: trims, strips trailing punctuation. +fn clean_employer(s: &str) -> String { + s.trim() + .trim_end_matches([',', '.', ';']) + .trim() + .to_string() +} + +/// Tries to extract an employer from patterns: `"@ Company"` or `"Company, Inc"`. +fn extract_employer_pattern(s: &str) -> Option { + let t = s.trim(); + // "@ Company Name" + if let Some(stripped) = t.strip_prefix('@') { + let name = stripped.trim(); + if !name.is_empty() && !name.contains('@') { + return Some(name.to_string()); + } + } + // "Company, Inc" / "Company LLC" / "Company Ltd" + let lower = t.to_ascii_lowercase(); + let corp_suffixes = [ + ", inc", " inc.", " llc", " ltd", " limited", " corp", " co.", + ]; + for suffix in corp_suffixes { + if lower.ends_with(suffix) || lower.contains(&format!("{suffix} ")) { + if is_plausible_employer(t) { + return Some(clean_employer(t)); + } + } + } + None +} + +/// Attempts to extract a city/state location from a line. +/// Very conservative: requires a comma between city and state/country-like fragment. +fn extract_location(s: &str) -> Option { + let t = s.trim(); + if t.contains('@') || t.contains("://") { + return None; + } + // Simple heuristic: "City, ST" or "City, Country" + if let Some(comma_pos) = t.find(',') { + let city = t[..comma_pos].trim(); + let region = t[comma_pos + 1..].trim(); + // City: 2-30 chars, no digits, starts with uppercase. + let city_ok = city.len() >= 2 + && city.len() <= 30 + && city.chars().next().map_or(false, |c| c.is_uppercase()) + && !city.chars().any(|c| c.is_ascii_digit()); + // Region: 2-20 chars. + let region_ok = region.len() >= 2 && region.len() <= 20; + if city_ok && region_ok { + return Some(t.to_string()); + } + } + None +} + +fn now_secs() -> f64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64() +} + +// ── Subscriber ─────────────────────────────────────────────────────────────── + +/// Event subscriber that reacts to `DocumentCanonicalized` events for email +/// sources and routes detected identity candidates into the global buffer. +pub struct EmailSignatureSubscriber; + +#[async_trait] +impl EventHandler for EmailSignatureSubscriber { + fn name(&self) -> &str { + "learning::extract::signature" + } + + fn domains(&self) -> Option<&[&str]> { + Some(&["memory"]) + } + + async fn handle(&self, event: &DomainEvent) { + if let DomainEvent::DocumentCanonicalized { + source_id, + source_kind, + body_preview, + chunk_ids, + .. + } = event + { + if source_kind != "email" { + return; + } + let body = match body_preview { + Some(b) => b, + None => { + tracing::debug!( + "[learning::extract::signature] no body_preview on DocumentCanonicalized \ + source_id={} — skipping signature parse", + source_id + ); + return; + } + }; + + // Use the first chunk_id as the message_id if no dedicated field is + // available. The EmailMessage evidence variant carries both the + // source_id and message_id for provenance. + let message_id = chunk_ids + .first() + .cloned() + .unwrap_or_else(|| source_id.clone()); + + tracing::debug!( + "[learning::extract::signature] parsing email signature source_id={} body_len={}", + source_id, + body.len() + ); + + let candidates = parse_signature(body, source_id, &message_id); + let count = candidates.len(); + let buf = candidate::global(); + for c in candidates { + buf.push(c); + } + tracing::debug!( + "[learning::extract::signature] pushed {} identity candidate(s) for source_id={}", + count, + source_id + ); + } + } +} + +/// Register the email signature subscriber on the global event bus. +/// +/// Must be called at startup after [`crate::core::event_bus::init_global`]. +/// The returned handle keeps the subscription alive — store it in a long-lived +/// container (e.g. alongside other `SubscriptionHandle`s in startup). +pub fn register_email_signature_subscriber() -> Option { + subscribe_global(Arc::new(EmailSignatureSubscriber)) +} + +// ── Tests ──────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod signature_tests { + use super::*; + use crate::openhuman::learning::candidate::{CueFamily, EvidenceRef, FacetClass}; + + fn extract(body: &str) -> Vec { + parse_signature(body, "gmail:test", "") + } + + fn find<'a>(cs: &'a [LearningCandidate], key: &str) -> Option<&'a LearningCandidate> { + cs.iter().find(|c| c.key == key) + } + + #[test] + fn parse_signature_extracts_name_role_timezone_employer() { + let body = "Hi, great to hear from you!\n\n\ + Please find the docs attached.\n\n\ + Thanks,\n\ + Alice Johnson\n\ + Senior Software Engineer\n\ + Acme Corp\n\ + San Francisco, CA\n\ + PST"; + let candidates = extract(body); + + let name = find(&candidates, "name").expect("name candidate"); + assert_eq!(name.value, "Alice Johnson"); + assert!((name.initial_confidence - CONF_NAME).abs() < 0.01); + + let role = find(&candidates, "role").expect("role candidate"); + assert!(role.value.contains("Engineer") || role.value.contains("engineer")); + + let tz = find(&candidates, "timezone").expect("timezone candidate"); + assert_eq!(tz.value, "PST"); + assert!((tz.initial_confidence - CONF_TIMEZONE).abs() < 0.01); + + let emp = find(&candidates, "employer").expect("employer candidate"); + assert!(emp.value.contains("Acme")); + } + + #[test] + fn parse_signature_handles_no_signature() { + let body = "just some content here nothing looks like a sig"; + let cs = extract(body); + // No strong signals → no candidates emitted. The < 0.6 filter at the + // tail of parse_signature drops anything below confidence threshold + // (including the gated lone-location case), so the result must be empty. + assert!( + cs.is_empty(), + "expected zero candidates from non-signature body, got {cs:?}" + ); + } + + #[test] + fn parse_signature_ignores_quoted_replies() { + // Even when the body has quoted sections, only the window of non-empty lines + // at the very end is scanned. + let body = "> On Monday, Alice wrote:\n\ + > Great to meet you!\n\ + >\n\ + Sure, let's connect.\n\n\ + Bob Smith\n\ + Product Manager\n\ + UTC+1"; + let cs = extract(body); + let name = find(&cs, "name").expect("name candidate"); + assert_eq!(name.value, "Bob Smith"); + let tz = find(&cs, "timezone").expect("timezone"); + assert!(tz.value.starts_with("UTC")); + } + + #[test] + fn parse_signature_low_confidence_for_lone_location() { + // Location alone (no other strong signals) should NOT produce a candidate. + let body = "The meeting is on Thursday.\nSan Francisco, CA"; + let cs = extract(body); + assert!( + find(&cs, "location").is_none(), + "location candidate must not be emitted without other strong signals" + ); + } + + #[test] + fn parse_signature_emits_evidence_email_message_variant() { + let body = "Alice Smith\nCTO\nStartup Inc\nPST"; + let cs = extract(body); + for c in &cs { + assert!( + matches!( + &c.evidence, + EvidenceRef::EmailMessage { source_id, message_id } + if source_id == "gmail:test" && message_id == "" + ), + "expected EmailMessage evidence, got {:?}", + c.evidence + ); + assert_eq!(c.cue_family, CueFamily::Structural); + assert_eq!(c.class, FacetClass::Identity); + } + } +} diff --git a/src/openhuman/learning/extract/summary_facets.rs b/src/openhuman/learning/extract/summary_facets.rs new file mode 100644 index 000000000..704a6b485 --- /dev/null +++ b/src/openhuman/learning/extract/summary_facets.rs @@ -0,0 +1,343 @@ +//! Structured facet schema and routing from LLM summariser output. +//! +//! The LLM summariser is extended (in `memory/tree/tree_source/summariser/llm.rs`) +//! to produce a second JSON block after the prose summary. This module defines +//! the serde shapes for that block ([`StructuredSummary`], [`ParsedFacet`]) and +//! provides [`route_facets_to_buffer`], which validates each facet and pushes +//! valid candidates to [`crate::openhuman::learning::candidate::global()`]. +//! +//! ## Provenance contract +//! +//! Every facet must cite at least one `chunk_id` in its `evidence_chunks` array. +//! Facets with an empty `evidence_chunks` are silently dropped — unattributed +//! observations cannot be scored or cited later. + +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde::{Deserialize, Serialize}; + +use crate::openhuman::learning::candidate::{ + self, CueFamily, EvidenceRef, FacetClass, LearningCandidate, +}; + +// ── Serde types ────────────────────────────────────────────────────────────── + +/// A single facet extracted by the LLM during summarisation. +#[derive(Deserialize, Serialize, Debug, Clone)] +#[serde(rename_all = "snake_case")] +pub struct ParsedFacet { + /// Facet class as a string — `"style"` | `"identity"` | `"tooling"` | + /// `"veto"` | `"goal"` | `"channel"`. + pub class: String, + /// Canonical slug key within the class, e.g. `"verbosity"`, `"timezone"`. + pub key: String, + /// Detected value string. + pub value: String, + /// Chunk IDs from the current seal batch that evidence this facet. + /// Must be non-empty for the facet to be accepted. + #[serde(default)] + pub evidence_chunks: Vec, + /// Source confidence `0.0..=1.0`. + pub confidence: f64, + /// How the signal was produced — `"explicit"` | `"structural"` | `"behavioral"`. + #[serde(default = "default_cue")] + pub cue_family: String, +} + +fn default_cue() -> String { + "behavioral".into() +} + +/// The full structured output expected from the LLM summariser when +/// `structured_facet_extraction` is enabled. +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct StructuredSummary { + /// Prose summary (the text that was previously the only output). + pub summary: String, + /// Optional extracted facets. Empty by default when the LLM found nothing + /// clearly evidenced. + #[serde(default)] + pub facets: Vec, +} + +// ── Routing ─────────────────────────────────────────────────────────────────── + +/// Validate each [`ParsedFacet`] in `parsed` and push valid candidates to +/// [`candidate::global()`]. +/// +/// Drops facets that: +/// - have an unrecognised `class` string +/// - have an empty `evidence_chunks` array (provenance is mandatory) +/// - have `confidence` outside `0.0..=1.0` +/// +/// Maps `cue_family` strings to [`CueFamily`]; unknown strings default to +/// [`CueFamily::Behavioral`]. +/// +/// Uses the first non-empty `evidence_chunks` entry as the `chunk_id` in +/// [`EvidenceRef::DocumentChunk`]. +pub fn route_facets_to_buffer(parsed: &StructuredSummary, source_id: &str) { + let now = now_secs(); + let buf = candidate::global(); + let mut pushed = 0usize; + + for facet in &parsed.facets { + // Validate evidence. + let chunk_id = match facet.evidence_chunks.first() { + Some(id) if !id.is_empty() => id.clone(), + _ => { + tracing::debug!( + "[learning::extract::summary_facets] dropping facet key={} \ + (no evidence_chunks) source_id={}", + facet.key, + source_id + ); + continue; + } + }; + + // Map class string. + let class = match parse_facet_class(&facet.class) { + Some(c) => c, + None => { + tracing::debug!( + "[learning::extract::summary_facets] dropping facet key={} \ + (unknown class={:?}) source_id={}", + facet.key, + facet.class, + source_id + ); + continue; + } + }; + + // Clamp confidence. + let confidence = facet.confidence.clamp(0.0, 1.0); + + let cue_family = parse_cue_family(&facet.cue_family); + + let candidate = LearningCandidate { + class, + key: facet.key.clone(), + value: facet.value.clone(), + cue_family, + evidence: EvidenceRef::DocumentChunk { + source_id: source_id.to_string(), + chunk_id, + }, + initial_confidence: confidence, + observed_at: now, + }; + + tracing::debug!( + "[learning::extract::summary_facets] routing facet class={:?} key={} \ + value={:?} confidence={:.2} source_id={}", + candidate.class, + candidate.key, + candidate.value, + candidate.initial_confidence, + source_id + ); + + buf.push(candidate); + pushed += 1; + } + + tracing::debug!( + "[learning::extract::summary_facets] route_facets_to_buffer source_id={} \ + facets_in={} pushed={}", + source_id, + parsed.facets.len(), + pushed + ); +} + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +fn parse_facet_class(s: &str) -> Option { + match s { + "style" => Some(FacetClass::Style), + "identity" => Some(FacetClass::Identity), + "tooling" => Some(FacetClass::Tooling), + "veto" => Some(FacetClass::Veto), + "goal" => Some(FacetClass::Goal), + "channel" => Some(FacetClass::Channel), + _ => None, + } +} + +fn parse_cue_family(s: &str) -> CueFamily { + match s { + "explicit" => CueFamily::Explicit, + "structural" => CueFamily::Structural, + "behavioral" => CueFamily::Behavioral, + "recurrence" => CueFamily::Recurrence, + _ => CueFamily::Behavioral, + } +} + +fn now_secs() -> f64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64() +} + +// ── Tests ──────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::learning::candidate::{Buffer, CueFamily, FacetClass}; + + fn make_summary(facets: Vec) -> StructuredSummary { + StructuredSummary { + summary: "A summary.".into(), + facets, + } + } + + fn make_facet( + class: &str, + key: &str, + value: &str, + evidence_chunks: Vec<&str>, + confidence: f64, + cue_family: &str, + ) -> ParsedFacet { + ParsedFacet { + class: class.into(), + key: key.into(), + value: value.into(), + evidence_chunks: evidence_chunks.into_iter().map(str::to_string).collect(), + confidence, + cue_family: cue_family.into(), + } + } + + #[test] + fn parse_well_formed_structured_summary() { + let json = r#"{ + "summary": "The user prefers pnpm.", + "facets": [ + { + "class": "tooling", + "key": "package_manager", + "value": "pnpm", + "evidence_chunks": ["chunk-abc"], + "confidence": 0.85, + "cue_family": "explicit" + } + ] + }"#; + let parsed: StructuredSummary = serde_json::from_str(json).unwrap(); + assert_eq!(parsed.summary, "The user prefers pnpm."); + assert_eq!(parsed.facets.len(), 1); + assert_eq!(parsed.facets[0].key, "package_manager"); + assert_eq!(parsed.facets[0].cue_family, "explicit"); + } + + #[test] + fn drops_facet_with_unknown_class() { + let buf = Buffer::new(64); + let before = buf.len(); + + // Route into global but we check the drop via tracing — here we test + // that the function doesn't panic and the drop is silent. + let s = make_summary(vec![make_facet( + "unknown_class", + "key", + "val", + vec!["c1"], + 0.8, + "behavioral", + )]); + // Use global buffer for integration. + let before_global = candidate::global().len(); + route_facets_to_buffer(&s, "src-1"); + let after_global = candidate::global().len(); + // No new candidates pushed. + assert_eq!( + after_global, before_global, + "unknown class should drop the facet" + ); + let _ = (buf, before); + } + + #[test] + fn drops_facet_without_evidence_chunks() { + let s = make_summary(vec![make_facet( + "style", + "verbosity", + "terse", + vec![], // empty — must be dropped + 0.8, + "explicit", + )]); + let before = candidate::global().len(); + route_facets_to_buffer(&s, "src-2"); + assert_eq!( + candidate::global().len(), + before, + "facet without evidence_chunks must be dropped" + ); + } + + #[test] + fn defaults_cue_family_to_behavioral() { + let json = r#"{ + "summary": "x", + "facets": [ + { + "class": "style", + "key": "verbosity", + "value": "terse", + "evidence_chunks": ["c1"], + "confidence": 0.7 + } + ] + }"#; + let parsed: StructuredSummary = serde_json::from_str(json).unwrap(); + assert_eq!(parsed.facets[0].cue_family, "behavioral"); + } + + #[test] + fn route_pushes_to_global_buffer() { + let s = make_summary(vec![ + make_facet( + "style", + "verbosity", + "terse", + vec!["chunk-1"], + 0.75, + "explicit", + ), + make_facet( + "identity", + "timezone", + "UTC+5:30", + vec!["chunk-2"], + 0.9, + "structural", + ), + ]); + let before = candidate::global().len(); + route_facets_to_buffer(&s, "notion:doc-1"); + let after = candidate::global().len(); + assert_eq!( + after, + before + 2, + "two valid facets should push two candidates" + ); + + let all = candidate::global().peek(); + let tz = all.iter().find(|c| c.key == "timezone"); + let tz = tz.expect("timezone candidate in buffer"); + assert_eq!(tz.value, "UTC+5:30"); + assert_eq!(tz.class, FacetClass::Identity); + assert_eq!(tz.cue_family, CueFamily::Structural); + assert!( + matches!(&tz.evidence, EvidenceRef::DocumentChunk { source_id, chunk_id } + if source_id == "notion:doc-1" && chunk_id == "chunk-2") + ); + } +} diff --git a/src/openhuman/learning/mod.rs b/src/openhuman/learning/mod.rs index 81a729cf1..2329a2984 100644 --- a/src/openhuman/learning/mod.rs +++ b/src/openhuman/learning/mod.rs @@ -2,19 +2,50 @@ //! //! Post-turn hooks that reflect on completed turns, extract user preferences, //! track tool effectiveness, and store learnings in the Memory backend. +//! +//! # Phase 1 additions (#566) +//! +//! - [`candidate`] — `LearningCandidate`, `FacetClass`, `CueFamily`, `EvidenceRef`, +//! and the thread-safe ring-buffer [`candidate::Buffer`] that collects evidence +//! from producers (Phase 2) for consumption by the stability detector (Phase 3). +//! +//! # Phase 2 additions (#566) +//! +//! - [`extract`] — producer modules: `signature` (email identity parser), +//! `heuristics` (length-ratio + edit-window + correction-repeat detectors), +//! `summary_facets` (structured facets from the LLM summariser). +//! +//! # Phase 3 additions (#566) +//! +//! - [`cache`] — `FacetCache` wrapper over `user_profile_facets` table. +//! - [`stability_detector`] — rebuild cycle: drain, aggregate, score, resolve, persist. +//! - [`scheduler`] — periodic + event-driven rebuild scheduling. +pub mod cache; +pub mod candidate; +pub mod extract; pub mod linkedin_enrichment; +pub mod profile_md_renderer; pub mod prompt_sections; pub mod reflection; +pub mod scheduler; pub mod schemas; +pub mod stability_detector; pub mod tool_tracker; pub mod transcript_ingest; pub mod user_profile; -pub use prompt_sections::{LearnedContextSection, UserProfileSection}; +pub use cache::FacetCache; +pub use candidate::{Buffer, CueFamily, EvidenceRef, FacetClass, LearningCandidate}; +pub use profile_md_renderer::ProfileMdRenderer; +pub use prompt_sections::{ + load_learned_from_cache, LearnedContextSection, MemoryAccessSection, UserProfileSection, + MEMORY_ACCESS_INSTRUCTION, +}; pub use reflection::ReflectionHook; pub use schemas::{ all_learning_controller_schemas, all_learning_registered_controllers, learning_schemas, }; +pub use stability_detector::StabilityDetector; pub use tool_tracker::ToolTrackerHook; pub use user_profile::UserProfileHook; diff --git a/src/openhuman/learning/profile_md_renderer.rs b/src/openhuman/learning/profile_md_renderer.rs new file mode 100644 index 000000000..484bb6ae9 --- /dev/null +++ b/src/openhuman/learning/profile_md_renderer.rs @@ -0,0 +1,514 @@ +//! Profile-MD renderer for the learning subsystem. +//! +//! Subscribes to [`DomainEvent::CacheRebuilt`] and re-renders the five +//! cache-derived managed blocks in `PROFILE.md`: +//! +//! | Block name | Heading | Facet class | +//! |------------|---------|-------------| +//! | `style` | `## Style` | `FacetClass::Style` | +//! | `identity` | `## Identity`| `FacetClass::Identity`| +//! | `tooling` | `## Tooling` | `FacetClass::Tooling` | +//! | `vetoes` | `## Vetoes` | `FacetClass::Veto` | +//! | `goals` | `## Goals` | `FacetClass::Goal` | +//! +//! The `connected-accounts` block is NOT touched by this renderer; it is +//! owned exclusively by the provider path +//! (`composio::providers::profile_md::merge_provider_into_profile_md`). +//! +//! ## Rendering rules +//! +//! - Only `Active` rows are rendered in the visible blocks. +//! - Within each block, rows are sorted by `stability` desc, then by `key` asc. +//! - `Pinned` entries get a trailing ` *(pinned)*` indicator. +//! - Format per class: +//! - Style / Identity / Tooling / Vetoes: `- **{suffix}**: {value}` +//! where `suffix` is the portion of the key after the first `/`. +//! - Goals: `- {value}` (full sentence — no key prefix). +//! - Empty classes render the `*(no entries yet)*` placeholder (never +//! delete the block markers). +//! +//! ## Subscription +//! +//! [`ProfileMdRenderer::subscribe`] registers an `EventHandler` that calls +//! [`ProfileMdRenderer::render`] on every `CacheRebuilt` event. The render +//! is synchronous (SQLite reads + file writes) and runs on the Tokio blocking +//! thread pool. + +use std::path::PathBuf; +use std::sync::Arc; + +use async_trait::async_trait; + +use crate::core::event_bus::{subscribe_global, DomainEvent, EventHandler, SubscriptionHandle}; +use crate::openhuman::composio::providers::profile_md::replace_managed_block; +use crate::openhuman::learning::cache::FacetCache; +use crate::openhuman::memory::store::profile::UserState; + +// ── Class → block metadata ──────────────────────────────────────────────────── + +struct BlockSpec { + block_name: &'static str, + heading: &'static str, + class_prefix: &'static str, + /// When true, render `- {value}` (goal style). Otherwise `- **{key_suffix}**: {value}`. + value_only: bool, +} + +const BLOCK_SPECS: &[BlockSpec] = &[ + BlockSpec { + block_name: "style", + heading: "## Style", + class_prefix: "style/", + value_only: false, + }, + BlockSpec { + block_name: "identity", + heading: "## Identity", + class_prefix: "identity/", + value_only: false, + }, + BlockSpec { + block_name: "tooling", + heading: "## Tooling", + class_prefix: "tooling/", + value_only: false, + }, + BlockSpec { + block_name: "vetoes", + heading: "## Vetoes", + class_prefix: "veto/", + value_only: false, + }, + BlockSpec { + block_name: "goals", + heading: "## Goals", + class_prefix: "goal/", + value_only: true, + }, +]; + +// ── ProfileMdRenderer ───────────────────────────────────────────────────────── + +/// Renders Active facets from the `FacetCache` into the five cache-derived +/// managed blocks of `PROFILE.md`. +pub struct ProfileMdRenderer { + cache: Arc, + workspace_dir: PathBuf, +} + +impl ProfileMdRenderer { + /// Create a new renderer backed by `cache`, writing to + /// `workspace_dir/PROFILE.md`. + pub fn new(cache: Arc, workspace_dir: PathBuf) -> Self { + Self { + cache, + workspace_dir, + } + } + + /// Read all Active facets from the cache and re-render each of the five + /// cache-owned blocks. Never touches the `connected-accounts` block. + pub fn render(&self) -> anyhow::Result<()> { + tracing::debug!("[learning::profile_md_renderer] render triggered — reading active facets"); + + let active_facets = self.cache.list_active()?; + + for spec in BLOCK_SPECS { + // Filter to this class, sort by stability desc then key asc. + let mut rows: Vec<_> = active_facets + .iter() + .filter(|f| f.key.starts_with(spec.class_prefix)) + .collect(); + rows.sort_by(|a, b| { + b.stability + .partial_cmp(&a.stability) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.key.cmp(&b.key)) + }); + + let body = if rows.is_empty() { + String::new() // replace_managed_block renders the placeholder + } else { + let mut lines: Vec = Vec::with_capacity(rows.len()); + for f in &rows { + let pinned_suffix = if f.user_state == UserState::Pinned { + " *(pinned)*" + } else { + "" + }; + let line = if spec.value_only { + format!("- {}{}", f.value, pinned_suffix) + } else { + let key_suffix = f + .key + .strip_prefix(spec.class_prefix) + .unwrap_or(f.key.as_str()); + format!("- **{}**: {}{}", key_suffix, f.value, pinned_suffix) + }; + lines.push(line); + } + lines.join("\n") + }; + + replace_managed_block(&self.workspace_dir, spec.block_name, spec.heading, body) + .map_err(|e| { + anyhow::anyhow!( + "[learning::profile_md_renderer] failed to write block '{}': {e}", + spec.block_name + ) + })?; + + tracing::debug!( + "[learning::profile_md_renderer] wrote block '{}' ({} entries)", + spec.block_name, + rows.len() + ); + } + + tracing::info!("[learning::profile_md_renderer] PROFILE.md updated successfully"); + Ok(()) + } + + /// Register this renderer as an event subscriber for + /// [`DomainEvent::CacheRebuilt`] events. + /// + /// Returns the [`SubscriptionHandle`] — hold it alive for the lifetime of + /// the process (e.g. by leaking it into a static). + pub fn subscribe(renderer: Arc) -> Option { + subscribe_global(Arc::new(RendererSubscriber(renderer))) + } +} + +// ── Event subscriber ───────────────────────────────────────────────────────── + +struct RendererSubscriber(Arc); + +#[async_trait] +impl EventHandler for RendererSubscriber { + fn name(&self) -> &str { + "learning::profile_md_renderer" + } + + fn domains(&self) -> Option<&[&str]> { + Some(&["memory"]) + } + + async fn handle(&self, event: &DomainEvent) { + if let DomainEvent::CacheRebuilt { .. } = event { + let renderer = Arc::clone(&self.0); + // Move the blocking I/O (SQLite reads + fs writes) off the async + // executor thread. + tokio::task::spawn_blocking(move || { + if let Err(e) = renderer.render() { + tracing::warn!( + "[learning::profile_md_renderer] render on CacheRebuilt failed: {e:#}" + ); + } + }); + } + } +} + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::composio::providers::profile_md::{block_end, block_start}; + use crate::openhuman::memory::store::profile::{ + FacetState, FacetType, ProfileFacet, UserState, PROFILE_INIT_SQL, + }; + use parking_lot::Mutex; + use rusqlite::Connection; + use std::sync::Arc; + use tempfile::TempDir; + + fn make_cache(conn: Arc>) -> Arc { + Arc::new(FacetCache::new(conn)) + } + + fn insert_facet( + cache: &FacetCache, + key: &str, + value: &str, + state: FacetState, + user_state: UserState, + stability: f64, + ) { + let facet = ProfileFacet { + facet_id: format!("f-{key}"), + facet_type: FacetType::Preference, + key: key.into(), + value: value.into(), + confidence: 0.9, + evidence_count: 3, + source_segment_ids: None, + first_seen_at: 1000.0, + last_seen_at: 2000.0, + state, + stability, + user_state, + evidence_refs: vec![], + class: key.split('/').next().map(|s| s.to_string()), + cue_families: None, + }; + cache.upsert(&facet).unwrap(); + } + + fn make_renderer() -> (Arc, ProfileMdRenderer, TempDir) { + let tmp = TempDir::new().unwrap(); + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(PROFILE_INIT_SQL).unwrap(); + let cache = make_cache(Arc::new(Mutex::new(conn))); + let renderer = ProfileMdRenderer::new(Arc::clone(&cache), tmp.path().to_path_buf()); + (cache, renderer, tmp) + } + + #[test] + fn renders_active_facets_to_class_blocks() { + let (cache, renderer, tmp) = make_renderer(); + insert_facet( + &cache, + "style/verbosity", + "terse", + FacetState::Active, + UserState::Auto, + 2.0, + ); + insert_facet( + &cache, + "identity/name", + "Alice", + FacetState::Active, + UserState::Auto, + 1.8, + ); + insert_facet( + &cache, + "tooling/editor", + "neovim", + FacetState::Active, + UserState::Auto, + 1.5, + ); + insert_facet( + &cache, + "veto/no-em-dashes", + "avoid em dashes in prose", + FacetState::Active, + UserState::Auto, + 1.2, + ); + insert_facet( + &cache, + "goal/learn-rust", + "Learn Rust this year", + FacetState::Active, + UserState::Auto, + 1.0, + ); + + renderer.render().unwrap(); + + let body = std::fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); + assert!( + body.contains("- **verbosity**: terse"), + "style block:\n{body}" + ); + assert!( + body.contains("- **name**: Alice"), + "identity block:\n{body}" + ); + assert!( + body.contains("- **editor**: neovim"), + "tooling block:\n{body}" + ); + assert!( + body.contains("- **no-em-dashes**: avoid em dashes"), + "vetoes block:\n{body}" + ); + assert!( + body.contains("- Learn Rust this year"), + "goals block:\n{body}" + ); + } + + #[test] + fn skips_empty_classes_renders_placeholder() { + let (cache, renderer, tmp) = make_renderer(); + // Only insert a style facet; all other classes will be empty. + insert_facet( + &cache, + "style/verbosity", + "terse", + FacetState::Active, + UserState::Auto, + 2.0, + ); + + renderer.render().unwrap(); + + let body = std::fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); + // Empty classes get the placeholder. + assert!( + body.contains("*(no entries yet)*"), + "placeholder missing:\n{body}" + ); + // The style class has real content. + assert!(body.contains("- **verbosity**: terse")); + } + + #[test] + fn pinned_facets_marked_in_output() { + let (cache, renderer, tmp) = make_renderer(); + insert_facet( + &cache, + "style/format", + "markdown", + FacetState::Active, + UserState::Pinned, + 1.0, + ); + + renderer.render().unwrap(); + + let body = std::fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); + assert!( + body.contains("*(pinned)*"), + "pinned marker missing:\n{body}" + ); + assert!(body.contains("- **format**: markdown *(pinned)*")); + } + + #[test] + fn provisional_facets_excluded_from_output() { + let (cache, renderer, tmp) = make_renderer(); + insert_facet( + &cache, + "style/tone", + "formal", + FacetState::Provisional, + UserState::Auto, + 0.8, + ); + insert_facet( + &cache, + "style/verbosity", + "terse", + FacetState::Active, + UserState::Auto, + 2.0, + ); + + renderer.render().unwrap(); + + let body = std::fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); + assert!( + !body.contains("formal"), + "provisional must not appear:\n{body}" + ); + assert!(body.contains("terse")); + } + + #[test] + fn re_renders_idempotently_on_repeated_cache_rebuilt() { + let (cache, renderer, tmp) = make_renderer(); + insert_facet( + &cache, + "style/verbosity", + "terse", + FacetState::Active, + UserState::Auto, + 2.0, + ); + + renderer.render().unwrap(); + let body1 = std::fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); + renderer.render().unwrap(); + let body2 = std::fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); + + assert_eq!(body1, body2, "second render should be idempotent"); + } + + #[test] + fn renders_dont_clobber_connected_accounts_block() { + let (cache, renderer, tmp) = make_renderer(); + // Manually write a connected-accounts block first. + let ca_content = format!( + "{}\n## Connected Accounts\n\n- **Gmail** (c-1): jane@test.com\n{}\n", + block_start("connected-accounts"), + block_end("connected-accounts"), + ); + let profile_path = tmp.path().join("PROFILE.md"); + std::fs::write(&profile_path, format!("# User Profile\n\n{ca_content}")).unwrap(); + + insert_facet( + &cache, + "style/verbosity", + "terse", + FacetState::Active, + UserState::Auto, + 2.0, + ); + renderer.render().unwrap(); + + let body = std::fs::read_to_string(&profile_path).unwrap(); + // connected-accounts block preserved. + assert!( + body.contains("acct:gmail:c-1"), + "CA block clobbered:\n{body}" + ); + assert!( + body.contains("jane@test.com"), + "CA block clobbered:\n{body}" + ); + // Style block also written. + assert!(body.contains("terse")); + } + + #[test] + fn renders_dont_touch_user_authored_text_outside_blocks() { + let (cache, renderer, tmp) = make_renderer(); + let profile_path = tmp.path().join("PROFILE.md"); + std::fs::write( + &profile_path, + "# User Profile\n\nHand-written note by the user.\n", + ) + .unwrap(); + + insert_facet( + &cache, + "style/verbosity", + "terse", + FacetState::Active, + UserState::Auto, + 2.0, + ); + renderer.render().unwrap(); + + let body = std::fs::read_to_string(&profile_path).unwrap(); + assert!( + body.contains("Hand-written note by the user."), + "user text lost:\n{body}" + ); + assert!(body.contains("terse")); + } + + #[test] + fn subscribes_and_handles_cache_rebuilt_event() { + // Verify that ProfileMdRenderer::subscribe compiles and returns a handle. + // Full async event delivery is tested in the integration test. + let tmp = TempDir::new().unwrap(); + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(PROFILE_INIT_SQL).unwrap(); + let cache = make_cache(Arc::new(Mutex::new(conn))); + let renderer = Arc::new(ProfileMdRenderer::new(cache, tmp.path().to_path_buf())); + // subscribe_global requires a running runtime; just verify the type works. + let _renderer_ref = Arc::clone(&renderer); + // We can't call subscribe_global in a unit test without a tokio runtime, + // but we verify the subscriber type implements EventHandler correctly. + let subscriber = RendererSubscriber(renderer); + assert_eq!(subscriber.name(), "learning::profile_md_renderer"); + assert_eq!(subscriber.domains(), Some(["memory"].as_slice())); + } +} diff --git a/src/openhuman/learning/prompt_sections.rs b/src/openhuman/learning/prompt_sections.rs index 8402662f2..e32741d73 100644 --- a/src/openhuman/learning/prompt_sections.rs +++ b/src/openhuman/learning/prompt_sections.rs @@ -2,6 +2,21 @@ //! //! These sections read pre-fetched data from `PromptContext.learned` — no async //! or blocking I/O happens during prompt building. +//! +//! ## Phase 3 addition (#566) +//! +//! [`load_learned_from_cache`] reads Active facets from the `FacetCache` +//! (backed by `user_profile_facets`) and returns them as a list of formatted +//! strings suitable for injection into `LearnedContextData.user_profile`. +//! +//! The existing KV-namespace reads in `fetch_learned_context` are preserved +//! (both paths active in this phase; KV path will be removed in a follow-up). +//! +//! ## Phase 4 addition (#566) +//! +//! [`MemoryAccessSection`] — a static prompt section that instructs the agent to +//! call `memory_recall` / `memory_search` before answering questions that draw on +//! prior sessions. Registered after `LearnedContextSection` in the section chain. use crate::openhuman::context::prompt::{PromptContext, PromptSection}; use anyhow::Result; @@ -82,6 +97,124 @@ impl PromptSection for UserProfileSection { } } +// ── MemoryAccessSection ─────────────────────────────────────────────────────── + +/// Static bias instruction that tells the agent to call `memory_recall` before +/// answering questions involving named people, projects, prior decisions, or +/// anything the user mentioned in past sessions. +/// +/// The text is frozen at compile time — no I/O at build time. +/// Register this section after [`LearnedContextSection`] in the prompt-section +/// composition order (see `SystemPromptBuilder::with_defaults`). +pub struct MemoryAccessSection; + +/// The static prose injected into every system prompt. Kept at ≤ 80 tokens. +pub const MEMORY_ACCESS_INSTRUCTION: &str = "\ +## Memory access\n\ +\n\ +Before answering questions involving named people, projects, threads, prior \ +decisions, recurring topics, or anything the user has mentioned in past sessions, \ +call `memory_recall` (or `memory_search` for keyword lookups) to retrieve \ +relevant context. Surface what matters in your reply; don't stitch together \ +continuity from prompt history alone. Skip retrieval for purely procedural \ +requests where prior context isn't relevant."; + +impl PromptSection for MemoryAccessSection { + fn name(&self) -> &str { + "memory_access" + } + + fn build(&self, _ctx: &PromptContext<'_>) -> Result { + Ok(MEMORY_ACCESS_INSTRUCTION.to_string()) + } +} + +// ── Cache-backed loader ─────────────────────────────────────────────────────── + +/// Maximum number of facets to include in the ambient prompt injection. +/// +/// Corresponds to "~25 entries total" from the Phase 3 spec. +const CACHE_PROMPT_CAP: usize = 25; + +/// Load Active facets from the `FacetCache` and format them for prompt injection. +/// +/// Returns a list of strings in the form `class/key: value`, sorted by stability +/// descending within each class, then alphabetically by class. The total is capped +/// at [`CACHE_PROMPT_CAP`] entries. +/// +/// This function is **synchronous** and performs only SQLite reads — safe to call +/// from the synchronous part of the system prompt build path. The caller should +/// keep both this path and the existing KV-namespace path active until the KV path +/// is removed in a follow-up phase. +pub fn load_learned_from_cache( + cache: &crate::openhuman::learning::cache::FacetCache, +) -> Vec { + let facets = match cache.list_active() { + Ok(f) => f, + Err(e) => { + tracing::warn!("[learning::prompt] load_learned_from_cache failed: {e}"); + return Vec::new(); + } + }; + + if facets.is_empty() { + return Vec::new(); + } + + // Group by class prefix (portion before the first '/'), then sort within + // each class by stability descending, then by key alphabetically. + use crate::openhuman::memory::store::profile::ProfileFacet; + use std::collections::BTreeMap; + let mut by_class: BTreeMap> = BTreeMap::new(); + + for (idx, f) in facets.iter().enumerate() { + let class = f + .key + .split_once('/') + .map(|(prefix, _rest)| prefix.to_string()) + .unwrap_or_else(|| "other".to_string()); + by_class.entry(class).or_default().push(idx); + } + + for indices in by_class.values_mut() { + indices.sort_by(|&a, &b| { + facets[b] + .stability + .partial_cmp(&facets[a].stability) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| facets[a].key.cmp(&facets[b].key)) + }); + } + + let mut result = Vec::with_capacity(CACHE_PROMPT_CAP); + 'outer: for indices in by_class.values() { + for &idx in indices { + if result.len() >= CACHE_PROMPT_CAP { + break 'outer; + } + let f: &ProfileFacet = &facets[idx]; + // Phase 4: render in structured `class/key: value` form so the + // agent can parse the source. Goal class keeps value-only (full + // sentence, no key prefix). Pinned entries get a trailing suffix. + let pinned = + if f.user_state == crate::openhuman::memory::store::profile::UserState::Pinned { + " *(pinned)*" + } else { + "" + }; + let entry = if f.key.starts_with("goal/") { + // Goal class: render just the value, it's a sentence. + format!("{}{}", f.value, pinned) + } else { + format!("**{}**: {}{}", f.key, f.value, pinned) + }; + result.push(entry); + } + } + + result +} + #[cfg(test)] mod tests { use super::*; @@ -233,4 +366,153 @@ mod tests { .unwrap() .is_empty()); } + + // ── load_learned_from_cache ─────────────────────────────────────────────── + + #[test] + fn load_learned_from_cache_formats_active_facets() { + use crate::openhuman::learning::cache::FacetCache; + use crate::openhuman::memory::store::profile::{ + FacetState, FacetType, ProfileFacet, UserState, PROFILE_INIT_SQL, + }; + use parking_lot::Mutex; + use rusqlite::Connection; + + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(PROFILE_INIT_SQL).unwrap(); + let cache = FacetCache::new(Arc::new(Mutex::new(conn))); + + let make_facet = |id: &str, key: &str, value: &str, stab: f64| ProfileFacet { + facet_id: id.into(), + facet_type: FacetType::Preference, + key: key.into(), + value: value.into(), + confidence: 0.8, + evidence_count: 2, + source_segment_ids: None, + first_seen_at: 1000.0, + last_seen_at: 1200.0, + state: FacetState::Active, + stability: stab, + user_state: UserState::Auto, + evidence_refs: vec![], + class: None, + cue_families: None, + }; + + cache + .upsert(&make_facet("f1", "style/verbosity", "terse", 2.0)) + .unwrap(); + cache + .upsert(&make_facet("f2", "identity/name", "Alice", 1.8)) + .unwrap(); + cache + .upsert(&make_facet( + "f3", + "goal/learn_rust", + "Learn Rust this year", + 1.6, + )) + .unwrap(); + + // Provisional — should NOT appear. + let mut prov = make_facet("f4", "style/tone", "formal", 0.8); + prov.state = FacetState::Provisional; + cache.upsert(&prov).unwrap(); + + let result = load_learned_from_cache(&cache); + + assert!( + !result.is_empty(), + "should produce entries for Active facets" + ); + // Phase 4 format: "**style/verbosity**: terse" + assert!( + result.iter().any(|s| s.contains("style/verbosity")), + "style/verbosity should appear" + ); + assert!( + result + .iter() + .any(|s| s.contains("**style/verbosity**: terse")), + "style/verbosity should use Phase 4 bold format" + ); + // Goal class → value only (no key prefix) + assert!( + result.iter().any(|s| s == "Learn Rust this year"), + "goal class should render value only" + ); + // Provisional should not appear + assert!( + !result.iter().any(|s| s.contains("style/tone")), + "provisional facet must not appear in cache prompt" + ); + } + + #[test] + fn load_learned_from_cache_empty_when_no_active_facets() { + use crate::openhuman::learning::cache::FacetCache; + use crate::openhuman::memory::store::profile::PROFILE_INIT_SQL; + use parking_lot::Mutex; + use rusqlite::Connection; + + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(PROFILE_INIT_SQL).unwrap(); + let cache = FacetCache::new(Arc::new(Mutex::new(conn))); + + let result = load_learned_from_cache(&cache); + assert!(result.is_empty()); + } + + // ── MemoryAccessSection ─────────────────────────────────────────────────── + + #[test] + fn memory_access_section_renders_static_text() { + let section = MemoryAccessSection; + assert_eq!(section.name(), "memory_access"); + let rendered = section + .build(&prompt_context(LearnedContextData::default())) + .unwrap(); + assert!( + rendered.contains("## Memory access"), + "heading missing:\n{rendered}" + ); + assert!( + rendered.contains("memory_recall"), + "memory_recall tool not mentioned:\n{rendered}" + ); + assert!( + rendered.contains("memory_search"), + "memory_search tool not mentioned:\n{rendered}" + ); + // Verify the rendered text matches the constant. + assert_eq!(rendered.trim(), MEMORY_ACCESS_INSTRUCTION.trim()); + } + + #[test] + fn memory_access_section_present_in_system_prompt_compose() { + // Verify the section renders correctly when added to a prompt composition. + let section = MemoryAccessSection; + let rendered = section + .build(&prompt_context(LearnedContextData::default())) + .unwrap(); + // Spot-check the content constraint: ≤ 80 tokens (rough word count). + let word_count = rendered.split_whitespace().count(); + assert!( + word_count <= 100, + "MemoryAccessSection is too long ({word_count} words, target ≤ 80 tokens)" + ); + // The section name must be stable (used for insert_section_before). + assert_eq!(section.name(), "memory_access"); + // Content check: the section must mention both retrieval tools. + assert!(rendered.contains("memory_recall")); + assert!(rendered.contains("memory_search")); + // Verify it is non-empty for any PromptContext (not context-gated). + let empty_ctx = prompt_context(LearnedContextData::default()); + let rendered_empty_ctx = section.build(&empty_ctx).unwrap(); + assert!( + !rendered_empty_ctx.trim().is_empty(), + "must render regardless of learned context" + ); + } } diff --git a/src/openhuman/learning/reflection.rs b/src/openhuman/learning/reflection.rs index a2089c902..ffcc291c4 100644 --- a/src/openhuman/learning/reflection.rs +++ b/src/openhuman/learning/reflection.rs @@ -6,12 +6,14 @@ use crate::openhuman::agent::hooks::{PostTurnHook, TurnContext}; use crate::openhuman::config::{Config, LearningConfig, ReflectionSource}; +use crate::openhuman::learning::candidate::{self, CueFamily, EvidenceRef, FacetClass}; use crate::openhuman::memory::{Memory, MemoryCategory}; use async_trait::async_trait; use parking_lot::Mutex; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; /// Memory namespace + custom-category tag for explicit user reflections. /// @@ -285,6 +287,105 @@ impl ReflectionHook { Ok(()) } + /// Emit [`LearningCandidate`]s into the global buffer from a completed + /// reflection output. This runs after the existing KV memory writes so + /// backward compatibility is preserved. + /// + /// - Heuristic cues → `Goal` candidates (Explicit, confidence 0.85) + /// - LLM `user_preferences` entries → `Style` candidates (Behavioral, 0.7) + /// - LLM `user_reflections` entries → `Goal` candidates (Behavioral, 0.7) + fn emit_candidates_from_reflection(output: &ReflectionOutput, episodic_id: i64) { + let now = now_secs(); + let buf = candidate::global(); + let mut count = 0usize; + + // user_preferences — coarse style candidates. + for (n, pref) in output.user_preferences.iter().enumerate() { + let pref = pref.trim(); + if pref.is_empty() { + continue; + } + // Attempt cheap key=value parse first ("verbosity=terse"). + let (key, value) = if let Some(eq_pos) = pref.find('=') { + let k = pref[..eq_pos].trim().to_string(); + let v = pref[eq_pos + 1..].trim().to_string(); + if !k.is_empty() && !v.is_empty() { + (k, v) + } else { + (format!("raw_pref_{n}"), pref.to_string()) + } + } else { + (format!("raw_pref_{n}"), pref.to_string()) + }; + + buf.push(crate::openhuman::learning::candidate::LearningCandidate { + class: FacetClass::Style, + key, + value, + cue_family: CueFamily::Behavioral, + evidence: EvidenceRef::Episodic { episodic_id }, + initial_confidence: 0.70, + observed_at: now, + }); + count += 1; + } + + // user_reflections — explicit user self-statements, map to Goal. + for reflection in &output.user_reflections { + let trimmed = reflection.trim(); + if trimmed.is_empty() { + continue; + } + let key = slugify_candidate(trimmed); + buf.push(crate::openhuman::learning::candidate::LearningCandidate { + class: FacetClass::Goal, + key, + value: trimmed.to_string(), + cue_family: CueFamily::Behavioral, + evidence: EvidenceRef::Episodic { episodic_id }, + initial_confidence: 0.70, + observed_at: now, + }); + count += 1; + } + + if count > 0 { + log::debug!( + "[learning::reflection] emitted {} candidate(s) to buffer from reflection output", + count + ); + } + } + + /// Emit Goal candidates into the global buffer from heuristic cue sentences. + fn emit_candidates_from_cues(cues: &[String], episodic_id: i64) { + if cues.is_empty() { + return; + } + let now = now_secs(); + let buf = candidate::global(); + for cue in cues { + let trimmed = cue.trim(); + if trimmed.is_empty() { + continue; + } + let key = slugify_candidate(trimmed); + buf.push(crate::openhuman::learning::candidate::LearningCandidate { + class: FacetClass::Goal, + key, + value: trimmed.to_string(), + cue_family: CueFamily::Explicit, + evidence: EvidenceRef::Episodic { episodic_id }, + initial_confidence: 0.85, + observed_at: now, + }); + } + log::debug!( + "[learning::reflection] emitted {} heuristic cue candidate(s) to buffer", + cues.len() + ); + } + /// Persist a single reflection sentence into the dedicated namespace. /// Public to the crate so the heuristic fast-path can reuse the same /// storage shape without going through the LLM round-trip. @@ -423,9 +524,16 @@ impl PostTurnHook for ReflectionHook { // reflections like "remember that I prefer terse answers" are // promoted to the privileged reflection namespace without paying // for a reflection-LLM round-trip. + // + // Also emits Goal candidates to the learning buffer (Phase 2). if self.config.enabled { - for cue in extract_reflection_cues(&ctx.user_message) { - if let Err(e) = self.persist_reflection_deduped(&cue, &mut seen).await { + let cues = extract_reflection_cues(&ctx.user_message); + // Emit candidates before the KV persist so even a failed persist + // doesn't prevent the buffer from seeing the signal. + let episodic_id = crate::openhuman::learning::candidate::global().len() as i64; + Self::emit_candidates_from_cues(&cues, episodic_id); + for cue in &cues { + if let Err(e) = self.persist_reflection_deduped(cue, &mut seen).await { log::warn!("[learning] failed to persist heuristic reflection: {e}"); } } @@ -480,6 +588,12 @@ impl PostTurnHook for ReflectionHook { return Err(e); } + // Emit learning candidates from LLM output into the global buffer (Phase 2). + // Use the current buffer length as a synthetic episodic_id proxy — Phase 3 + // will replace this with a real episodic_log row id. + let episodic_id = crate::openhuman::learning::candidate::global().len() as i64; + Self::emit_candidates_from_reflection(&output, episodic_id); + // Persist LLM-extracted reflections through the shared dedupe // set so any sentence the heuristic already captured above is // not written twice. Failures here are logged but never roll @@ -520,6 +634,19 @@ fn slugify(s: &str) -> String { .collect() } +/// Slugify for candidate keys — same logic as `slugify` but named distinctly to +/// clarify the caller's intent. +fn slugify_candidate(s: &str) -> String { + slugify(s) +} + +fn now_secs() -> f64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64() +} + #[cfg(test)] #[path = "reflection_tests.rs"] mod tests; diff --git a/src/openhuman/learning/reflection_tests.rs b/src/openhuman/learning/reflection_tests.rs index 8d64255a7..9d9d65f85 100644 --- a/src/openhuman/learning/reflection_tests.rs +++ b/src/openhuman/learning/reflection_tests.rs @@ -439,3 +439,81 @@ async fn on_turn_complete_rolls_back_counter_when_reflection_call_fails() { 0 ); } + +#[tokio::test] +async fn on_turn_complete_emits_candidates_to_buffer_for_heuristic_cues() { + use crate::openhuman::learning::candidate::{self, FacetClass}; + + let memory: Arc = Arc::new(MockMemory::default()); + // Use local reflection source with complexity gated high so only the heuristic + // fast-path runs (no LLM round-trip needed). + let mut cfg = reflection_config(); + cfg.min_turn_complexity = 99; + cfg.reflection_source = ReflectionSource::Local; + let hook = ReflectionHook::new(cfg, Arc::new(Config::default()), memory, None); + + // Snapshot the buffer before the call. + let before = candidate::global().len(); + + let turn = TurnContext { + user_message: "Going forward I want concise replies only.".into(), + assistant_response: "ok".into(), + tool_calls: Vec::new(), + turn_duration_ms: 10, + session_id: Some("buffer-test".into()), + iteration_count: 1, + }; + hook.on_turn_complete(&turn).await.unwrap(); + + let after = candidate::global().peek(); + let new_candidates: Vec<_> = after.into_iter().skip(before).collect(); + + assert!( + !new_candidates.is_empty(), + "on_turn_complete should push at least one candidate to the buffer for heuristic cues" + ); + assert!( + new_candidates.iter().any(|c| c.class == FacetClass::Goal), + "heuristic cues should produce Goal candidates" + ); +} + +#[tokio::test] +async fn on_turn_complete_emits_style_candidates_from_llm_preferences() { + use crate::openhuman::learning::candidate::{self, FacetClass}; + use crate::openhuman::providers::Provider; + + struct StubPrefProvider; + #[async_trait] + impl Provider for StubPrefProvider { + async fn chat_with_system( + &self, + _system_prompt: Option<&str>, + _message: &str, + _model: &str, + _temperature: f64, + ) -> anyhow::Result { + Ok(r#"{"observations":[],"patterns":[],"user_preferences":["verbosity=terse"],"user_reflections":[]}"#.into()) + } + } + + let memory: Arc = Arc::new(MockMemory::default()); + let hook = ReflectionHook::new( + reflection_config(), + Arc::new(Config::default()), + memory, + Some(Arc::new(StubPrefProvider)), + ); + + let before = candidate::global().len(); + hook.on_turn_complete(&reflective_turn()).await.unwrap(); + let all = candidate::global().peek(); + let new_candidates: Vec<_> = all.into_iter().skip(before).collect(); + + assert!( + new_candidates + .iter() + .any(|c| c.class == FacetClass::Style && c.key == "verbosity"), + "user_preferences with key=value format should produce a Style candidate" + ); +} diff --git a/src/openhuman/learning/scheduler.rs b/src/openhuman/learning/scheduler.rs new file mode 100644 index 000000000..ce1faff70 --- /dev/null +++ b/src/openhuman/learning/scheduler.rs @@ -0,0 +1,164 @@ +//! Background scheduler for the stability detector rebuild cycle. +//! +//! Provides two scheduling mechanisms: +//! +//! 1. **Periodic timer** — `spawn_rebuild_loop` runs a rebuild every `interval` +//! (default 30 minutes). +//! +//! 2. **Event-driven trigger** — `spawn_event_driven_trigger` subscribes to +//! [`DomainEvent::DocumentCanonicalized`] and [`DomainEvent::TreeSummarizerPropagated`] +//! events and schedules a debounced out-of-cycle rebuild 60 seconds after receipt. +//! +//! Both mechanisms respect a shutdown signal from a +//! `tokio::sync::watch::Receiver` channel. + +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use async_trait::async_trait; +use tokio::sync::watch; + +use crate::core::event_bus::{subscribe_global, DomainEvent, EventHandler, SubscriptionHandle}; +// Arc imported for detector sharing across async tasks +use crate::openhuman::learning::stability_detector::StabilityDetector; + +// ── Default intervals ───────────────────────────────────────────────────────── + +/// Default periodic rebuild interval. +pub const DEFAULT_REBUILD_INTERVAL: Duration = Duration::from_secs(30 * 60); + +/// Debounce delay for event-driven rebuilds — we wait this long after the last +/// triggering event before running the rebuild. +const EVENT_REBUILD_DELAY: Duration = Duration::from_secs(60); + +// ── Periodic rebuild loop ───────────────────────────────────────────────────── + +/// Spawn a background task that runs a stability detector rebuild every `interval`. +/// +/// The task shuts down when `shutdown_rx` signals `true` or when the tokio +/// runtime exits. +/// +/// # Arguments +/// +/// * `detector` — the rebuild engine to call. +/// * `interval` — how often to rebuild (default: [`DEFAULT_REBUILD_INTERVAL`]). +/// * `shutdown_rx` — watch channel; the task exits when the value becomes `true`. +pub fn spawn_rebuild_loop( + detector: Arc, + interval: Duration, + mut shutdown_rx: watch::Receiver, +) { + tracing::info!( + "[learning::scheduler] starting periodic rebuild loop (interval={}s)", + interval.as_secs() + ); + + tokio::spawn(async move { + let mut ticker = tokio::time::interval(interval); + // Skip the first immediate tick so we don't rebuild at startup before + // any producers have had a chance to emit candidates. + ticker.tick().await; + + loop { + tokio::select! { + _ = ticker.tick() => { + run_rebuild_logged(&detector, "periodic").await; + } + _ = shutdown_rx.changed() => { + if *shutdown_rx.borrow() { + tracing::info!("[learning::scheduler] shutdown signal received, stopping rebuild loop"); + break; + } + } + } + } + + tracing::info!("[learning::scheduler] rebuild loop stopped"); + }); +} + +// ── Event-driven trigger ────────────────────────────────────────────────────── + +/// Event handler that schedules a debounced rebuild on relevant domain events. +/// +/// Listens for: +/// - [`DomainEvent::DocumentCanonicalized`] with `source_kind == "email"` or `"document"` +/// - [`DomainEvent::TreeSummarizerPropagated`] (tree summariser flush signal) +struct RebuildTriggerHandler { + detector: Arc, +} + +#[async_trait] +impl EventHandler for RebuildTriggerHandler { + fn name(&self) -> &str { + "learning::rebuild_trigger" + } + + fn domains(&self) -> Option<&[&str]> { + Some(&["memory", "tree_summarizer"]) + } + + async fn handle(&self, event: &DomainEvent) { + let should_trigger = match event { + DomainEvent::DocumentCanonicalized { source_kind, .. } => { + matches!(source_kind.as_str(), "email" | "document") + } + DomainEvent::TreeSummarizerPropagated { .. } => true, + _ => false, + }; + + if !should_trigger { + return; + } + + tracing::debug!( + "[learning::scheduler] event-driven rebuild triggered, scheduling in {}s", + EVENT_REBUILD_DELAY.as_secs() + ); + + let detector = Arc::clone(&self.detector); + let delay = EVENT_REBUILD_DELAY; + + tokio::spawn(async move { + tokio::time::sleep(delay).await; + run_rebuild_logged(&detector, "event-driven").await; + }); + } +} + +/// Register the event-driven trigger subscriber. +/// +/// The returned `SubscriptionHandle` must be kept alive for the subscription to +/// remain active. Callers should store it in a static `OnceLock` (same pattern +/// as the `EmailSignatureSubscriber`). +pub fn register_event_trigger(detector: Arc) -> Option { + subscribe_global(Arc::new(RebuildTriggerHandler { detector })) +} + +// ── Shared rebuild runner ───────────────────────────────────────────────────── + +async fn run_rebuild_logged(detector: &StabilityDetector, source: &str) { + let now = now_secs(); + match detector.rebuild(now) { + Ok(outcome) => { + tracing::info!( + "[learning::scheduler] {source} rebuild complete: \ + added={} evicted={} kept={} total={}", + outcome.added, + outcome.evicted, + outcome.kept, + outcome.total_size, + ); + } + Err(e) => { + tracing::warn!("[learning::scheduler] {source} rebuild failed (non-fatal): {e}"); + } + } +} + +fn now_secs() -> f64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64() +} diff --git a/src/openhuman/learning/schemas.rs b/src/openhuman/learning/schemas.rs index de57cf449..62594eeb1 100644 --- a/src/openhuman/learning/schemas.rs +++ b/src/openhuman/learning/schemas.rs @@ -11,6 +11,15 @@ pub fn all_learning_controller_schemas() -> Vec { vec![ learning_schemas("learning_linkedin_enrichment"), learning_schemas("learning_save_profile"), + learning_schemas("learning_rebuild_cache"), + learning_schemas("learning_cache_stats"), + learning_schemas("learning_list_facets"), + learning_schemas("learning_get_facet"), + learning_schemas("learning_update_facet"), + learning_schemas("learning_pin_facet"), + learning_schemas("learning_unpin_facet"), + learning_schemas("learning_forget_facet"), + learning_schemas("learning_reset_cache"), ] } @@ -24,6 +33,42 @@ pub fn all_learning_registered_controllers() -> Vec { schema: learning_schemas("learning_save_profile"), handler: handle_save_profile, }, + RegisteredController { + schema: learning_schemas("learning_rebuild_cache"), + handler: handle_rebuild_cache, + }, + RegisteredController { + schema: learning_schemas("learning_cache_stats"), + handler: handle_cache_stats, + }, + RegisteredController { + schema: learning_schemas("learning_list_facets"), + handler: handle_list_facets, + }, + RegisteredController { + schema: learning_schemas("learning_get_facet"), + handler: handle_get_facet, + }, + RegisteredController { + schema: learning_schemas("learning_update_facet"), + handler: handle_update_facet, + }, + RegisteredController { + schema: learning_schemas("learning_pin_facet"), + handler: handle_pin_facet, + }, + RegisteredController { + schema: learning_schemas("learning_unpin_facet"), + handler: handle_unpin_facet, + }, + RegisteredController { + schema: learning_schemas("learning_forget_facet"), + handler: handle_forget_facet, + }, + RegisteredController { + schema: learning_schemas("learning_reset_cache"), + handler: handle_reset_cache, + }, ] } @@ -99,6 +144,280 @@ pub fn learning_schemas(function: &str) -> ControllerSchema { }, ], }, + "learning_rebuild_cache" => ControllerSchema { + namespace: "learning", + function: "rebuild_cache", + description: "Manually trigger a stability-detector rebuild cycle. \ + Drains the candidate buffer, scores all (class, key) pairs, \ + applies class budgets, and persists the updated ambient cache. \ + Returns rebuild statistics.", + inputs: vec![], + outputs: vec![ + FieldSchema { + name: "added", + ty: TypeSchema::U64, + comment: "Facet rows newly created in this cycle.", + required: true, + }, + FieldSchema { + name: "evicted", + ty: TypeSchema::U64, + comment: "Facet rows demoted to Dropped or deleted.", + required: true, + }, + FieldSchema { + name: "kept", + ty: TypeSchema::U64, + comment: "Facet rows carried over unchanged.", + required: true, + }, + FieldSchema { + name: "total_size", + ty: TypeSchema::U64, + comment: "Total Active rows after the rebuild.", + required: true, + }, + ], + }, + "learning_cache_stats" => ControllerSchema { + namespace: "learning", + function: "cache_stats", + description: "Return current ambient cache statistics — total row count, \ + per-state breakdown, and per-class breakdown.", + inputs: vec![], + outputs: vec![ + FieldSchema { + name: "total", + ty: TypeSchema::U64, + comment: "Total rows in the cache (all states).", + required: true, + }, + FieldSchema { + name: "active", + ty: TypeSchema::U64, + comment: "Rows with state=active.", + required: true, + }, + FieldSchema { + name: "provisional", + ty: TypeSchema::U64, + comment: "Rows with state=provisional.", + required: true, + }, + FieldSchema { + name: "candidate", + ty: TypeSchema::U64, + comment: "Rows with state=candidate.", + required: true, + }, + FieldSchema { + name: "dropped", + ty: TypeSchema::U64, + comment: "Rows with state=dropped.", + required: true, + }, + FieldSchema { + name: "by_class", + ty: TypeSchema::Json, + comment: "Map of class name → row count (active rows only).", + required: true, + }, + ], + }, + "learning_list_facets" => ControllerSchema { + namespace: "learning", + function: "list_facets", + description: "List all facets in the ambient cache (active + provisional). \ + Optionally filter by class.", + inputs: vec![FieldSchema { + name: "class", + ty: TypeSchema::Option(Box::new(TypeSchema::String)), + comment: + "Optional class filter: style | identity | tooling | veto | goal | channel.", + required: false, + }], + outputs: vec![ + FieldSchema { + name: "facets", + ty: TypeSchema::Array(Box::new(TypeSchema::Json)), + comment: + "Array of facet objects with key, value, state, user_state, stability.", + required: true, + }, + FieldSchema { + name: "count", + ty: TypeSchema::U64, + comment: "Total number of facets returned.", + required: true, + }, + ], + }, + "learning_get_facet" => ControllerSchema { + namespace: "learning", + function: "get_facet", + description: + "Fetch a single facet by class and key suffix (e.g. class=style, key=verbosity).", + inputs: vec![ + FieldSchema { + name: "class", + ty: TypeSchema::String, + comment: "Facet class: style | identity | tooling | veto | goal | channel.", + required: true, + }, + FieldSchema { + name: "key", + ty: TypeSchema::String, + comment: + "Key suffix within the class (e.g. \"verbosity\" for style/verbosity).", + required: true, + }, + ], + outputs: vec![ + FieldSchema { + name: "facet", + ty: TypeSchema::Option(Box::new(TypeSchema::Json)), + comment: "The facet object, or null if not found.", + required: false, + }, + FieldSchema { + name: "found", + ty: TypeSchema::Bool, + comment: "Whether the facet was found.", + required: true, + }, + ], + }, + "learning_update_facet" => ControllerSchema { + namespace: "learning", + function: "update_facet", + description: "Update the value of an existing facet and pin it (user_state=Pinned). \ + Returns the updated facet.", + inputs: vec![ + FieldSchema { + name: "class", + ty: TypeSchema::String, + comment: "Facet class: style | identity | tooling | veto | goal | channel.", + required: true, + }, + FieldSchema { + name: "key", + ty: TypeSchema::String, + comment: "Key suffix within the class.", + required: true, + }, + FieldSchema { + name: "value", + ty: TypeSchema::String, + comment: "New value to set.", + required: true, + }, + ], + outputs: vec![FieldSchema { + name: "facet", + ty: TypeSchema::Json, + comment: "The updated facet object.", + required: true, + }], + }, + "learning_pin_facet" => ControllerSchema { + namespace: "learning", + function: "pin_facet", + description: + "Pin a facet (user_state=Pinned): locks Active regardless of stability score.", + inputs: vec![ + FieldSchema { + name: "class", + ty: TypeSchema::String, + comment: "Facet class.", + required: true, + }, + FieldSchema { + name: "key", + ty: TypeSchema::String, + comment: "Key suffix within the class.", + required: true, + }, + ], + outputs: vec![FieldSchema { + name: "facet", + ty: TypeSchema::Json, + comment: "The updated facet object.", + required: true, + }], + }, + "learning_unpin_facet" => ControllerSchema { + namespace: "learning", + function: "unpin_facet", + description: + "Unpin a facet (user_state=Auto): returns stability management to the detector.", + inputs: vec![ + FieldSchema { + name: "class", + ty: TypeSchema::String, + comment: "Facet class.", + required: true, + }, + FieldSchema { + name: "key", + ty: TypeSchema::String, + comment: "Key suffix within the class.", + required: true, + }, + ], + outputs: vec![FieldSchema { + name: "facet", + ty: TypeSchema::Json, + comment: "The updated facet object.", + required: true, + }], + }, + "learning_forget_facet" => ControllerSchema { + namespace: "learning", + function: "forget_facet", + description: + "Forget a facet (user_state=Forgotten): locks Dropped and blocks re-promotion.", + inputs: vec![ + FieldSchema { + name: "class", + ty: TypeSchema::String, + comment: "Facet class.", + required: true, + }, + FieldSchema { + name: "key", + ty: TypeSchema::String, + comment: "Key suffix within the class.", + required: true, + }, + ], + outputs: vec![FieldSchema { + name: "facet", + ty: TypeSchema::Option(Box::new(TypeSchema::Json)), + comment: "The facet in its Dropped state, or null if it didn't exist.", + required: false, + }], + }, + "learning_reset_cache" => ControllerSchema { + namespace: "learning", + function: "reset_cache", + description: "Reset the ambient cache: delete all Auto rows, preserve Pinned rows. \ + The next rebuild repopulates from the substrate.", + inputs: vec![], + outputs: vec![ + FieldSchema { + name: "deleted", + ty: TypeSchema::U64, + comment: "Number of Auto rows deleted.", + required: true, + }, + FieldSchema { + name: "pinned_preserved", + ty: TypeSchema::U64, + comment: "Number of Pinned rows kept.", + required: true, + }, + ], + }, _ => ControllerSchema { namespace: "learning", function: "unknown", @@ -119,13 +438,13 @@ mod tests { use super::*; #[test] - fn all_schemas_returns_two() { - assert_eq!(all_learning_controller_schemas().len(), 2); + fn all_schemas_returns_eleven() { + assert_eq!(all_learning_controller_schemas().len(), 11); } #[test] - fn all_controllers_returns_two() { - assert_eq!(all_learning_registered_controllers().len(), 2); + fn all_controllers_returns_eleven() { + assert_eq!(all_learning_registered_controllers().len(), 11); } #[test] @@ -162,6 +481,57 @@ mod tests { let c = all_learning_registered_controllers(); assert_eq!(s[0].function, c[0].schema.function); } + + #[test] + fn list_facets_schema_shape() { + let s = learning_schemas("learning_list_facets"); + assert_eq!(s.namespace, "learning"); + assert_eq!(s.function, "list_facets"); + assert!(s.inputs.iter().any(|f| f.name == "class" && !f.required)); + assert!(s.outputs.iter().any(|f| f.name == "facets")); + assert!(s.outputs.iter().any(|f| f.name == "count")); + } + + #[test] + fn get_facet_schema_shape() { + let s = learning_schemas("learning_get_facet"); + assert_eq!(s.function, "get_facet"); + assert!(s.inputs.iter().any(|f| f.name == "class" && f.required)); + assert!(s.inputs.iter().any(|f| f.name == "key" && f.required)); + } + + #[test] + fn update_facet_schema_shape() { + let s = learning_schemas("learning_update_facet"); + assert_eq!(s.function, "update_facet"); + assert!(s.inputs.iter().any(|f| f.name == "value" && f.required)); + } + + #[test] + fn pin_facet_schema_shape() { + let s = learning_schemas("learning_pin_facet"); + assert_eq!(s.function, "pin_facet"); + } + + #[test] + fn unpin_facet_schema_shape() { + let s = learning_schemas("learning_unpin_facet"); + assert_eq!(s.function, "unpin_facet"); + } + + #[test] + fn forget_facet_schema_shape() { + let s = learning_schemas("learning_forget_facet"); + assert_eq!(s.function, "forget_facet"); + } + + #[test] + fn reset_cache_schema_shape() { + let s = learning_schemas("learning_reset_cache"); + assert_eq!(s.function, "reset_cache"); + assert!(s.outputs.iter().any(|f| f.name == "deleted")); + assert!(s.outputs.iter().any(|f| f.name == "pinned_preserved")); + } } fn handle_linkedin_enrichment(params: Map) -> ControllerFuture { @@ -231,3 +601,444 @@ fn handle_save_profile(params: Map) -> ControllerFuture { RpcOutcome::new(payload, log).into_cli_compatible_json() }) } + +fn handle_rebuild_cache(_params: Map) -> ControllerFuture { + Box::pin(async move { + use crate::openhuman::learning::cache::FacetCache; + use crate::openhuman::learning::stability_detector::StabilityDetector; + use std::time::{SystemTime, UNIX_EPOCH}; + + tracing::debug!("[learning.rebuild_cache] manual rebuild requested via RPC"); + + let client = crate::openhuman::memory::global::client_if_ready() + .ok_or_else(|| "memory client not ready".to_string())?; + let conn = client.profile_conn(); + let cache = FacetCache::new(conn); + let detector = StabilityDetector::new(cache); + + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64(); + + let outcome = detector + .rebuild(now) + .map_err(|e| format!("rebuild failed: {e:#}"))?; + + let log = vec![format!( + "learning.rebuild_cache: added={} evicted={} kept={} total={}", + outcome.added, outcome.evicted, outcome.kept, outcome.total_size, + )]; + + let payload = serde_json::json!({ + "added": outcome.added, + "evicted": outcome.evicted, + "kept": outcome.kept, + "total_size": outcome.total_size, + }); + + RpcOutcome::new(payload, log).into_cli_compatible_json() + }) +} + +fn handle_cache_stats(_params: Map) -> ControllerFuture { + Box::pin(async move { + use crate::openhuman::learning::cache::FacetCache; + use crate::openhuman::memory::store::profile::FacetState; + + tracing::debug!("[learning.cache_stats] cache stats requested via RPC"); + + let client = crate::openhuman::memory::global::client_if_ready() + .ok_or_else(|| "memory client not ready".to_string())?; + let conn = client.profile_conn(); + let cache = FacetCache::new(conn); + + let all_facets = cache + .list_all() + .map_err(|e| format!("list_all failed: {e:#}"))?; + + let total = all_facets.len(); + let active = all_facets + .iter() + .filter(|f| f.state == FacetState::Active) + .count(); + let provisional = all_facets + .iter() + .filter(|f| f.state == FacetState::Provisional) + .count(); + let candidate = all_facets + .iter() + .filter(|f| f.state == FacetState::Candidate) + .count(); + let dropped = all_facets + .iter() + .filter(|f| f.state == FacetState::Dropped) + .count(); + + // Per-class count (Active rows only, keyed by class field or key prefix). + let mut by_class: std::collections::HashMap = + std::collections::HashMap::new(); + for f in all_facets.iter().filter(|f| f.state == FacetState::Active) { + let cls = f + .class + .clone() + .or_else(|| f.key.split_once('/').map(|(p, _)| p.to_string())) + .unwrap_or_else(|| "_other".to_string()); + *by_class.entry(cls).or_insert(0) += 1; + } + + let log = vec![format!( + "learning.cache_stats: total={total} active={active} provisional={provisional} \ + candidate={candidate} dropped={dropped}" + )]; + + let payload = serde_json::json!({ + "total": total, + "active": active, + "provisional": provisional, + "candidate": candidate, + "dropped": dropped, + "by_class": by_class, + }); + + RpcOutcome::new(payload, log).into_cli_compatible_json() + }) +} + +// ── Helper: shared cache access ─────────────────────────────────────────────── + +/// Build a [`FacetCache`] from the global memory client, or return a string error. +fn get_cache() -> Result { + let client = crate::openhuman::memory::global::client_if_ready() + .ok_or_else(|| "memory client not ready".to_string())?; + Ok(crate::openhuman::learning::cache::FacetCache::new( + client.profile_conn(), + )) +} + +/// Build the full facet key from class string + key suffix. +/// E.g. (`"style"`, `"verbosity"`) → `"style/verbosity"`. +fn full_key(class_str: &str, key_suffix: &str) -> String { + format!("{class_str}/{key_suffix}") +} + +/// Serialize a [`ProfileFacet`] to a serde_json [`Value`] for RPC output. +fn facet_to_json(f: &crate::openhuman::memory::store::profile::ProfileFacet) -> serde_json::Value { + serde_json::json!({ + "key": f.key, + "value": f.value, + "state": f.state.as_str(), + "user_state": f.user_state.as_str(), + "stability": f.stability, + "confidence": f.confidence, + "evidence_count": f.evidence_count, + "first_seen_at": f.first_seen_at, + "last_seen_at": f.last_seen_at, + "class": f.class, + }) +} + +// ── list_facets ─────────────────────────────────────────────────────────────── + +fn handle_list_facets(params: Map) -> ControllerFuture { + Box::pin(async move { + use crate::openhuman::memory::store::profile::FacetState; + + tracing::debug!("[learning.list_facets] called"); + + let class_filter = params + .get("class") + .and_then(Value::as_str) + .map(str::to_string); + + let cache = get_cache()?; + + // list_all returns all states (active + provisional + candidate + dropped). + let all = cache + .list_all() + .map_err(|e| format!("list_all failed: {e:#}"))?; + + let facets: Vec = all + .iter() + .filter(|f| { + // Expose Active and Provisional rows to the user. + f.state == FacetState::Active || f.state == FacetState::Provisional + }) + .filter(|f| { + if let Some(cls) = &class_filter { + f.class.as_deref() == Some(cls.as_str()) + || f.key.starts_with(&format!("{cls}/")) + } else { + true + } + }) + .map(facet_to_json) + .collect(); + + let count = facets.len(); + let log = vec![format!( + "learning.list_facets: returned {count} facets (class_filter={:?})", + class_filter + )]; + + let payload = serde_json::json!({ "facets": facets, "count": count }); + RpcOutcome::new(payload, log).into_cli_compatible_json() + }) +} + +// ── get_facet ───────────────────────────────────────────────────────────────── + +fn handle_get_facet(params: Map) -> ControllerFuture { + Box::pin(async move { + let class_str = params + .get("class") + .and_then(Value::as_str) + .ok_or_else(|| "missing required `class`".to_string())? + .to_string(); + let key_suffix = params + .get("key") + .and_then(Value::as_str) + .ok_or_else(|| "missing required `key`".to_string())? + .to_string(); + + let fk = full_key(&class_str, &key_suffix); + tracing::debug!("[learning.get_facet] key={fk}"); + + let cache = get_cache()?; + let facet = cache.get(&fk).map_err(|e| format!("get failed: {e:#}"))?; + + let (found, facet_val) = match &facet { + Some(f) => (true, facet_to_json(f)), + None => (false, serde_json::Value::Null), + }; + + let log = vec![format!("learning.get_facet: key={fk} found={found}")]; + let payload = serde_json::json!({ "facet": facet_val, "found": found }); + RpcOutcome::new(payload, log).into_cli_compatible_json() + }) +} + +// ── update_facet ────────────────────────────────────────────────────────────── + +fn handle_update_facet(params: Map) -> ControllerFuture { + Box::pin(async move { + use crate::openhuman::memory::store::profile::UserState; + + let class_str = params + .get("class") + .and_then(Value::as_str) + .ok_or_else(|| "missing required `class`".to_string())? + .to_string(); + let key_suffix = params + .get("key") + .and_then(Value::as_str) + .ok_or_else(|| "missing required `key`".to_string())? + .to_string(); + let new_value = params + .get("value") + .and_then(Value::as_str) + .ok_or_else(|| "missing required `value`".to_string())? + .to_string(); + + let fk = full_key(&class_str, &key_suffix); + tracing::debug!("[learning.update_facet] key={fk} value={new_value}"); + + let cache = get_cache()?; + + let mut facet = cache + .get(&fk) + .map_err(|e| format!("get failed: {e:#}"))? + .ok_or_else(|| format!("facet not found: {fk}"))?; + + // Update value and pin so this survives future rebuilds. + facet.value = new_value.clone(); + facet.user_state = UserState::Pinned; + + cache + .upsert(&facet) + .map_err(|e| format!("upsert failed: {e:#}"))?; + + let updated = cache + .get(&fk) + .map_err(|e| format!("re-read failed: {e:#}"))? + .ok_or_else(|| "facet disappeared after upsert".to_string())?; + + let log = vec![format!( + "learning.update_facet: key={fk} new_value={new_value} user_state=pinned" + )]; + let payload = serde_json::json!({ "facet": facet_to_json(&updated) }); + RpcOutcome::new(payload, log).into_cli_compatible_json() + }) +} + +// ── pin_facet ───────────────────────────────────────────────────────────────── + +fn handle_pin_facet(params: Map) -> ControllerFuture { + Box::pin(async move { + use crate::openhuman::memory::store::profile::UserState; + + let class_str = params + .get("class") + .and_then(Value::as_str) + .ok_or_else(|| "missing required `class`".to_string())? + .to_string(); + let key_suffix = params + .get("key") + .and_then(Value::as_str) + .ok_or_else(|| "missing required `key`".to_string())? + .to_string(); + + let fk = full_key(&class_str, &key_suffix); + tracing::debug!("[learning.pin_facet] key={fk}"); + + let cache = get_cache()?; + let updated = cache + .set_user_state(&fk, UserState::Pinned) + .map_err(|e| format!("set_user_state failed: {e:#}"))?; + + if !updated { + return Err(format!("facet not found: {fk}")); + } + + let facet = cache + .get(&fk) + .map_err(|e| format!("re-read failed: {e:#}"))? + .ok_or_else(|| "facet disappeared after update".to_string())?; + + let log = vec![format!("learning.pin_facet: key={fk} user_state=pinned")]; + let payload = serde_json::json!({ "facet": facet_to_json(&facet) }); + RpcOutcome::new(payload, log).into_cli_compatible_json() + }) +} + +// ── unpin_facet ─────────────────────────────────────────────────────────────── + +fn handle_unpin_facet(params: Map) -> ControllerFuture { + Box::pin(async move { + use crate::openhuman::memory::store::profile::UserState; + + let class_str = params + .get("class") + .and_then(Value::as_str) + .ok_or_else(|| "missing required `class`".to_string())? + .to_string(); + let key_suffix = params + .get("key") + .and_then(Value::as_str) + .ok_or_else(|| "missing required `key`".to_string())? + .to_string(); + + let fk = full_key(&class_str, &key_suffix); + tracing::debug!("[learning.unpin_facet] key={fk}"); + + let cache = get_cache()?; + let updated = cache + .set_user_state(&fk, UserState::Auto) + .map_err(|e| format!("set_user_state failed: {e:#}"))?; + + if !updated { + return Err(format!("facet not found: {fk}")); + } + + let facet = cache + .get(&fk) + .map_err(|e| format!("re-read failed: {e:#}"))? + .ok_or_else(|| "facet disappeared after update".to_string())?; + + let log = vec![format!("learning.unpin_facet: key={fk} user_state=auto")]; + let payload = serde_json::json!({ "facet": facet_to_json(&facet) }); + RpcOutcome::new(payload, log).into_cli_compatible_json() + }) +} + +// ── forget_facet ────────────────────────────────────────────────────────────── + +fn handle_forget_facet(params: Map) -> ControllerFuture { + Box::pin(async move { + use crate::openhuman::memory::store::profile::{FacetState, UserState}; + + let class_str = params + .get("class") + .and_then(Value::as_str) + .ok_or_else(|| "missing required `class`".to_string())? + .to_string(); + let key_suffix = params + .get("key") + .and_then(Value::as_str) + .ok_or_else(|| "missing required `key`".to_string())? + .to_string(); + + let fk = full_key(&class_str, &key_suffix); + tracing::debug!("[learning.forget_facet] key={fk}"); + + let cache = get_cache()?; + + let facet_before = cache.get(&fk).map_err(|e| format!("get failed: {e:#}"))?; + + let facet_json = if let Some(mut f) = facet_before { + // Mark Forgotten + Dropped so it doesn't resurface. + f.user_state = UserState::Forgotten; + f.state = FacetState::Dropped; + cache + .upsert(&f) + .map_err(|e| format!("upsert failed: {e:#}"))?; + let updated = cache + .get(&fk) + .map_err(|e| format!("re-read failed: {e:#}"))? + .unwrap_or(f); + facet_to_json(&updated) + } else { + serde_json::Value::Null + }; + + let log = vec![format!( + "learning.forget_facet: key={fk} state=dropped user_state=forgotten" + )]; + let payload = serde_json::json!({ "facet": facet_json }); + RpcOutcome::new(payload, log).into_cli_compatible_json() + }) +} + +// ── reset_cache ─────────────────────────────────────────────────────────────── + +fn handle_reset_cache(_params: Map) -> ControllerFuture { + Box::pin(async move { + use crate::openhuman::memory::store::profile::UserState; + + tracing::debug!("[learning.reset_cache] called"); + + let cache = get_cache()?; + + let all = cache + .list_all() + .map_err(|e| format!("list_all failed: {e:#}"))?; + + let pinned_preserved = all + .iter() + .filter(|f| f.user_state == UserState::Pinned) + .count(); + + // Delete all non-Pinned rows. + let mut deleted = 0usize; + for f in &all { + if f.user_state != UserState::Pinned { + if cache.delete(&f.key).unwrap_or(false) { + deleted += 1; + } + } + } + + tracing::info!( + "[learning.reset_cache] deleted={deleted} pinned_preserved={pinned_preserved}" + ); + + let log = vec![format!( + "learning.reset_cache: deleted={deleted} pinned_preserved={pinned_preserved}" + )]; + let payload = serde_json::json!({ + "deleted": deleted, + "pinned_preserved": pinned_preserved, + }); + RpcOutcome::new(payload, log).into_cli_compatible_json() + }) +} diff --git a/src/openhuman/learning/stability_detector.rs b/src/openhuman/learning/stability_detector.rs new file mode 100644 index 000000000..da689ba8b --- /dev/null +++ b/src/openhuman/learning/stability_detector.rs @@ -0,0 +1,846 @@ +//! Stability detector — Phase 3 of issue #566. +//! +//! Reads candidates from `candidate::global()`, aggregates them with existing +//! rows from `user_profile_facets`, scores each (class, key) pair by the +//! stability formula, resolves value conflicts, applies per-class budgets, +//! assigns lifecycle states, writes the result back to the table, and emits +//! [`DomainEvent::CacheRebuilt`]. +//! +//! # Stability formula +//! +//! ```text +//! stability(class, key) = base × cue_mult × user_state_mult +//! +//! base = Σ(cue_family.weight() × exp(-Δt / half_life(class)) × ln(1 + evidence_count)) +//! cue_mult = 2.0 if any contributing evidence has CueFamily::Explicit, else 1.0 +//! user_state_mult = ∞ if Pinned, 0 if Forgotten, 1 otherwise +//! ``` +//! +//! # Thresholds +//! +//! | Symbol | Value | Meaning | +//! |--------|-------|---------| +//! | τ_promote | 1.5 | Enter as Active | +//! | τ_provisional | 0.7 | Enter as Provisional | +//! | τ_evict | 0.4 | Keep as Candidate | +//! | < τ_evict | — | Dropped | +//! +//! # Class budgets +//! +//! After state assignment, each class retains at most `class_budget(class)` Active +//! rows by stability. Excess Active rows are demoted to Provisional. A cross-class +//! overflow pool holds up to `BUDGET_OVERFLOW` extra Provisional rows. + +use std::collections::HashMap; +use std::sync::Arc; + +use crate::core::event_bus; +use crate::core::event_bus::DomainEvent; +use crate::openhuman::learning::cache::FacetCache; +use crate::openhuman::learning::candidate::{self, CueFamily, FacetClass, LearningCandidate}; +use crate::openhuman::memory::store::profile::{FacetState, FacetType, ProfileFacet, UserState}; + +// ── Thresholds ──────────────────────────────────────────────────────────────── + +/// Stability threshold to enter / stay as Active. +pub const TAU_PROMOTE: f64 = 1.5; +/// Stability threshold to enter / stay as Provisional. +pub const TAU_PROVISIONAL: f64 = 0.7; +/// Stability threshold to retain as Candidate (below → Dropped). +pub const TAU_EVICT: f64 = 0.4; + +// ── Half-lives (seconds) ────────────────────────────────────────────────────── + +pub const HALF_LIFE_IDENTITY: f64 = 90.0 * 86400.0; +pub const HALF_LIFE_VETO: f64 = 60.0 * 86400.0; +pub const HALF_LIFE_TOOLING: f64 = 30.0 * 86400.0; +pub const HALF_LIFE_GOAL: f64 = 30.0 * 86400.0; +pub const HALF_LIFE_STYLE: f64 = 14.0 * 86400.0; +pub const HALF_LIFE_CHANNEL: f64 = 7.0 * 86400.0; + +/// Half-life in seconds for the given class. +pub fn half_life(class: FacetClass) -> f64 { + match class { + FacetClass::Identity => HALF_LIFE_IDENTITY, + FacetClass::Veto => HALF_LIFE_VETO, + FacetClass::Tooling => HALF_LIFE_TOOLING, + FacetClass::Goal => HALF_LIFE_GOAL, + FacetClass::Style => HALF_LIFE_STYLE, + FacetClass::Channel => HALF_LIFE_CHANNEL, + } +} + +// ── Class budgets ───────────────────────────────────────────────────────────── + +pub const BUDGET_STYLE: usize = 4; +pub const BUDGET_IDENTITY: usize = 4; +pub const BUDGET_TOOLING: usize = 5; +pub const BUDGET_VETO: usize = 3; +pub const BUDGET_GOAL: usize = 3; +pub const BUDGET_CHANNEL: usize = 1; +/// Cross-class overflow pool for Provisional rows that didn't make a class budget. +pub const BUDGET_OVERFLOW: usize = 5; + +/// Per-class top-N budget for Active rows. +pub fn class_budget(class: FacetClass) -> usize { + match class { + FacetClass::Style => BUDGET_STYLE, + FacetClass::Identity => BUDGET_IDENTITY, + FacetClass::Tooling => BUDGET_TOOLING, + FacetClass::Veto => BUDGET_VETO, + FacetClass::Goal => BUDGET_GOAL, + FacetClass::Channel => BUDGET_CHANNEL, + } +} + +// ── Stability formula ───────────────────────────────────────────────────────── + +/// Compute the stability score for a `(class, key)` aggregate. +/// +/// # Arguments +/// +/// * `cue_family` — dominant cue family of the evidence set +/// * `evidence_count` — total pieces of evidence contributing +/// * `last_reinforced_at` — Unix seconds of the most recent evidence +/// * `now` — current Unix seconds +/// * `class` — facet class (determines half-life) +/// * `has_explicit_evidence` — whether any evidence has `CueFamily::Explicit` +/// * `user_state` — user override (Pinned → ∞, Forgotten → 0) +pub fn stability( + cue_family: CueFamily, + evidence_count: u32, + last_reinforced_at: f64, + now: f64, + class: FacetClass, + has_explicit_evidence: bool, + user_state: UserState, +) -> f64 { + if matches!(user_state, UserState::Pinned) { + return f64::INFINITY; + } + if matches!(user_state, UserState::Forgotten) { + return 0.0; + } + + let dt = (now - last_reinforced_at).max(0.0); + let recency = (-dt / half_life(class)).exp(); + let base = cue_family.weight() * recency * (1.0 + evidence_count as f64).ln(); + let cue_mult = if has_explicit_evidence { 2.0 } else { 1.0 }; + base * cue_mult +} + +// ── Rebuild outcome ─────────────────────────────────────────────────────────── + +/// Summary of a single rebuild cycle. +#[derive(Debug, Clone)] +pub struct RebuildOutcome { + /// Facet rows newly created in this cycle (key not previously in the table). + pub added: usize, + /// Facet rows that were demoted to Dropped or deleted. + pub evicted: usize, + /// Facet rows carried over from the previous cycle without changes. + pub kept: usize, + /// Total rows in the cache after the rebuild. + pub total_size: usize, +} + +// ── StabilityDetector ───────────────────────────────────────────────────────── + +/// The stability detector. +/// +/// Owns a [`FacetCache`] and a reference to the global [`candidate::Buffer`]. +/// Call [`StabilityDetector::rebuild`] to run one full cycle. +pub struct StabilityDetector { + pub(crate) cache: FacetCache, + pub(crate) buffer: &'static candidate::Buffer, +} + +impl StabilityDetector { + /// Create a new detector backed by the given cache. + /// + /// Uses [`candidate::global()`] as the source buffer. + pub fn new(cache: FacetCache) -> Self { + Self { + cache, + buffer: candidate::global(), + } + } + + /// Run one full rebuild cycle. + /// + /// 1. Drain the global candidate buffer. + /// 2. Load all existing facets from the cache. + /// 3. For each `(class, key)`, aggregate evidence + existing stability. + /// 4. Choose the winning value via `argmax(stability)`. + /// 5. Assign lifecycle state based on thresholds. + /// 6. Apply per-class budgets (demote excess Active → Provisional). + /// 7. Persist changes and delete Dropped rows. + /// 8. Emit `DomainEvent::CacheRebuilt`. + pub fn rebuild(&self, now: f64) -> anyhow::Result { + tracing::debug!("[learning::stability] rebuild starting at t={now:.0}"); + + // Step 1 — drain buffer. + let candidates = self.buffer.drain(); + tracing::debug!( + "[learning::stability] drained {} candidates from buffer", + candidates.len() + ); + + // Step 2 — load existing facets. + let existing_facets = self.cache.list_all()?; + let existing_by_key: HashMap = existing_facets + .into_iter() + .map(|f| (f.key.clone(), f)) + .collect(); + + // Step 3 — group candidates by (class, key). + // For candidates whose key has no class prefix, we skip them (they're legacy rows). + let mut groups: HashMap<(FacetClass, String), Vec> = HashMap::new(); + + for cand in candidates { + let full_key = format!("{}/{}", class_prefix(cand.class), cand.key); + groups.entry((cand.class, full_key)).or_default().push(cand); + } + + // Also process existing facets that have a known class prefix (so they decay). + for key in existing_by_key.keys() { + if let Some(class) = crate::openhuman::learning::cache::class_from_key(key) { + groups.entry((class, key.to_string())).or_default(); // ensure the group exists even if no new candidates + } + } + + // Step 4-6: compute new state for each (class, key). + // We accumulate the final facets per class so we can apply budgets. + let mut by_class: HashMap> = HashMap::new(); + + for ((class, full_key), cands) in &groups { + let existing = existing_by_key.get(full_key); + + // Respect Forgotten user_state: block re-promotion. + let user_state = existing.map(|f| f.user_state).unwrap_or(UserState::Auto); + + // Step 4: for each distinct value, compute a candidate score. + let winning_value = select_winning_value(cands, existing, now, *class); + + // Step 5: compute stability of the winning (class, key) aggregate. + let (agg_score, has_explicit) = aggregate_stability(cands, existing, now, *class); + let final_stability = stability( + dominant_cue(cands, existing), + total_evidence_count(cands, existing), + most_recent_reinforcement(cands, existing, now), + now, + *class, + has_explicit, + user_state, + ); + + tracing::trace!( + "[learning::stability] {full_key}: value={:?} agg={agg_score:.3} final={final_stability:.3}", + winning_value.as_deref().unwrap_or("(none)"), + ); + + // Step 7: state assignment. + let state = state_from_stability(final_stability, user_state); + + let value = match winning_value.or_else(|| existing.map(|f| f.value.clone())) { + Some(v) => v, + None => { + // No candidates and no existing row — shouldn't happen but skip. + continue; + } + }; + + let existing_count = existing.map(|f| f.evidence_count).unwrap_or(0); + let new_evidence_count = (existing_count + cands.len() as i32).max(1); + + let facet_id = existing + .map(|f| f.facet_id.clone()) + .unwrap_or_else(|| format!("learn-{}", full_key.replace('/', "-"))); + + let first_seen = existing.map(|f| f.first_seen_at).unwrap_or(now); + + // Collect evidence refs from new candidates. + let new_refs: Vec = + cands.iter().map(|c| c.evidence.clone()).collect(); + + let mut all_refs = existing + .map(|f| f.evidence_refs.clone()) + .unwrap_or_default(); + all_refs.extend(new_refs); + // Deduplicate refs by serialised form (cheap for small vecs). + all_refs + .dedup_by(|a, b| serde_json::to_string(a).ok() == serde_json::to_string(b).ok()); + + // Build cue-families counts from this cycle's candidates. + let mut cue_counts: HashMap = HashMap::new(); + for c in cands { + *cue_counts + .entry(format!("{:?}", c.cue_family).to_lowercase()) + .or_insert(0) += 1; + } + // Merge with existing cue counts if present. + if let Some(existing_cues) = existing.and_then(|f| f.cue_families.as_ref()) { + for (k, v) in existing_cues { + *cue_counts.entry(k.clone()).or_insert(0) += v; + } + } + + let computed = ComputedFacet { + is_new: existing.is_none(), + facet: ProfileFacet { + facet_id, + facet_type: FacetType::Preference, + key: full_key.clone(), + value, + confidence: (agg_score).min(1.0).max(0.0), + evidence_count: new_evidence_count, + source_segment_ids: existing.and_then(|f| f.source_segment_ids.clone()), + first_seen_at: first_seen, + last_seen_at: now, + state, + stability: final_stability, + user_state, + evidence_refs: all_refs, + // Class derived from the key prefix (always set for learning rows). + class: Some(class_prefix(*class).to_string()), + cue_families: if cue_counts.is_empty() { + None + } else { + Some(cue_counts) + }, + }, + }; + + by_class.entry(*class).or_default().push(computed); + } + + // Step 6 (budget): per-class enforce top-N Active. + let mut overflow_provisional: Vec = Vec::new(); + let mut all_final: Vec = Vec::new(); + + for (class, mut facets) in by_class { + // Sort Active rows by stability desc. + facets.sort_by(|a, b| { + b.facet + .stability + .partial_cmp(&a.facet.stability) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + let budget = class_budget(class); + let mut active_count = 0usize; + + for mut cf in facets { + if cf.facet.state == FacetState::Active { + if active_count < budget { + active_count += 1; + } else { + // Demote excess Active → Provisional and send to overflow pool. + cf.facet.state = FacetState::Provisional; + overflow_provisional.push(cf); + continue; + } + } + all_final.push(cf); + } + } + + // Apply overflow budget: keep top BUDGET_OVERFLOW Provisional rows. + overflow_provisional.sort_by(|a, b| { + b.facet + .stability + .partial_cmp(&a.facet.stability) + .unwrap_or(std::cmp::Ordering::Equal) + }); + for cf in overflow_provisional.into_iter().take(BUDGET_OVERFLOW) { + all_final.push(cf); + } + + // Step 7 — persist and compute outcome. + let mut added = 0usize; + let mut kept = 0usize; + let mut evicted = 0usize; + + for cf in &all_final { + if cf.facet.state == FacetState::Dropped { + evicted += 1; + } else if cf.is_new { + added += 1; + } else { + kept += 1; + } + self.cache.upsert(&cf.facet)?; + } + + // (Existing keys not in the rebuild output are legacy/non-class rows — skip.) + + // Clean up Dropped rows from the table. + let cleaned = self.cache.drop_below_threshold(TAU_EVICT)?; + if cleaned > 0 { + tracing::debug!( + "[learning::stability] cleaned {cleaned} rows below threshold from table" + ); + } + + let active_rows = self.cache.list_active()?; + let total_size = active_rows.len(); + + tracing::info!( + "[learning::stability] rebuild added={added} evicted={evicted} kept={kept} total={total_size}" + ); + + // Step 8 — publish CacheRebuilt event. + event_bus::publish_global(DomainEvent::CacheRebuilt { + added, + evicted, + kept, + total_size, + rebuilt_at: now, + }); + + Ok(RebuildOutcome { + added, + evicted, + kept, + total_size, + }) + } +} + +// ── Rebuild internals ───────────────────────────────────────────────────────── + +struct ComputedFacet { + is_new: bool, + facet: ProfileFacet, +} + +/// Choose the winning value for a `(class, key)` group via `argmax(stability)`. +/// +/// Returns the value with the highest combined evidence weight. Falls back to the +/// existing row's value if no candidates are present. +fn select_winning_value( + cands: &[LearningCandidate], + existing: Option<&ProfileFacet>, + now: f64, + class: FacetClass, +) -> Option { + if cands.is_empty() { + return existing.map(|f| f.value.clone()); + } + + // Score each distinct value. + let mut value_scores: HashMap<&str, f64> = HashMap::new(); + for c in cands { + let dt = (now - c.observed_at).max(0.0); + let recency = (-dt / half_life(class)).exp(); + let score = c.cue_family.weight() * recency * c.initial_confidence; + *value_scores.entry(c.value.as_str()).or_default() += score; + } + + // If existing row matches a candidate value, add its weight too. + if let Some(existing) = existing { + let dt = (now - existing.last_seen_at).max(0.0); + let recency = (-dt / half_life(class)).exp(); + let existing_score = recency * existing.confidence; + *value_scores.entry(existing.value.as_str()).or_default() += existing_score; + } + + value_scores + .into_iter() + .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)) + .map(|(v, _)| v.to_string()) +} + +/// Aggregate stability contribution from all candidates (not per-value). +/// Returns (aggregate_score, has_explicit_evidence). +fn aggregate_stability( + cands: &[LearningCandidate], + existing: Option<&ProfileFacet>, + now: f64, + class: FacetClass, +) -> (f64, bool) { + let mut score = 0.0f64; + let mut has_explicit = false; + + for c in cands { + let dt = (now - c.observed_at).max(0.0); + let recency = (-dt / half_life(class)).exp(); + score += c.cue_family.weight() * recency; + if matches!(c.cue_family, CueFamily::Explicit) { + has_explicit = true; + } + } + + if let Some(existing) = existing { + let dt = (now - existing.last_seen_at).max(0.0); + let recency = (-dt / half_life(class)).exp(); + score += existing.confidence * recency; + } + + (score, has_explicit) +} + +/// Determine the dominant cue family (highest weight). +fn dominant_cue(cands: &[LearningCandidate], _existing: Option<&ProfileFacet>) -> CueFamily { + cands + .iter() + .max_by(|a, b| { + a.cue_family + .weight() + .partial_cmp(&b.cue_family.weight()) + .unwrap_or(std::cmp::Ordering::Equal) + }) + .map(|c| c.cue_family) + .unwrap_or(CueFamily::Behavioral) +} + +/// Total evidence count from candidates + existing row. +fn total_evidence_count(cands: &[LearningCandidate], existing: Option<&ProfileFacet>) -> u32 { + let from_existing = existing.map(|f| f.evidence_count as u32).unwrap_or(0); + from_existing + cands.len() as u32 +} + +/// The most recent observation timestamp across candidates and the existing row. +fn most_recent_reinforcement( + cands: &[LearningCandidate], + existing: Option<&ProfileFacet>, + now: f64, +) -> f64 { + let newest_cand = cands + .iter() + .map(|c| c.observed_at) + .fold(f64::NEG_INFINITY, f64::max); + let existing_ts = existing + .map(|f| f.last_seen_at) + .unwrap_or(f64::NEG_INFINITY); + newest_cand + .max(existing_ts) + .max(now - half_life(FacetClass::Style)) +} + +/// Map a stability score + user_state to a lifecycle state. +fn state_from_stability(score: f64, user_state: UserState) -> FacetState { + // Pinned → always Active; Forgotten → always Dropped. + if matches!(user_state, UserState::Pinned) { + return FacetState::Active; + } + if matches!(user_state, UserState::Forgotten) { + return FacetState::Dropped; + } + + if score.is_infinite() { + FacetState::Active + } else if score >= TAU_PROMOTE { + FacetState::Active + } else if score >= TAU_PROVISIONAL { + FacetState::Provisional + } else if score >= TAU_EVICT { + FacetState::Candidate + } else { + FacetState::Dropped + } +} + +/// Canonical key prefix string for a class (used when grouping candidates). +fn class_prefix(class: FacetClass) -> &'static str { + crate::openhuman::learning::cache::class_prefix(class) +} + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::learning::cache::FacetCache; + use crate::openhuman::learning::candidate::{ + Buffer, EvidenceRef, FacetClass, LearningCandidate, + }; + use crate::openhuman::memory::store::profile::PROFILE_INIT_SQL; + use parking_lot::Mutex; + use rusqlite::Connection; + use std::sync::Arc; + + fn make_detector() -> StabilityDetector { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(PROFILE_INIT_SQL).unwrap(); + let cache = FacetCache::new(Arc::new(Mutex::new(conn))); + // Use a private buffer so tests don't interfere with the global singleton. + let buffer: &'static Buffer = Box::leak(Box::new(Buffer::new(256))); + StabilityDetector { cache, buffer } + } + + fn make_candidate( + class: FacetClass, + key: &str, + value: &str, + cue: CueFamily, + observed_at: f64, + ) -> LearningCandidate { + LearningCandidate { + class, + key: key.into(), + value: value.into(), + cue_family: cue, + evidence: EvidenceRef::Episodic { episodic_id: 1 }, + initial_confidence: 0.8, + observed_at, + } + } + + // ── stability formula ──────────────────────────────────────────────────── + + #[test] + fn stability_pinned_returns_infinity() { + let s = stability( + CueFamily::Behavioral, + 5, + 0.0, + 1000.0, + FacetClass::Style, + false, + UserState::Pinned, + ); + assert!(s.is_infinite() && s > 0.0); + } + + #[test] + fn stability_forgotten_returns_zero() { + let s = stability( + CueFamily::Explicit, + 100, + 0.0, + 1000.0, + FacetClass::Style, + true, + UserState::Forgotten, + ); + assert_eq!(s, 0.0); + } + + #[test] + fn stability_explicit_doubles_score() { + let base = stability( + CueFamily::Explicit, + 3, + 1_000_000.0, + 1_000_001.0, + FacetClass::Style, + false, // no_explicit + UserState::Auto, + ); + let with_explicit = stability( + CueFamily::Explicit, + 3, + 1_000_000.0, + 1_000_001.0, + FacetClass::Style, + true, // has_explicit + UserState::Auto, + ); + assert!( + (with_explicit - 2.0 * base).abs() < 1e-9, + "explicit multiplier must be exactly 2x: base={base:.6} explicit={with_explicit:.6}" + ); + } + + #[test] + fn stability_decays_over_time() { + let now = 1_000_000.0_f64; + let recent = stability( + CueFamily::Behavioral, + 5, + now - 100.0, // observed 100 s ago + now, + FacetClass::Style, + false, + UserState::Auto, + ); + let old = stability( + CueFamily::Behavioral, + 5, + now - HALF_LIFE_STYLE, // observed one half-life ago + now, + FacetClass::Style, + false, + UserState::Auto, + ); + assert!( + recent > old, + "recent evidence should produce higher stability: recent={recent:.4} old={old:.4}" + ); + // At exactly one half-life, recency = exp(-1) ≈ 0.368. + assert!( + old / recent < 0.4, + "decay over one half-life should be substantial: ratio={}", + old / recent + ); + } + + // ── rebuild ────────────────────────────────────────────────────────────── + + #[test] + fn rebuild_empty_buffer_no_candidates_is_noop() { + let detector = make_detector(); + let now = 1_000_000.0; + // No candidates, no existing rows → rebuild is a no-op. + let outcome = detector.rebuild(now).unwrap(); + assert_eq!(outcome.added, 0); + assert_eq!(outcome.evicted, 0); + assert_eq!(outcome.kept, 0); + assert_eq!(outcome.total_size, 0); + } + + #[test] + fn rebuild_strong_candidate_becomes_active() { + let detector = make_detector(); + let now = 1_000_000.0; + + // Push enough explicit evidence to clear τ_promote. + for i in 0..5 { + detector.buffer.push(make_candidate( + FacetClass::Style, + "verbosity", + "terse", + CueFamily::Explicit, + now - i as f64 * 10.0, + )); + } + + let outcome = detector.rebuild(now).unwrap(); + assert_eq!(outcome.added, 1); + + let actives = detector.cache.list_active().unwrap(); + assert_eq!(actives.len(), 1); + assert_eq!(actives[0].key, "style/verbosity"); + assert_eq!(actives[0].value, "terse"); + assert_eq!(actives[0].state, FacetState::Active); + } + + #[test] + fn rebuild_conflict_resolution_picks_stronger_value() { + let detector = make_detector(); + let now = 1_000_000.0; + + // 3 explicit candidates for "terse", 1 behavioral for "verbose". + for _ in 0..3 { + detector.buffer.push(make_candidate( + FacetClass::Style, + "verbosity", + "terse", + CueFamily::Explicit, + now - 10.0, + )); + } + detector.buffer.push(make_candidate( + FacetClass::Style, + "verbosity", + "verbose", + CueFamily::Behavioral, + now - 5.0, + )); + + detector.rebuild(now).unwrap(); + let actives = detector.cache.list_active().unwrap(); + assert!(!actives.is_empty(), "should have at least one active row"); + let verbosity = actives.iter().find(|f| f.key == "style/verbosity").unwrap(); + assert_eq!( + verbosity.value, "terse", + "terse had stronger evidence and should win" + ); + } + + #[test] + fn rebuild_class_budget_respected() { + let detector = make_detector(); + let now = 1_000_000.0; + + // Push 6 different style keys — budget is BUDGET_STYLE = 4. + for i in 0..6 { + let key = format!("style_key_{i}"); + // Push several candidates per key so they clear τ_promote. + for j in 0..5 { + detector.buffer.push(LearningCandidate { + class: FacetClass::Style, + key: key.clone(), + value: "v".into(), + cue_family: CueFamily::Explicit, + evidence: EvidenceRef::Episodic { + episodic_id: i * 10 + j, + }, + initial_confidence: 0.9, + observed_at: now - j as f64, + }); + } + } + + detector.rebuild(now).unwrap(); + + let by_class = detector.cache.list_by_class(FacetClass::Style).unwrap(); + assert!( + by_class.len() <= BUDGET_STYLE, + "style class should have at most {BUDGET_STYLE} active rows, got {}", + by_class.len() + ); + } + + #[test] + fn rebuild_pinned_facet_stays_active_regardless_of_stability() { + let detector = make_detector(); + let now = 1_000_000.0; + + // Manually insert a Pinned row. + use crate::openhuman::memory::store::profile::{FacetState, FacetType, UserState}; + let pinned = ProfileFacet { + facet_id: "f-pinned".into(), + facet_type: FacetType::Preference, + key: "style/format".into(), + value: "markdown".into(), + confidence: 0.9, + evidence_count: 1, + source_segment_ids: None, + first_seen_at: 1000.0, + last_seen_at: 1000.0, // very old — would normally decay + state: FacetState::Active, + stability: 0.0, + user_state: UserState::Pinned, + evidence_refs: vec![], + class: Some("style".into()), + cue_families: None, + }; + detector.cache.upsert(&pinned).unwrap(); + + // No new candidates for this key → only decay applies. + detector.rebuild(now).unwrap(); + + let f = detector + .cache + .get("style/format") + .unwrap() + .expect("pinned row must survive"); + assert_eq!(f.state, FacetState::Active); + } + + // ── half_life ──────────────────────────────────────────────────────────── + + #[test] + fn half_life_ordering_matches_spec() { + // Identity decays slowest; Channel decays fastest. + assert!(half_life(FacetClass::Identity) > half_life(FacetClass::Veto)); + assert!(half_life(FacetClass::Veto) > half_life(FacetClass::Tooling)); + assert!(half_life(FacetClass::Tooling) >= half_life(FacetClass::Goal)); + assert!(half_life(FacetClass::Goal) > half_life(FacetClass::Style)); + assert!(half_life(FacetClass::Style) > half_life(FacetClass::Channel)); + } + + // ── class_budget ──────────────────────────────────────────────────────── + + #[test] + fn class_budget_values_match_spec() { + assert_eq!(class_budget(FacetClass::Style), 4); + assert_eq!(class_budget(FacetClass::Identity), 4); + assert_eq!(class_budget(FacetClass::Tooling), 5); + assert_eq!(class_budget(FacetClass::Veto), 3); + assert_eq!(class_budget(FacetClass::Goal), 3); + assert_eq!(class_budget(FacetClass::Channel), 1); + } +} diff --git a/src/openhuman/memory/store/unified/init.rs b/src/openhuman/memory/store/unified/init.rs index d25bf9bec..c42b86f8a 100644 --- a/src/openhuman/memory/store/unified/init.rs +++ b/src/openhuman/memory/store/unified/init.rs @@ -131,6 +131,23 @@ impl UnifiedMemory { // User profile accumulation table. conn.execute_batch(super::profile::PROFILE_INIT_SQL)?; + // Phase 3 (#566): idempotently add new columns to existing databases. + // New installs already have these columns via PROFILE_INIT_SQL above. + // Existing DBs need the migration to add state/stability/user_state/evidence_refs_json. + // We run the ALTER TABLE statements directly on `conn` before wrapping it in Arc. + // SQL arrays are defined as constants in profile.rs to avoid duplication. + { + use super::profile::{PHASE3_COLUMNS_SQL, PHASE3_INDEXES_SQL}; + for sql in PHASE3_COLUMNS_SQL.iter().chain(PHASE3_INDEXES_SQL.iter()) { + match conn.execute(sql, []) { + Ok(_) => tracing::debug!("[profile:init] applied: {sql}"), + Err(e) => { + tracing::trace!("[profile:init] skipped (probably already exists): {e}") + } + } + } + } + // Idempotent legacy-namespace migration. // // Older writes via MemoryStoreTool packed the intended namespace into diff --git a/src/openhuman/memory/store/unified/profile.rs b/src/openhuman/memory/store/unified/profile.rs index bb3dbae4b..bd3984345 100644 --- a/src/openhuman/memory/store/unified/profile.rs +++ b/src/openhuman/memory/store/unified/profile.rs @@ -6,12 +6,21 @@ //! and evidence counts. On conflict (same facet_type + key), evidence_count //! is incremented; the value is only overwritten if the new confidence is //! higher. +//! +//! ## Phase 3 schema additions (#566) +//! +//! Added `state`, `stability`, `user_state`, and `evidence_refs_json` columns. +//! Existing databases are migrated idempotently via `ALTER TABLE … ADD COLUMN` +//! wrapped in `migrate_profile_schema()`. use parking_lot::Mutex; -use rusqlite::{params, Connection}; +use rusqlite::{params, Connection, OptionalExtension}; use serde::{Deserialize, Serialize}; +use std::collections::HashMap; use std::sync::Arc; +use crate::openhuman::learning::candidate::EvidenceRef; + /// SQL to create the user_profile table. Called during UnifiedMemory init. pub const PROFILE_INIT_SQL: &str = r#" CREATE TABLE IF NOT EXISTS user_profile ( @@ -24,13 +33,155 @@ CREATE TABLE IF NOT EXISTS user_profile ( source_segment_ids TEXT, first_seen_at REAL NOT NULL, last_seen_at REAL NOT NULL, + state TEXT NOT NULL DEFAULT 'active', + stability REAL NOT NULL DEFAULT 0.0, + user_state TEXT NOT NULL DEFAULT 'auto', + evidence_refs_json TEXT, + class TEXT, + cue_families_json TEXT, UNIQUE(facet_type, key) ); CREATE INDEX IF NOT EXISTS idx_profile_type ON user_profile(facet_type); + +CREATE INDEX IF NOT EXISTS idx_profile_state + ON user_profile(state); + +CREATE INDEX IF NOT EXISTS idx_profile_class + ON user_profile(class); "#; +/// Phase 3 ALTER TABLE statements for adding new columns to existing databases. +/// +/// Used by both `migrate_profile_schema` (post-Arc-wrap path) and +/// `init.rs` (pre-Arc-wrap path) to avoid duplicating the SQL. +pub const PHASE3_COLUMNS_SQL: &[&str] = &[ + "ALTER TABLE user_profile ADD COLUMN state TEXT NOT NULL DEFAULT 'active'", + "ALTER TABLE user_profile ADD COLUMN stability REAL NOT NULL DEFAULT 0.0", + "ALTER TABLE user_profile ADD COLUMN user_state TEXT NOT NULL DEFAULT 'auto'", + "ALTER TABLE user_profile ADD COLUMN evidence_refs_json TEXT", + "ALTER TABLE user_profile ADD COLUMN class TEXT", + "ALTER TABLE user_profile ADD COLUMN cue_families_json TEXT", +]; + +/// Phase 3 index creation statements. Idempotent (IF NOT EXISTS). +/// +/// Shared between `migrate_profile_schema` and `init.rs`. +pub const PHASE3_INDEXES_SQL: &[&str] = &[ + "CREATE INDEX IF NOT EXISTS idx_profile_state ON user_profile(state)", + "CREATE INDEX IF NOT EXISTS idx_profile_class ON user_profile(class)", +]; + +/// Idempotent schema migration for existing databases. +/// +/// New installs get the full schema from `PROFILE_INIT_SQL`. Existing databases +/// may be missing the Phase 3 columns. This function adds each new column if it +/// doesn't exist, ignoring the "duplicate column name" error that SQLite returns +/// when the column is already present. +pub fn migrate_profile_schema(conn: &Arc>) { + let conn = conn.lock(); + for sql in PHASE3_COLUMNS_SQL { + match conn.execute(sql, []) { + Ok(_) => { + tracing::debug!("[profile] schema migration applied: {sql}"); + } + Err(rusqlite::Error::SqliteFailure(err, _)) + if err.extended_code == rusqlite::ffi::SQLITE_ERROR => + { + // "duplicate column name" is not a named SQLite error code; it comes + // back as a generic SQLITE_ERROR with the text "duplicate column name". + // We tolerate any SQLITE_ERROR here because that's the only class of + // error this ALTER TABLE can produce when the column already exists. + tracing::trace!("[profile] column already present (ok): {sql}"); + } + Err(e) => { + tracing::warn!("[profile] schema migration failed (non-fatal): {sql}: {e}"); + } + } + } + + // Ensure the new indexes exist (idempotent — IF NOT EXISTS). + for sql in PHASE3_INDEXES_SQL { + if let Err(e) = conn.execute(sql, []) { + tracing::warn!("[profile] index creation failed (non-fatal): {sql}: {e}"); + } + } +} + +// ── FacetState ─────────────────────────────────────────────────────────────── + +/// Lifecycle state of a profile facet in the stability detector. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum FacetState { + /// Facet has cleared τ_promote and is included in the ambient cache. + #[default] + Active, + /// Facet is between τ_provisional and τ_promote — included at lower weight. + Provisional, + /// Facet is between τ_evict and τ_provisional — held as a candidate. + Candidate, + /// Facet fell below τ_evict — will be removed on next rebuild. + Dropped, +} + +impl FacetState { + pub fn as_str(self) -> &'static str { + match self { + Self::Active => "active", + Self::Provisional => "provisional", + Self::Candidate => "candidate", + Self::Dropped => "dropped", + } + } + + pub fn parse_or_default(s: &str) -> Self { + match s { + "provisional" => Self::Provisional, + "candidate" => Self::Candidate, + "dropped" => Self::Dropped, + _ => Self::Active, + } + } +} + +// ── UserState ──────────────────────────────────────────────────────────────── + +/// User-controlled override for a profile facet. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum UserState { + /// No user override — stability detector manages the lifecycle. + #[default] + Auto, + /// User has explicitly pinned this facet; it stays Active regardless of score. + Pinned, + /// User has explicitly forgotten this facet; it stays Dropped and cannot be + /// re-promoted by new evidence. + Forgotten, +} + +impl UserState { + pub fn as_str(self) -> &'static str { + match self { + Self::Auto => "auto", + Self::Pinned => "pinned", + Self::Forgotten => "forgotten", + } + } + + pub fn parse_or_default(s: &str) -> Self { + match s { + "pinned" => Self::Pinned, + "forgotten" => Self::Forgotten, + _ => Self::Auto, + } + } +} + +// ── FacetType ──────────────────────────────────────────────────────────────── + /// Profile facet types. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -67,7 +218,9 @@ impl FacetType { } } -/// A single profile facet. +// ── ProfileFacet ───────────────────────────────────────────────────────────── + +/// A single profile facet — extended with Phase 3 state + stability fields. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ProfileFacet { pub facet_id: String, @@ -79,13 +232,39 @@ pub struct ProfileFacet { pub source_segment_ids: Option, pub first_seen_at: f64, pub last_seen_at: f64, + // ── Phase 3 additions ── + /// Lifecycle state assigned by the stability detector. + pub state: FacetState, + /// Computed stability score from the last rebuild cycle. + pub stability: f64, + /// User-controlled override. + pub user_state: UserState, + /// Provenance references deserialized from `evidence_refs_json`. + pub evidence_refs: Vec, + /// Facet class (style / identity / tooling / veto / goal / channel). + /// + /// Derived from the key prefix (e.g. `"style/verbosity"` → `"style"`) for + /// learning-path rows. `None` for legacy provider rows whose key prefix + /// doesn't match a known class. + pub class: Option, + /// Per-cue-family evidence counts serialized as JSON. + /// + /// Shape: `{"explicit": N, "structural": N, "behavioral": N, "recurrence": N}`. + /// `None` until the stability detector writes the first rebuild. + pub cue_families: Option>, } -/// Upsert a profile facet. On conflict (same facet_type + key): +// ── Write helpers ───────────────────────────────────────────────────────────── + +/// Upsert a profile facet (legacy / provider path). On conflict (same facet_type + key): /// - Increments evidence_count /// - Updates last_seen_at /// - Appends segment_id to source_segment_ids /// - Only overwrites value if new confidence > existing confidence +/// +/// The new Phase 3 columns (`state`, `stability`, `user_state`, +/// `evidence_refs_json`) default to `active`, `0.0`, `auto`, and `NULL` +/// respectively, so existing callers need no changes. #[allow(clippy::too_many_arguments)] pub fn profile_upsert( conn: &Arc>, @@ -110,18 +289,7 @@ pub fn profile_upsert( .ok(); if let Some((existing_id, existing_confidence, existing_count, existing_segments)) = existing { - let new_segments = match (existing_segments, segment_id) { - (Some(existing), Some(sid)) => { - if existing.contains(sid) { - existing - } else { - format!("{existing},{sid}") - } - } - (Some(existing), None) => existing, - (None, Some(sid)) => sid.to_string(), - (None, None) => String::new(), - }; + let new_segments = merge_segments(existing_segments, segment_id); if confidence >= existing_confidence { // Higher or equal confidence: overwrite value + update metadata. @@ -155,13 +323,17 @@ pub fn profile_upsert( existing_count + 1 ); } else { - // Insert new facet. + // Insert new facet. Derive class from the key prefix for learning rows. let segments = segment_id.unwrap_or("").to_string(); + let class = infer_class_from_key(key, facet_type); conn.execute( "INSERT INTO user_profile (facet_id, facet_type, key, value, confidence, evidence_count, - source_segment_ids, first_seen_at, last_seen_at) - VALUES (?1, ?2, ?3, ?4, ?5, 1, ?6, ?7, ?7)", + source_segment_ids, first_seen_at, last_seen_at, + state, stability, user_state, evidence_refs_json, + class, cue_families_json) + VALUES (?1, ?2, ?3, ?4, ?5, 1, ?6, ?7, ?7, 'active', 0.0, 'auto', NULL, + ?8, NULL)", params![ facet_id, facet_type.as_str(), @@ -170,6 +342,7 @@ pub fn profile_upsert( confidence, segments, now, + class, ], )?; tracing::debug!( @@ -183,12 +356,133 @@ pub fn profile_upsert( Ok(()) } +/// Full upsert used by the stability detector rebuild path. +/// +/// Writes all Phase 3 columns explicitly. On conflict (same facet_type + key) +/// the row is replaced in full — the rebuild owns these rows. +pub fn profile_upsert_full( + conn: &Arc>, + facet: &ProfileFacet, +) -> anyhow::Result<()> { + let evidence_refs_json = if facet.evidence_refs.is_empty() { + None + } else { + Some(serde_json::to_string(&facet.evidence_refs)?) + }; + + let cue_families_json = facet + .cue_families + .as_ref() + .filter(|m| !m.is_empty()) + .map(|m| serde_json::to_string(m)) + .transpose()?; + + // Derive class from the facet's own class field or fall back to key prefix. + let class = facet + .class + .clone() + .or_else(|| infer_class_from_key(&facet.key, &facet.facet_type)); + + let conn = conn.lock(); + + // Use INSERT OR REPLACE to atomically update all columns including + // state/stability without reading the row first. Note: on a UNIQUE(facet_type, + // key) conflict, SQLite performs DELETE + INSERT rather than an in-place + // update, which means facet_id will change for conflicting rows. This is + // intentional: the stability detector owns these rows during rebuild and + // provides consistent facet_id values; external references by facet_id are + // not expected. + conn.execute( + "INSERT OR REPLACE INTO user_profile + (facet_id, facet_type, key, value, confidence, evidence_count, + source_segment_ids, first_seen_at, last_seen_at, + state, stability, user_state, evidence_refs_json, + class, cue_families_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, + ?14, ?15)", + params![ + facet.facet_id, + facet.facet_type.as_str(), + facet.key, + facet.value, + facet.confidence, + facet.evidence_count, + facet.source_segment_ids, + facet.first_seen_at, + facet.last_seen_at, + facet.state.as_str(), + facet.stability, + facet.user_state.as_str(), + evidence_refs_json, + class, + cue_families_json, + ], + )?; + + tracing::debug!( + "[profile] full-upsert facet {}:{} = {} (state={}, stability={:.3}, class={:?})", + facet.facet_type.as_str(), + facet.key, + facet.value, + facet.state.as_str(), + facet.stability, + class, + ); + Ok(()) +} + +/// Update the `user_state` column for a facet by key. +/// +/// Returns `Ok(true)` if a row was updated, `Ok(false)` if not found. +pub fn profile_set_user_state( + conn: &Arc>, + key: &str, + user_state: UserState, +) -> anyhow::Result { + let conn = conn.lock(); + let rows = conn.execute( + "UPDATE user_profile SET user_state = ?1 WHERE key = ?2", + params![user_state.as_str(), key], + )?; + Ok(rows > 0) +} + +/// Delete a facet by key. Returns `true` if a row was deleted. +pub fn profile_delete_by_key(conn: &Arc>, key: &str) -> anyhow::Result { + let conn = conn.lock(); + let rows = conn.execute("DELETE FROM user_profile WHERE key = ?1", params![key])?; + Ok(rows > 0) +} + +/// Delete all facets whose stability is below the given threshold. +/// +/// Facets with `user_state = 'pinned'` are never deleted regardless of score. +/// Returns the number of rows deleted. +pub fn profile_delete_below_threshold( + conn: &Arc>, + threshold: f64, +) -> anyhow::Result { + let conn = conn.lock(); + let rows = conn.execute( + "DELETE FROM user_profile + WHERE stability < ?1 + AND user_state != 'pinned' + AND state = 'dropped'", + params![threshold], + )?; + Ok(rows) +} + +// ── Read helpers ────────────────────────────────────────────────────────────── + /// Load all profile facets. pub fn profile_load_all(conn: &Arc>) -> anyhow::Result> { let conn = conn.lock(); let mut stmt = conn.prepare( "SELECT facet_id, facet_type, key, value, confidence, evidence_count, - source_segment_ids, first_seen_at, last_seen_at + source_segment_ids, first_seen_at, last_seen_at, + state, stability, user_state, evidence_refs_json, + class, cue_families_json FROM user_profile ORDER BY facet_type, evidence_count DESC", )?; @@ -198,7 +492,42 @@ pub fn profile_load_all(conn: &Arc>) -> anyhow::Result>) -> anyhow::Result> { + let conn = conn.lock(); + let mut stmt = conn.prepare( + "SELECT facet_id, facet_type, key, value, confidence, evidence_count, + source_segment_ids, first_seen_at, last_seen_at, + state, stability, user_state, evidence_refs_json, + class, cue_families_json + FROM user_profile + WHERE state = 'active' + ORDER BY stability DESC", + )?; + let rows = stmt + .query_map([], row_to_facet)? + .collect::, _>>()?; + Ok(rows) +} + +/// Load all facets regardless of state (used by the rebuild cycle for a full view). +pub fn profile_select_all(conn: &Arc>) -> anyhow::Result> { + let conn = conn.lock(); + let mut stmt = conn.prepare( + "SELECT facet_id, facet_type, key, value, confidence, evidence_count, + source_segment_ids, first_seen_at, last_seen_at, + state, stability, user_state, evidence_refs_json, + class, cue_families_json + FROM user_profile + ORDER BY stability DESC", + )?; + let rows = stmt + .query_map([], row_to_facet)? + .collect::, _>>()?; + Ok(rows) +} + +/// Load profile facets by type (legacy path). pub fn profile_facets_by_type( conn: &Arc>, facet_type: &FacetType, @@ -206,7 +535,9 @@ pub fn profile_facets_by_type( let conn = conn.lock(); let mut stmt = conn.prepare( "SELECT facet_id, facet_type, key, value, confidence, evidence_count, - source_segment_ids, first_seen_at, last_seen_at + source_segment_ids, first_seen_at, last_seen_at, + state, stability, user_state, evidence_refs_json, + class, cue_families_json FROM user_profile WHERE facet_type = ?1 ORDER BY evidence_count DESC", @@ -217,6 +548,51 @@ pub fn profile_facets_by_type( Ok(rows) } +/// Load a single facet by key. Returns `None` if not found. +pub fn profile_get_by_key( + conn: &Arc>, + key: &str, +) -> anyhow::Result> { + let conn = conn.lock(); + conn.query_row( + "SELECT facet_id, facet_type, key, value, confidence, evidence_count, + source_segment_ids, first_seen_at, last_seen_at, + state, stability, user_state, evidence_refs_json, + class, cue_families_json + FROM user_profile WHERE key = ?1", + params![key], + row_to_facet, + ) + .optional() + .map_err(Into::into) +} + +/// Count facets grouped by class prefix (the portion of `key` before the first `/`). +/// +/// For example, `style/verbosity` → class `"style"`. +/// Facets whose key contains no `/` are grouped under `"_other"`. +pub fn profile_count_by_class( + conn: &Arc>, +) -> anyhow::Result> { + let conn = conn.lock(); + let mut stmt = conn.prepare("SELECT key FROM user_profile WHERE state = 'active'")?; + let keys: Vec = stmt + .query_map([], |row| row.get(0))? + .collect::, _>>()?; + + let mut counts: HashMap = HashMap::new(); + for key in keys { + let class = key + .split_once('/') + .map(|(prefix, _)| prefix.to_string()) + .unwrap_or_else(|| "_other".to_string()); + *counts.entry(class).or_insert(0) += 1; + } + Ok(counts) +} + +// ── Rendering ───────────────────────────────────────────────────────────────── + /// Render profile facets as a markdown section for context assembly. pub fn render_profile_context(facets: &[ProfileFacet]) -> String { if facets.is_empty() { @@ -247,6 +623,55 @@ pub fn render_profile_context(facets: &[ProfileFacet]) -> String { parts.join("\n\n") } +// ── Internal helpers ────────────────────────────────────────────────────────── + +/// Infer the class label for a facet row from its key prefix and facet_type. +/// +/// Learning-path rows use a key like `"style/verbosity"` where the prefix +/// directly encodes the class. Legacy provider rows use `"skill:..."` keys +/// and are mapped via `facet_type`. +fn infer_class_from_key(key: &str, facet_type: &FacetType) -> Option { + // Try key prefix first (learning path: "style/verbosity" → "style"). + if let Some((prefix, _)) = key.split_once('/') { + let known = matches!( + prefix, + "style" | "identity" | "tooling" | "veto" | "goal" | "channel" + ); + if known { + return Some(prefix.to_string()); + } + } + // Legacy provider rows: skill:* keys → "tooling". + if key.starts_with("skill:") { + return Some("tooling".to_string()); + } + // Fall back on facet_type. + Some( + match facet_type { + FacetType::Role | FacetType::Personality => "identity", + FacetType::Skill => "tooling", + FacetType::Preference => "style", + FacetType::Context => "identity", + } + .to_string(), + ) +} + +fn merge_segments(existing: Option, new_sid: Option<&str>) -> String { + match (existing, new_sid) { + (Some(existing), Some(sid)) => { + if existing.contains(sid) { + existing + } else { + format!("{existing},{sid}") + } + } + (Some(existing), None) => existing, + (None, Some(sid)) => sid.to_string(), + (None, None) => String::new(), + } +} + fn capitalize(s: &str) -> String { let mut chars = s.chars(); match chars.next() { @@ -257,6 +682,22 @@ fn capitalize(s: &str) -> String { fn row_to_facet(row: &rusqlite::Row<'_>) -> rusqlite::Result { let facet_type_str: String = row.get(1)?; + let state_str: String = row.get(9)?; + let stability: f64 = row.get(10)?; + let user_state_str: String = row.get(11)?; + let evidence_refs_json: Option = row.get(12)?; + let class: Option = row.get(13)?; + let cue_families_json: Option = row.get(14)?; + + let evidence_refs = evidence_refs_json + .as_deref() + .and_then(|json| serde_json::from_str(json).ok()) + .unwrap_or_default(); + + let cue_families = cue_families_json + .as_deref() + .and_then(|json| serde_json::from_str(json).ok()); + Ok(ProfileFacet { facet_id: row.get(0)?, facet_type: FacetType::parse_or_default(&facet_type_str), @@ -267,6 +708,12 @@ fn row_to_facet(row: &rusqlite::Row<'_>) -> rusqlite::Result { source_segment_ids: row.get(6)?, first_seen_at: row.get(7)?, last_seen_at: row.get(8)?, + state: FacetState::parse_or_default(&state_str), + stability, + user_state: UserState::parse_or_default(&user_state_str), + evidence_refs, + class, + cue_families, }) } diff --git a/src/openhuman/memory/store/unified/profile_tests.rs b/src/openhuman/memory/store/unified/profile_tests.rs index 6583f6327..038c911ba 100644 --- a/src/openhuman/memory/store/unified/profile_tests.rs +++ b/src/openhuman/memory/store/unified/profile_tests.rs @@ -2,6 +2,257 @@ use super::*; +// ── Migration test ──────────────────────────────────────────────────────────── + +/// Verify that `migrate_profile_schema` adds Phase 3 columns to a database +/// that was created with the pre-Phase-3 schema (missing state/stability/…). +#[test] +fn migrate_adds_new_columns_to_existing_db() { + // Create the pre-Phase-3 schema manually (only original columns). + let pre_phase3_sql = r#" + CREATE TABLE IF NOT EXISTS user_profile ( + facet_id TEXT PRIMARY KEY, + facet_type TEXT NOT NULL, + key TEXT NOT NULL, + value TEXT NOT NULL, + confidence REAL NOT NULL DEFAULT 0.5, + evidence_count INTEGER NOT NULL DEFAULT 1, + source_segment_ids TEXT, + first_seen_at REAL NOT NULL, + last_seen_at REAL NOT NULL, + UNIQUE(facet_type, key) + ); + "#; + let raw_conn = Connection::open_in_memory().unwrap(); + raw_conn.execute_batch(pre_phase3_sql).unwrap(); + let conn = Arc::new(Mutex::new(raw_conn)); + + // Insert a row using the old schema. + { + let c = conn.lock(); + c.execute( + "INSERT INTO user_profile + (facet_id, facet_type, key, value, confidence, evidence_count, + first_seen_at, last_seen_at) + VALUES ('f-old', 'preference', 'theme', 'dark', 0.8, 1, 1000.0, 1000.0)", + [], + ) + .unwrap(); + } + + // Run the migration — should succeed without panicking. + migrate_profile_schema(&conn); + + // The new columns must be present and readable. + let facets = profile_load_all(&conn).unwrap(); + assert_eq!(facets.len(), 1); + let f = &facets[0]; + assert_eq!(f.key, "theme"); + // Defaults applied by ALTER TABLE … DEFAULT. + assert_eq!(f.state, FacetState::Active); + assert!((f.stability - 0.0).abs() < f64::EPSILON); + assert_eq!(f.user_state, UserState::Auto); + assert!(f.evidence_refs.is_empty()); +} + +/// Running migrate twice is idempotent (no panic on duplicate column). +#[test] +fn migrate_is_idempotent() { + let conn = setup_db(); + // First call — columns already exist in PROFILE_INIT_SQL. + migrate_profile_schema(&conn); + // Second call — must not panic. + migrate_profile_schema(&conn); +} + +// ── New column round-trip ───────────────────────────────────────────────────── + +#[test] +fn profile_upsert_full_persists_phase3_fields() { + use crate::openhuman::learning::candidate::EvidenceRef; + let conn = setup_db(); + let facet = ProfileFacet { + facet_id: "f-full".into(), + facet_type: FacetType::Preference, + key: "style/verbosity".into(), + value: "terse".into(), + confidence: 0.9, + evidence_count: 3, + source_segment_ids: None, + first_seen_at: 1000.0, + last_seen_at: 1200.0, + state: FacetState::Active, + stability: 1.8, + user_state: UserState::Auto, + evidence_refs: vec![EvidenceRef::Episodic { episodic_id: 42 }], + class: Some("style".into()), + cue_families: None, + }; + profile_upsert_full(&conn, &facet).unwrap(); + + let loaded = profile_load_all(&conn).unwrap(); + assert_eq!(loaded.len(), 1); + let f = &loaded[0]; + assert_eq!(f.key, "style/verbosity"); + assert_eq!(f.state, FacetState::Active); + assert!((f.stability - 1.8).abs() < 1e-9); + assert_eq!(f.user_state, UserState::Auto); + assert_eq!(f.evidence_refs.len(), 1); + assert_eq!( + f.evidence_refs[0], + EvidenceRef::Episodic { episodic_id: 42 } + ); +} + +#[test] +fn profile_select_active_filters_by_state() { + let conn = setup_db(); + + let active = ProfileFacet { + facet_id: "f-active".into(), + facet_type: FacetType::Preference, + key: "style/tone".into(), + value: "formal".into(), + confidence: 0.85, + evidence_count: 2, + source_segment_ids: None, + first_seen_at: 1000.0, + last_seen_at: 1100.0, + state: FacetState::Active, + stability: 1.6, + user_state: UserState::Auto, + evidence_refs: vec![], + class: Some("style".into()), + cue_families: None, + }; + let provisional = ProfileFacet { + facet_id: "f-prov".into(), + facet_type: FacetType::Preference, + key: "style/length".into(), + value: "short".into(), + confidence: 0.6, + evidence_count: 1, + source_segment_ids: None, + first_seen_at: 1000.0, + last_seen_at: 1000.0, + state: FacetState::Provisional, + stability: 0.8, + user_state: UserState::Auto, + evidence_refs: vec![], + class: Some("style".into()), + cue_families: None, + }; + profile_upsert_full(&conn, &active).unwrap(); + profile_upsert_full(&conn, &provisional).unwrap(); + + let actives = profile_select_active(&conn).unwrap(); + assert_eq!(actives.len(), 1); + assert_eq!(actives[0].key, "style/tone"); +} + +#[test] +fn profile_count_by_class_groups_keys() { + let conn = setup_db(); + for (id, key) in [ + ("f1", "style/verbosity"), + ("f2", "style/tone"), + ("f3", "identity/name"), + ("f4", "no_slash"), + ] { + let f = ProfileFacet { + facet_id: id.into(), + facet_type: FacetType::Preference, + key: key.into(), + value: "v".into(), + confidence: 0.8, + evidence_count: 1, + source_segment_ids: None, + first_seen_at: 1000.0, + last_seen_at: 1000.0, + state: FacetState::Active, + stability: 1.6, + user_state: UserState::Auto, + evidence_refs: vec![], + class: None, + cue_families: None, + }; + profile_upsert_full(&conn, &f).unwrap(); + } + + let counts = profile_count_by_class(&conn).unwrap(); + assert_eq!(counts.get("style"), Some(&2)); + assert_eq!(counts.get("identity"), Some(&1)); + assert_eq!(counts.get("_other"), Some(&1)); +} + +#[test] +fn profile_set_user_state_persists() { + let conn = setup_db(); + profile_upsert( + &conn, + "f-us", + &FacetType::Preference, + "tool/editor", + "neovim", + 0.8, + None, + 1000.0, + ) + .unwrap(); + let updated = profile_set_user_state(&conn, "tool/editor", UserState::Pinned).unwrap(); + assert!(updated); + let f = profile_get_by_key(&conn, "tool/editor").unwrap().unwrap(); + assert_eq!(f.user_state, UserState::Pinned); +} + +#[test] +fn profile_delete_below_threshold_removes_dropped_only() { + let conn = setup_db(); + + let dropped_low = ProfileFacet { + facet_id: "f-drop".into(), + facet_type: FacetType::Preference, + key: "style/dropped".into(), + value: "x".into(), + confidence: 0.3, + evidence_count: 1, + source_segment_ids: None, + first_seen_at: 1000.0, + last_seen_at: 1000.0, + state: FacetState::Dropped, + stability: 0.1, + user_state: UserState::Auto, + evidence_refs: vec![], + class: Some("style".into()), + cue_families: None, + }; + let active_low = ProfileFacet { + facet_id: "f-act".into(), + facet_type: FacetType::Preference, + key: "style/active".into(), + value: "y".into(), + confidence: 0.9, + evidence_count: 5, + source_segment_ids: None, + first_seen_at: 1000.0, + last_seen_at: 1000.0, + state: FacetState::Active, + stability: 0.1, + user_state: UserState::Auto, + evidence_refs: vec![], + class: Some("style".into()), + cue_families: None, + }; + profile_upsert_full(&conn, &dropped_low).unwrap(); + profile_upsert_full(&conn, &active_low).unwrap(); + + let deleted = profile_delete_below_threshold(&conn, 0.3).unwrap(); + assert_eq!(deleted, 1); // Only the Dropped one. + let all = profile_load_all(&conn).unwrap(); + assert_eq!(all.len(), 1); + assert_eq!(all[0].key, "style/active"); +} + fn setup_db() -> Arc> { let conn = Connection::open_in_memory().unwrap(); conn.execute_batch(PROFILE_INIT_SQL).unwrap(); @@ -94,6 +345,12 @@ fn render_profile_context_formats_correctly() { source_segment_ids: None, first_seen_at: 1000.0, last_seen_at: 1002.0, + state: FacetState::Active, + stability: 0.0, + user_state: UserState::Auto, + evidence_refs: vec![], + class: None, + cue_families: None, }, ProfileFacet { facet_id: "f-2".into(), @@ -105,6 +362,12 @@ fn render_profile_context_formats_correctly() { source_segment_ids: None, first_seen_at: 1000.0, last_seen_at: 1000.0, + state: FacetState::Active, + stability: 0.0, + user_state: UserState::Auto, + evidence_refs: vec![], + class: None, + cue_families: None, }, ]; diff --git a/src/openhuman/memory/tree/ingest.rs b/src/openhuman/memory/tree/ingest.rs index 3475526a8..d42661d2c 100644 --- a/src/openhuman/memory/tree/ingest.rs +++ b/src/openhuman/memory/tree/ingest.rs @@ -9,6 +9,7 @@ use anyhow::Result; use serde::{Deserialize, Serialize}; +use crate::core::event_bus::{publish_global, DomainEvent}; use crate::openhuman::config::Config; use crate::openhuman::memory::tree::canonicalize::{ chat::{self, ChatBatch}, @@ -23,6 +24,7 @@ use crate::openhuman::memory::tree::score::{self, ScoreResult, ScoringConfig}; use crate::openhuman::memory::tree::store; use crate::openhuman::memory::tree::types::SourceKind; use crate::openhuman::memory::tree::util::redact::redact; +use std::time::{SystemTime, UNIX_EPOCH}; /// Outcome of one ingest call. #[derive(Clone, Debug, Serialize, Deserialize)] @@ -154,6 +156,25 @@ async fn persist( canonical: CanonicalisedSource, ) -> Result { let source_kind_for_store = canonical.metadata.source_kind; + + // Capture body_preview before the canonical markdown is moved into the chunker. + // For email and document sources: last ≤ 2 048 chars of the canonical markdown + // are enough for signature parsing and similar lightweight subscribers. Chat + // sources are conversational and have no trailing structure worth scanning, so + // they get body_preview = None. + let body_preview: Option = match source_kind_for_store { + SourceKind::Email | SourceKind::Document => { + let md = &canonical.markdown; + let len = md.len(); + Some(if len <= 2048 { + md.clone() + } else { + md[len - 2048..].to_string() + }) + } + _ => None, + }; + let input = ChunkerInput { source_kind: canonical.metadata.source_kind, source_id: source_id.to_string(), @@ -308,11 +329,34 @@ async fn persist( jobs::wake_workers(); + let chunk_ids: Vec = staged.iter().map(|s| s.chunk.id.clone()).collect(); + + // Emit DocumentCanonicalized so Phase 2 producers (e.g. email-signature parser) + // can react to new canonicalised content. Non-fatal: ingest has already succeeded. + // `source_kind_for_store` is Copy so it is still accessible here after the closure. + let now_secs = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64(); + publish_global(DomainEvent::DocumentCanonicalized { + source_id: source_id.to_string(), + source_kind: source_kind_for_store.as_str().to_string(), + chunks_written: written, + chunk_ids: chunk_ids.clone(), + canonicalized_at: now_secs, + body_preview, + }); + tracing::debug!( + "[memory::tree::ingest] published DocumentCanonicalized source_id={} chunks={}", + source_id, + written + ); + Ok(IngestResult { source_id: source_id.to_string(), chunks_written: written, chunks_dropped: dropped, - chunk_ids: staged.iter().map(|s| s.chunk.id.clone()).collect(), + chunk_ids, already_ingested: false, }) } diff --git a/src/openhuman/memory/tree/tree_source/summariser/llm.rs b/src/openhuman/memory/tree/tree_source/summariser/llm.rs index 06a09d45a..a9f888ad3 100644 --- a/src/openhuman/memory/tree/tree_source/summariser/llm.rs +++ b/src/openhuman/memory/tree/tree_source/summariser/llm.rs @@ -42,6 +42,7 @@ use std::sync::Arc; use super::inert::InertSummariser; use super::{Summariser, SummaryContext, SummaryInput, SummaryOutput}; +use crate::openhuman::learning::extract::summary_facets::{self, StructuredSummary}; use crate::openhuman::memory::tree::chat::{ChatPrompt, ChatProvider}; use crate::openhuman::memory::tree::types::approx_token_count; @@ -78,12 +79,19 @@ pub struct LlmSummariserConfig { /// Model identifier (e.g. `summarization-v1` for cloud, `qwen2.5:0.5b` /// or `llama3.1:8b` for local Ollama). Diagnostic / log only. pub model: String, + /// When `true` (the default), the summariser appends a structured facet + /// extraction request to the prompt and parses the resulting JSON block. + /// Discovered facets are routed to the learning candidate buffer. + /// Set to `false` to restore the plain-text-only behaviour for A/B testing + /// or debugging. + pub structured_facet_extraction: bool, } impl Default for LlmSummariserConfig { fn default() -> Self { Self { model: "qwen2.5:0.5b".to_string(), + structured_facet_extraction: true, } } } @@ -108,9 +116,12 @@ impl LlmSummariser { } /// Build the chat prompt sent to the provider for a given seal. + /// + /// When `structured_facet_extraction` is enabled the system prompt includes + /// an instruction to emit a fenced `json` block after the prose summary. fn build_prompt(&self, prompt_body: &str, budget: u32) -> ChatPrompt { ChatPrompt { - system: system_prompt(budget), + system: system_prompt(budget, self.cfg.structured_facet_extraction), user: prompt_body.to_string(), temperature: 0.0, kind: "memory_tree::summarise", @@ -193,7 +204,46 @@ impl Summariser for LlmSummariser { } }; - let (content, token_count) = clamp_to_budget(raw.trim(), effective_budget); + // When structured_facet_extraction is enabled, attempt to split the response + // into a prose summary and an optional JSON block. On parse failure, the + // prose is used as-is and zero facets are emitted (fail-soft). + let summary_text: &str; + + if self.cfg.structured_facet_extraction { + let (prose, maybe_structured) = split_structured_response(raw.trim()); + summary_text = prose; + match maybe_structured { + Some(Ok(parsed)) => { + tracing::debug!( + "[learning::extract::summary] source_id={} facets_emitted={}", + ctx.tree_id, + parsed.facets.len() + ); + summary_facets::route_facets_to_buffer(&parsed, ctx.tree_id); + } + Some(Err(e)) => { + log::warn!( + "[tree_source::summariser::llm] structured facet parse failed \ + tree_id={} level={}: {e:#} — using raw prose, emitting 0 facets", + ctx.tree_id, + ctx.target_level + ); + } + None => { + // No JSON block present — normal for content with no clear signals. + tracing::debug!( + "[tree_source::summariser::llm] no structured JSON block in response \ + tree_id={} level={}", + ctx.tree_id, + ctx.target_level + ); + } + } + } else { + summary_text = raw.trim(); + } + + let (content, token_count) = clamp_to_budget(summary_text, effective_budget); log::debug!( "[tree_source::summariser::llm] sealed tree_id={} level={} inputs={} tokens={}", ctx.tree_id, @@ -235,19 +285,95 @@ fn build_user_prompt(inputs: &[SummaryInput], per_input_cap_tokens: u32) -> Stri out } -/// System prompt. Length isn't templated in — empirically, telling -/// instruction-tuned models "stay under N tokens" makes them produce -/// curt, generic output even when the input has plenty of substance. -/// Output is clamped post-generation by [`clamp_to_budget`] in the -/// caller, so we don't need the model to self-police length. -fn system_prompt(_budget: u32) -> String { - "You are a precise summariser. Summarise the user-provided contributions into a \ +/// System prompt. +/// +/// When `structured_facets` is `true`, appends instructions for the model to +/// emit a fenced `json` block after the prose summary containing any clearly +/// evidenced facets. +/// +/// Length isn't templated in — empirically, telling instruction-tuned models +/// "stay under N tokens" makes them produce curt, generic output even when the +/// input has plenty of substance. Output is clamped post-generation by +/// [`clamp_to_budget`] in the caller. +fn system_prompt(_budget: u32, structured_facets: bool) -> String { + let base = "You are a precise summariser. Summarise the user-provided contributions into a \ single cohesive passage that preserves concrete facts, decisions, \ and temporal ordering. Do not invent facts.\n\ \n\ - Return only the summary text. No commentary, no preamble, no headings, \ - no markdown wrappers, no JSON — just the prose summary." - .to_string() + Return the summary text first."; + + if !structured_facets { + return format!( + "{base} No commentary, no preamble, no headings, \ + no markdown wrappers, no JSON — just the prose summary." + ); + } + + format!( + "{base}\n\ + \n\ + After the summary, output a JSON object as the second part of your response, \ + fenced in a ```json block:\n\ + \n\ + ```json\n\ + {{\n\ + \"summary\": \"\",\n\ + \"facets\": [\n\ + {{\n\ + \"class\": \"style|identity|tooling|veto|goal\",\n\ + \"key\": \"\",\n\ + \"value\": \"\",\n\ + \"evidence_chunks\": [\"\", \"...\"],\n\ + \"confidence\": 0.0,\n\ + \"cue_family\": \"explicit|structural|behavioral\"\n\ + }}\n\ + ]\n\ + }}\n\ + ```\n\ + \n\ + Rules:\n\ + - Only include facets that are clearly evidenced in the content above.\n\ + - Each facet must cite at least one chunk_id from this batch (the id in brackets \ + before each contribution, e.g. [chunk-abc]).\n\ + - Use canonical keys: verbosity, format, name, timezone, role, package_manager, \ + lang, framework, runtime, etc.\n\ + - Cap the facets array at 8 items per call. Skip the array entirely (emit \ + facets: []) if no clear evidence.\n\ + - No commentary outside the prose summary and the JSON block." + ) +} + +/// Split a raw LLM response that may contain a trailing fenced `json` block. +/// +/// Returns `(prose, Option)` where: +/// - `prose` is the text before the ` ```json ` fence (trimmed), or the full +/// raw text when no fence is present. +/// - The second element is `None` when no fence was found, or +/// `Some(Ok(StructuredSummary))` / `Some(Err(…))` on parse success/failure. +fn split_structured_response(raw: &str) -> (&str, Option>) { + // Look for the opening ` ```json ` fence. + const OPEN_FENCE: &str = "```json"; + const CLOSE_FENCE: &str = "```"; + + let Some(fence_start) = raw.find(OPEN_FENCE) else { + return (raw, None); + }; + + let prose = raw[..fence_start].trim(); + let after_open = &raw[fence_start + OPEN_FENCE.len()..]; + + // Find the closing fence. + let json_str = if let Some(close_pos) = after_open.find(CLOSE_FENCE) { + after_open[..close_pos].trim() + } else { + // No closing fence — treat everything after the open as JSON. + after_open.trim() + }; + + let result = serde_json::from_str::(json_str) + .map_err(|e| anyhow::anyhow!("structured summary JSON parse error: {e}")); + + (prose, Some(result)) } /// Truncate to the caller's token budget using the same ~4 chars/token @@ -346,18 +472,74 @@ mod tests { #[test] fn system_prompt_describes_plain_text_output() { - // Budget is no longer templated into the prompt — models - // produced overly curt output when told to "stay under N tokens". - // The clamp in `clamp_to_budget` handles enforcement instead. - let p = system_prompt(4096); + // When structured_facets is disabled, the prompt asks for plain prose. + let p = system_prompt(4096, false); assert!(!p.contains("4096")); assert!(!p.contains("Stay well under")); - // Output is plain prose, not JSON. assert!(!p.contains("\"summary\"")); assert!(p.to_lowercase().contains("no commentary")); assert!(p.to_lowercase().contains("no json")); } + #[test] + fn extends_prompt_when_flag_enabled() { + let p = system_prompt(4096, true); + // When structured_facets is true, the prompt should contain the JSON fence instruction. + assert!( + p.contains("```json"), + "should contain JSON fence instruction" + ); + assert!(p.contains("\"facets\""), "should mention the facets array"); + assert!( + p.contains("evidence_chunks"), + "should mention evidence_chunks" + ); + assert!( + p.contains("canonical keys"), + "should specify canonical keys" + ); + } + + #[test] + fn parses_well_formed_response() { + let raw = "The user prefers pnpm.\n\n\ + ```json\n\ + {\"summary\": \"The user prefers pnpm.\", \"facets\": [\ + {\"class\": \"tooling\", \"key\": \"package_manager\", \ + \"value\": \"pnpm\", \"evidence_chunks\": [\"c1\"], \ + \"confidence\": 0.9, \"cue_family\": \"explicit\"}\ + ]}\n\ + ```"; + let (prose, maybe) = split_structured_response(raw); + assert!( + prose.contains("prefers pnpm"), + "prose should precede the JSON block" + ); + let parsed = maybe + .expect("should find JSON block") + .expect("should parse"); + assert_eq!(parsed.facets.len(), 1); + assert_eq!(parsed.facets[0].key, "package_manager"); + } + + #[test] + fn gracefully_falls_back_on_invalid_json() { + let raw = "Summary text.\n\n```json\nnot valid json\n```"; + let (prose, maybe) = split_structured_response(raw); + assert!(prose.contains("Summary"), "prose should be extracted"); + let result = maybe.expect("fence found"); + assert!(result.is_err(), "invalid JSON should produce Err"); + } + + #[test] + fn respects_disabled_flag() { + let p = system_prompt(4096, false); + assert!( + !p.contains("```json"), + "disabled flag must omit JSON instruction" + ); + } + #[test] fn clamp_to_budget_no_op_when_under() { let (out, t) = clamp_to_budget("short", 1000); @@ -409,13 +591,21 @@ mod tests { } } + /// Helper config with structured facet extraction disabled for legacy tests. + fn no_facets_cfg() -> LlmSummariserConfig { + LlmSummariserConfig { + model: "qwen2.5:0.5b".into(), + structured_facet_extraction: false, + } + } + #[tokio::test] async fn empty_inputs_yield_empty_summary_without_provider_call() { // All inputs are blank → prompt body is empty → the summariser // short-circuits and returns an empty output without invoking the // chat provider. let provider = std::sync::Arc::new(StubProvider::ok("never returned")); - let s = LlmSummariser::new(LlmSummariserConfig::default(), provider.clone()); + let s = LlmSummariser::new(no_facets_cfg(), provider.clone()); let inputs = vec![sample_input("a", " "), sample_input("b", "")]; let out = s.summarise(&inputs, &test_ctx()).await.unwrap(); assert!(out.content.is_empty()); @@ -433,7 +623,7 @@ mod tests { // InertSummariser's concatenate+truncate behaviour (content // present, entities empty). let provider = std::sync::Arc::new(StubProvider::err("simulated")); - let s = LlmSummariser::new(LlmSummariserConfig::default(), provider); + let s = LlmSummariser::new(no_facets_cfg(), provider); let inputs = vec![sample_input("a", "alice decided to ship friday")]; let out = s.summarise(&inputs, &test_ctx()).await.unwrap(); assert!(out.content.contains("alice decided to ship")); @@ -446,7 +636,7 @@ mod tests { // Provider returns plain text; summariser uses it verbatim // (after trim) and clamps to the budget. let provider = std::sync::Arc::new(StubProvider::ok("alice decided to ship friday\n")); - let s = LlmSummariser::new(LlmSummariserConfig::default(), provider.clone()); + let s = LlmSummariser::new(no_facets_cfg(), provider.clone()); let inputs = vec![sample_input("a", "alice ships friday")]; let out = s.summarise(&inputs, &test_ctx()).await.unwrap(); assert_eq!(out.content, "alice decided to ship friday"); @@ -457,9 +647,11 @@ mod tests { #[test] fn build_prompt_carries_body_and_kind_tag() { let provider = std::sync::Arc::new(StubProvider::ok("hi")); + // With structured_facet_extraction disabled, expect plain-text prompt. let s = LlmSummariser::new( LlmSummariserConfig { model: "llama3.1:8b".into(), + structured_facet_extraction: false, }, provider, ); diff --git a/src/openhuman/memory/tree/tree_source/summariser/mod.rs b/src/openhuman/memory/tree/tree_source/summariser/mod.rs index 0b0863c51..b0cd3af15 100644 --- a/src/openhuman/memory/tree/tree_source/summariser/mod.rs +++ b/src/openhuman/memory/tree/tree_source/summariser/mod.rs @@ -142,7 +142,10 @@ pub fn build_summariser(config: &Config) -> Arc { model ); Arc::new(llm::LlmSummariser::new( - llm::LlmSummariserConfig { model }, + llm::LlmSummariserConfig { + model, + structured_facet_extraction: true, + }, provider, )) } diff --git a/tests/learning_phase4_integration_test.rs b/tests/learning_phase4_integration_test.rs new file mode 100644 index 000000000..2cb02620f --- /dev/null +++ b/tests/learning_phase4_integration_test.rs @@ -0,0 +1,310 @@ +//! Phase 4 integration test for agent self-learning (#566). +//! +//! Exercises the full end-to-end pipeline: +//! 1. Initialize a temp memory client + facet cache + stability detector + profile_md renderer. +//! 2. Push 5 candidates spanning multiple classes. +//! 3. Call `StabilityDetector::rebuild()`. +//! 4. Verify `CacheRebuilt` event fired. +//! 5. Verify `ProfileMdRenderer` wrote to `PROFILE.md` with expected blocks + bullets. +//! 6. Call `learning.pin_facet` RPC for one entry → re-rebuild → verify it stays Active. +//! 7. Call `learning.forget_facet` for another → verify it disappears from PROFILE.md. +//! 8. Call `learning.list_facets` → verify shape. +//! +//! Run with: `cargo test --test learning_phase4_integration_test` + +use std::sync::Arc; +use std::time::SystemTime; +use std::time::UNIX_EPOCH; + +use openhuman_core::openhuman::learning::cache::{class_prefix, FacetCache}; +use openhuman_core::openhuman::learning::candidate::{ + self as candidate, CueFamily, EvidenceRef, FacetClass, LearningCandidate, +}; +use openhuman_core::openhuman::learning::profile_md_renderer::ProfileMdRenderer; +use openhuman_core::openhuman::learning::stability_detector::StabilityDetector; +use openhuman_core::openhuman::memory::store::profile::{ + FacetState, FacetType, ProfileFacet, UserState, PROFILE_INIT_SQL, +}; +use parking_lot::Mutex; +use rusqlite::Connection; +use tempfile::TempDir; + +fn now_secs() -> f64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64() +} + +fn make_candidate( + class: FacetClass, + key: &str, + value: &str, + cue: CueFamily, + now: f64, +) -> LearningCandidate { + LearningCandidate { + class, + key: key.into(), + value: value.into(), + cue_family: cue, + evidence: EvidenceRef::Episodic { episodic_id: 1 }, + initial_confidence: 0.9, + observed_at: now, + } +} + +/// Build a test harness backed by an in-memory SQLite database. +/// +/// Uses the global candidate buffer (Phase 3 public API). To avoid test +/// interference the harness immediately drains the buffer before pushing +/// its own candidates. +struct TestHarness { + cache: Arc, + detector: StabilityDetector, + renderer: Arc, + workspace: TempDir, +} + +impl TestHarness { + fn new() -> Self { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(PROFILE_INIT_SQL).unwrap(); + let conn = Arc::new(Mutex::new(conn)); + + let cache = Arc::new(FacetCache::new(Arc::clone(&conn))); + + let workspace = TempDir::new().unwrap(); + let renderer = Arc::new(ProfileMdRenderer::new( + Arc::clone(&cache), + workspace.path().to_path_buf(), + )); + + // Drain any stale candidates from prior tests so they don't affect + // this test's results. + let _ = candidate::global().drain(); + + let detector = StabilityDetector::new(FacetCache::new(conn)); + + TestHarness { + cache, + detector, + renderer, + workspace, + } + } +} + +// ── The integration test ────────────────────────────────────────────────────── + +#[test] +fn phase4_end_to_end_pin_forget_profile_md_list() { + let harness = TestHarness::new(); + let now = now_secs(); + + // Step 1: Push 5 candidates spanning multiple classes. + // Push enough explicit evidence per candidate to clear τ_promote = 1.5. + let candidates = [ + (FacetClass::Style, "verbosity", "terse"), + (FacetClass::Identity, "name", "Alice"), + (FacetClass::Tooling, "editor", "neovim"), + ( + FacetClass::Goal, + "primary", + "Ship the agent self-learning feature", + ), + (FacetClass::Veto, "no-emojis", "avoid emojis in output"), + ]; + + for (class, key, value) in &candidates { + // Push 5 explicit candidates per key so the aggregated stability clears τ_promote. + for j in 0..5_u32 { + candidate::global().push(make_candidate( + *class, + key, + value, + CueFamily::Explicit, + now - f64::from(j), + )); + } + } + + // Step 2: Run rebuild. + let outcome = harness.detector.rebuild(now).unwrap(); + assert!( + outcome.added >= 1, + "rebuild should have added rows: {outcome:?}" + ); + + // Step 3: Verify all 5 candidates are now Active. + let active = harness.cache.list_active().unwrap(); + assert!( + active.len() >= 5, + "expected ≥ 5 active rows, got {}: {:?}", + active.len(), + active.iter().map(|f| &f.key).collect::>(), + ); + + // Step 4: Render PROFILE.md via the renderer. + harness.renderer.render().unwrap(); + + let profile_path = harness.workspace.path().join("PROFILE.md"); + assert!(profile_path.exists(), "PROFILE.md was not created"); + let profile_content = std::fs::read_to_string(&profile_path).unwrap(); + + // Verify expected blocks. + assert!( + profile_content.contains("## Style"), + "Style block missing:\n{profile_content}" + ); + assert!( + profile_content.contains("## Identity"), + "Identity block missing:\n{profile_content}" + ); + assert!( + profile_content.contains("## Tooling"), + "Tooling block missing:\n{profile_content}" + ); + assert!( + profile_content.contains("## Goals"), + "Goals block missing:\n{profile_content}" + ); + assert!( + profile_content.contains("## Vetoes"), + "Vetoes block missing:\n{profile_content}" + ); + // Style facet should be in **key**: value format. + assert!( + profile_content.contains("terse"), + "style/verbosity=terse missing:\n{profile_content}" + ); + // Goal should render as plain sentence. + assert!( + profile_content.contains("Ship the agent self-learning feature"), + "goal sentence missing:\n{profile_content}" + ); + + // Step 5: Pin the style/verbosity facet. + let style_key = format!("{}/verbosity", class_prefix(FacetClass::Style)); + harness + .cache + .set_user_state(&style_key, UserState::Pinned) + .unwrap(); + + // Re-rebuild with no new candidates (only decay applies). + let outcome2 = harness.detector.rebuild(now).unwrap(); + // The pinned row should remain Active regardless of decay. + let pinned_facet = harness.cache.get(&style_key).unwrap(); + assert!(pinned_facet.is_some(), "pinned row must survive re-rebuild"); + let pf = pinned_facet.unwrap(); + assert_eq!( + pf.state, + FacetState::Active, + "pinned row must stay Active after re-rebuild" + ); + assert_eq!(pf.user_state, UserState::Pinned); + let _ = outcome2; // used for assertion comment + + // Re-render and verify pin marker. + harness.renderer.render().unwrap(); + let profile_after_pin = std::fs::read_to_string(&profile_path).unwrap(); + assert!( + profile_after_pin.contains("*(pinned)*"), + "pinned marker missing from PROFILE.md:\n{profile_after_pin}" + ); + + // Step 6: Forget the identity/name facet. + let identity_key = format!("{}/name", class_prefix(FacetClass::Identity)); + let mut identity_facet = harness.cache.get(&identity_key).unwrap().unwrap(); + identity_facet.user_state = UserState::Forgotten; + identity_facet.state = FacetState::Dropped; + harness.cache.upsert(&identity_facet).unwrap(); + + // Re-render. + harness.renderer.render().unwrap(); + let profile_after_forget = std::fs::read_to_string(&profile_path).unwrap(); + // identity/name=Alice should no longer appear in the visible sections. + // (The identity block placeholder renders if all identity rows are non-active.) + let identity_block_start = profile_after_forget + .find("") + .unwrap_or(0); + let identity_block_end = profile_after_forget + .find("") + .unwrap_or(profile_after_forget.len()); + let identity_block_content = &profile_after_forget[identity_block_start..identity_block_end]; + assert!( + !identity_block_content.contains("Alice") + || identity_block_content.contains("*(no entries yet)*"), + "forgotten facet Alice must not appear in identity block:\n{identity_block_content}" + ); + + // Step 7: list_facets — verify shape. + let all_active = harness.cache.list_active().unwrap(); + // The style facet should be present (pinned, Active). + assert!( + all_active.iter().any(|f| f.key == style_key), + "pinned style/verbosity must appear in list_active" + ); + // The forgotten identity/name should NOT appear in Active list. + assert!( + !all_active.iter().any(|f| f.key == identity_key), + "forgotten identity/name must not appear in list_active" + ); + + // Verify the connected-accounts block is untouched (never written by renderer). + assert!( + !profile_after_forget.contains(""), + "connected-accounts block should not be written by the renderer" + ); + + println!("phase4_end_to_end_pin_forget_profile_md_list: all assertions passed"); +} + +// ── list_facets unit-level smoke test (no RPC server needed) ───────────────── + +#[test] +fn list_facets_cache_direct_active_vs_all() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(PROFILE_INIT_SQL).unwrap(); + let cache = FacetCache::new(Arc::new(Mutex::new(conn))); + + let make = |id: &str, key: &str, state: FacetState| ProfileFacet { + facet_id: id.into(), + facet_type: FacetType::Preference, + key: key.into(), + value: "val".into(), + confidence: 0.8, + evidence_count: 2, + source_segment_ids: None, + first_seen_at: 1000.0, + last_seen_at: 2000.0, + state, + stability: 2.0, + user_state: UserState::Auto, + evidence_refs: vec![], + class: key.split('/').next().map(str::to_string), + cue_families: None, + }; + + cache + .upsert(&make("f1", "style/verbosity", FacetState::Active)) + .unwrap(); + cache + .upsert(&make("f2", "style/tone", FacetState::Provisional)) + .unwrap(); + cache + .upsert(&make("f3", "identity/name", FacetState::Dropped)) + .unwrap(); + + let active = cache.list_active().unwrap(); + assert_eq!( + active.len(), + 1, + "list_active should only return Active rows" + ); + assert_eq!(active[0].key, "style/verbosity"); + + let all = cache.list_all().unwrap(); + // All 3 rows (Active + Provisional + Dropped). + assert_eq!(all.len(), 3, "list_all should return all rows"); +}