From 238a9a5ad9b89521ddc19b8849fb65dc405e808f Mon Sep 17 00:00:00 2001 From: sanil-23 Date: Mon, 27 Apr 2026 07:11:24 +0530 Subject: [PATCH] feat(slack): backfill ingestion + LLM summariser + tree fanout gate (#934) Co-authored-by: Claude Opus 4.7 (1M context) --- Cargo.toml | 4 + src/bin/slack_backfill.rs | 409 ++++++++ src/core/all.rs | 3 + src/openhuman/about_app/catalog.rs | 10 + src/openhuman/channels/runtime/startup.rs | 4 + src/openhuman/composio/providers/mod.rs | 1 + src/openhuman/composio/providers/registry.rs | 1 + src/openhuman/composio/providers/slack/mod.rs | 16 + .../composio/providers/slack/provider.rs | 920 ++++++++++++++++++ .../composio/providers/slack/sync.rs | 397 ++++++++ .../composio/providers/slack/users.rs | 364 +++++++ src/openhuman/config/schema/load.rs | 54 + src/openhuman/config/schema/storage_memory.rs | 66 ++ src/openhuman/memory/mod.rs | 4 + .../memory/slack_ingestion/bucketer.rs | 261 +++++ src/openhuman/memory/slack_ingestion/mod.rs | 46 + src/openhuman/memory/slack_ingestion/ops.rs | 234 +++++ src/openhuman/memory/slack_ingestion/rpc.rs | 177 ++++ .../memory/slack_ingestion/schemas.rs | 128 +++ src/openhuman/memory/slack_ingestion/types.rs | 69 ++ src/openhuman/memory/tree/chunker.rs | 3 +- .../memory/tree/global_tree/digest_tests.rs | 4 +- .../memory/tree/global_tree/recap.rs | 4 +- src/openhuman/memory/tree/ingest.rs | 23 +- .../memory/tree/retrieval/drill_down.rs | 2 +- src/openhuman/memory/tree/retrieval/fetch.rs | 2 +- src/openhuman/memory/tree/retrieval/global.rs | 2 +- .../memory/tree/retrieval/integration_test.rs | 2 +- src/openhuman/memory/tree/retrieval/rpc.rs | 2 +- src/openhuman/memory/tree/retrieval/source.rs | 2 +- src/openhuman/memory/tree/score/mod.rs | 58 ++ src/openhuman/memory/tree/score/mod_tests.rs | 2 +- .../memory/tree/source_tree/bucket_seal.rs | 73 +- .../tree/source_tree/bucket_seal_tests.rs | 147 ++- .../memory/tree/source_tree/flush.rs | 4 +- src/openhuman/memory/tree/source_tree/mod.rs | 2 +- .../memory/tree/source_tree/summariser/llm.rs | 607 ++++++++++++ .../memory/tree/source_tree/summariser/mod.rs | 67 ++ .../memory/tree/source_tree/types.rs | 14 + src/openhuman/memory/tree/store_tests.rs | 2 +- .../memory/tree/topic_tree/backfill.rs | 2 +- .../memory/tree/topic_tree/curator.rs | 2 +- .../memory/tree/topic_tree/routing.rs | 2 +- src/openhuman/memory/tree/types.rs | 48 +- 44 files changed, 4196 insertions(+), 48 deletions(-) create mode 100644 src/bin/slack_backfill.rs create mode 100644 src/openhuman/composio/providers/slack/mod.rs create mode 100644 src/openhuman/composio/providers/slack/provider.rs create mode 100644 src/openhuman/composio/providers/slack/sync.rs create mode 100644 src/openhuman/composio/providers/slack/users.rs create mode 100644 src/openhuman/memory/slack_ingestion/bucketer.rs create mode 100644 src/openhuman/memory/slack_ingestion/mod.rs create mode 100644 src/openhuman/memory/slack_ingestion/ops.rs create mode 100644 src/openhuman/memory/slack_ingestion/rpc.rs create mode 100644 src/openhuman/memory/slack_ingestion/schemas.rs create mode 100644 src/openhuman/memory/slack_ingestion/types.rs create mode 100644 src/openhuman/memory/tree/source_tree/summariser/llm.rs diff --git a/Cargo.toml b/Cargo.toml index feacc638f..38b113570 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,10 @@ autobins = false name = "openhuman-core" path = "src/main.rs" +[[bin]] +name = "slack-backfill" +path = "src/bin/slack_backfill.rs" + [lib] name = "openhuman_core" crate-type = ["rlib"] diff --git a/src/bin/slack_backfill.rs b/src/bin/slack_backfill.rs new file mode 100644 index 000000000..fffeffd78 --- /dev/null +++ b/src/bin/slack_backfill.rs @@ -0,0 +1,409 @@ +//! Manual smoke/backfill trigger for the Composio-backed Slack +//! provider. +//! +//! Invokes the same path the 15-minute periodic scheduler uses — +//! `SlackProvider::sync()` for each active Slack Composio connection — +//! but runs exactly **once** so operators can observe results end to +//! end before trusting the scheduler. +//! +//! # Prerequisites +//! +//! - A working openhuman install (same workspace dir the desktop app +//! uses) with a signed-in session JWT. +//! - A Slack connection created via Composio's OAuth flow (e.g. from +//! the desktop app's Integrations screen). No self-hosted Slack App +//! or bot token is needed — authorization lives in Composio. +//! - Ollama pulled with whatever models you want the ingest pipeline to +//! use (embedder, LLM NER, LLM summariser). Any of these can be left +//! unconfigured — `memory/tree/ingest` soft-falls-back per call. +//! +//! # Usage +//! +//! ```sh +//! export OPENHUMAN_WORKSPACE=/path/to/workspace # must match desktop app +//! export OPENHUMAN_MEMORY_EMBED_ENDPOINT=http://localhost:11434 +//! export OPENHUMAN_MEMORY_EMBED_MODEL=nomic-embed-text +//! export OPENHUMAN_MEMORY_EXTRACT_ENDPOINT=http://localhost:11434 +//! export OPENHUMAN_MEMORY_EXTRACT_MODEL=qwen2.5:0.5b +//! export OPENHUMAN_MEMORY_SUMMARISE_ENDPOINT=http://localhost:11434 +//! export OPENHUMAN_MEMORY_SUMMARISE_MODEL=llama3.1:8b +//! export RUST_LOG=info,openhuman_core::openhuman::composio::providers::slack=debug,openhuman_core::openhuman::memory=debug +//! +//! cargo run --bin slack-backfill # all active slack connections +//! cargo run --bin slack-backfill -- --connection conn_abc # one specific connection +//! ``` + +use std::sync::Arc; +use std::time::Instant; + +use anyhow::{bail, Context, Result}; +use clap::Parser; + +use openhuman_core::openhuman::composio::client::build_composio_client; +use openhuman_core::openhuman::composio::providers::registry::{ + get_provider, init_default_providers, +}; +use openhuman_core::openhuman::composio::providers::slack::run_backfill_via_search; +use openhuman_core::openhuman::composio::providers::{ProviderContext, SyncReason}; +use openhuman_core::openhuman::config::Config; +use openhuman_core::openhuman::memory; + +#[derive(Parser, Debug)] +#[command( + name = "slack-backfill", + about = "Run SlackProvider::sync() once against the user's Composio-authorized Slack connection(s)." +)] +struct Cli { + /// Optional Composio connection id. When omitted, every active + /// Slack connection is synced. + #[arg(long = "connection")] + connection_id: Option, + + /// Reset the per-connection SyncState before syncing — wipes the + /// per-channel cursor map + dedup set + daily budget. The next + /// sync re-walks the full backfill window. Useful when you've + /// changed canonicalisation logic and want to overwrite existing + /// chunks (chunk-id determinism makes the rewrite an UPSERT). + #[arg(long = "reset-state", default_value_t = false)] + reset_state: bool, + + /// One-shot: invoke `SLACK_SEARCH_MESSAGES` with a small query and + /// print the raw response, then exit. Probe to see if the + /// workspace's Slack plan supports `search.messages` (paid plans + /// only) before we consider rebuilding the provider around it. + /// Skips the normal backfill flow. + #[arg(long = "probe-search", default_value_t = false)] + probe_search: bool, + + /// Use the workspace-wide `SLACK_SEARCH_MESSAGES` path instead of + /// per-channel `conversations.history`. Better quota efficiency + /// (each successful call returns matches across many channels) + /// but requires the workspace to be on a paid Slack plan. + /// `--days` controls the backfill window. + #[arg(long = "use-search", default_value_t = false)] + use_search: bool, + + /// Backfill window in days when `--use-search` is set. Defaults to + /// 30 unless `OPENHUMAN_SLACK_BACKFILL_DAYS` overrides. + #[arg(long = "days", default_value_t = 30)] + days: i64, + + /// Synthesise a tiny single-message `ChatBatch` and ingest it + /// under the existing per-connection `source_id` to trigger a + /// seal cascade against the existing L0 buffer (without + /// re-fetching from Slack/Composio). Useful after fixing a seal- + /// downstream bug — the existing 15k-token buffer immediately + /// re-attempts cascade on the next append. + #[arg(long = "seal-probe", default_value_t = false)] + seal_probe: bool, +} + +#[tokio::main] +async fn main() -> Result<()> { + // env_logger captures `log::*` events (used by reqwest, the + // memory-tree pipeline, the slack ingestion ops layer, …). + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")) + .format_timestamp_secs() + .try_init() + .ok(); // ignore double-init in test harness scenarios. + + // tracing-subscriber captures `tracing::*` events (used by the + // composio-side providers, including SlackProvider). Without this, + // channel-level warn logs from `process_channel` are silent and + // backfill failures look like silent zeros. Filter respects + // `RUST_LOG` (e.g. `RUST_LOG=info,openhuman_core=debug`). + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .with_target(true) + .try_init() + .ok(); + + let cli = Cli::parse(); + + // Load real on-disk config — same path the full core uses — so + // `memory_tree.embedding_*`, `llm_extractor_*`, and + // `llm_summariser_*` settings apply automatically. + let config = Config::load_or_init() + .await + .context("[slack_backfill] Config::load_or_init failed")?; + std::fs::create_dir_all(&config.workspace_dir).with_context(|| { + format!( + "failed to create workspace dir: {}", + config.workspace_dir.display() + ) + })?; + let config = Arc::new(config); + + // Bootstrap the memory global so `SyncState` KV reads/writes work + // from inside `SlackProvider::sync()`. `init` is idempotent and + // returns the (possibly pre-existing) client. + memory::global::init(config.workspace_dir.clone()) + .map_err(|e| anyhow::anyhow!("[slack_backfill] memory::global::init failed: {e}"))?; + + // Register the default Composio providers (gmail, notion, slack). + // Idempotent — safe even if called twice. + init_default_providers(); + + let provider = get_provider("slack").ok_or_else(|| { + anyhow::anyhow!("SlackProvider not registered after init_default_providers") + })?; + + let client = build_composio_client(&config).ok_or_else(|| { + anyhow::anyhow!( + "No Composio client — user not signed in (no JWT). \ + Sign in via the desktop app first, then re-run this binary." + ) + })?; + + if cli.seal_probe { + use chrono::{Duration, Utc}; + use openhuman_core::openhuman::memory::tree::canonicalize::chat::{ChatBatch, ChatMessage}; + use openhuman_core::openhuman::memory::tree::ingest::ingest_chat; + + let connection_id = cli.connection_id.clone().ok_or_else(|| { + anyhow::anyhow!( + "--seal-probe requires --connection so the probe message \ + lands on a real Slack source tree (no implicit default)" + ) + })?; + let source_id = format!("slack:{connection_id}"); + let batch = ChatBatch { + platform: "slack".into(), + channel_label: "#seal-probe".into(), + messages: vec![ChatMessage { + author: "seal-probe".into(), + timestamp: Utc::now() - Duration::days(2), + text: format!( + "Seal-cascade probe message at {} — triggers append_leaf \ + against the existing per-connection source tree's L0 \ + buffer (already over 10k tokens) so cascade_seals fires \ + immediately. Used to verify the LlmSummariser→embedder \ + fix without re-fetching from Composio.", + Utc::now().to_rfc3339() + ), + source_ref: Some("probe://seal-cascade".into()), + }], + }; + log::info!( + "[slack_backfill] seal-probe: ingesting 1 message under source_id={}", + source_id + ); + let result = ingest_chat( + &config, + &source_id, + "", + vec!["probe".into(), "seal-cascade".into()], + batch, + ) + .await + .context("[slack_backfill] seal-probe ingest_chat failed")?; + println!( + "seal-probe done — chunks_written={} chunks_dropped={} chunk_ids={:?}", + result.chunks_written, result.chunks_dropped, result.chunk_ids + ); + return Ok(()); + } + + if cli.probe_search { + // Probe whether the workspace's Slack plan supports + // `search.messages` (paid plans only). One small query, print + // raw response, exit. Lets us decide whether to rebuild the + // provider around SEARCH_MESSAGES (1 paginated call workspace- + // wide) instead of per-channel `conversations.history` calls. + let now = chrono::Utc::now(); + let after = (now - chrono::Duration::days(7)) + .format("%Y-%m-%d") + .to_string(); + let args = serde_json::json!({ + "query": format!("after:{after}"), + "count": 5, + "sort": "timestamp", + "sort_dir": "desc", + }); + log::info!( + "[slack_backfill] probing SLACK_SEARCH_MESSAGES with query={}", + args["query"] + ); + let resp = client + .execute_tool("SLACK_SEARCH_MESSAGES", Some(args)) + .await + .map_err(|e| anyhow::anyhow!("SLACK_SEARCH_MESSAGES failed: {e:#}"))?; + println!("=== SLACK_SEARCH_MESSAGES probe ==="); + println!("successful: {}", resp.successful); + println!("error: {:?}", resp.error); + println!("cost_usd: {}", resp.cost_usd); + println!("data:"); + println!( + "{}", + serde_json::to_string_pretty(&resp.data).unwrap_or_default() + ); + return Ok(()); + } + + let connections = client + .list_connections() + .await + .map_err(|e| anyhow::anyhow!("list_connections failed: {e:#}"))?; + + if cli.use_search { + let mut slack_conns: Vec<_> = connections + .connections + .iter() + .filter(|c| { + c.toolkit.eq_ignore_ascii_case("slack") + && matches!(c.status.as_str(), "ACTIVE" | "CONNECTED") + }) + .cloned() + .collect(); + if let Some(ref wanted) = cli.connection_id { + slack_conns.retain(|c| &c.id == wanted); + } + if slack_conns.is_empty() { + bail!("no active Slack connection found"); + } + let started = Instant::now(); + let mut total_buckets = 0usize; + for conn in &slack_conns { + let ctx = ProviderContext { + config: Arc::clone(&config), + client: client.clone(), + toolkit: conn.toolkit.clone(), + connection_id: Some(conn.id.clone()), + }; + match run_backfill_via_search(&ctx, cli.days).await { + Ok(outcome) => { + total_buckets += outcome.items_ingested; + println!( + "connection={} buckets={} elapsed_ms={} summary={:?}", + conn.id, + outcome.items_ingested, + outcome.elapsed_ms(), + outcome.summary, + ); + } + Err(err) => { + eprintln!("connection={} search-backfill failed: {err:#}", conn.id); + } + } + } + println!( + "slack-backfill (search) done in {:.1}s — total_buckets={}", + started.elapsed().as_secs_f64(), + total_buckets + ); + return Ok(()); + } + + let mut candidates: Vec<_> = connections + .connections + .into_iter() + .filter(|c| { + c.toolkit.eq_ignore_ascii_case("slack") + && matches!(c.status.as_str(), "ACTIVE" | "CONNECTED") + }) + .collect(); + + if let Some(ref wanted) = cli.connection_id { + candidates.retain(|c| &c.id == wanted); + if candidates.is_empty() { + bail!("no active Slack connection found with id={wanted}"); + } + } + + if candidates.is_empty() { + bail!( + "no active Slack connections in Composio. \ + Connect Slack from the desktop app's Integrations screen first." + ); + } + + log::info!( + "[slack_backfill] workspace={} connections={} embedder={} extractor={} summariser={}", + config.workspace_dir.display(), + candidates.len(), + component_status( + &config.memory_tree.embedding_endpoint, + &config.memory_tree.embedding_model, + ), + component_status( + &config.memory_tree.llm_extractor_endpoint, + &config.memory_tree.llm_extractor_model, + ), + component_status( + &config.memory_tree.llm_summariser_endpoint, + &config.memory_tree.llm_summariser_model, + ), + ); + + let started = Instant::now(); + let mut total_buckets: usize = 0; + let mut connections_ok: usize = 0; + + for conn in &candidates { + if cli.reset_state { + let key = format!("slack:{}", conn.id); + match memory::global::client_if_ready() { + Some(mem) => match mem.kv_delete(Some("composio-sync-state"), &key).await { + Ok(true) => log::info!( + "[slack_backfill] reset SyncState for connection={} (cleared cursors)", + conn.id + ), + Ok(false) => log::info!( + "[slack_backfill] no SyncState to reset for connection={}", + conn.id + ), + Err(e) => log::warn!( + "[slack_backfill] reset SyncState failed for connection={}: {e:#}", + conn.id + ), + }, + None => { + log::warn!("[slack_backfill] memory client not ready; skipping --reset-state") + } + } + } + let ctx = ProviderContext { + config: Arc::clone(&config), + client: client.clone(), + toolkit: conn.toolkit.clone(), + connection_id: Some(conn.id.clone()), + }; + match provider.sync(&ctx, SyncReason::Manual).await { + Ok(outcome) => { + connections_ok += 1; + total_buckets += outcome.items_ingested; + println!( + "connection={} buckets_flushed={} elapsed_ms={} summary={:?}", + conn.id, + outcome.items_ingested, + outcome.elapsed_ms(), + outcome.summary, + ); + } + Err(err) => { + eprintln!("connection={} sync failed: {err:#}", conn.id); + } + } + } + + println!( + "slack_backfill done in {:.1}s — connections_ok={}/{} total_buckets_flushed={}", + started.elapsed().as_secs_f64(), + connections_ok, + candidates.len(), + total_buckets, + ); + Ok(()) +} + +fn component_status(endpoint: &Option, model: &Option) -> String { + match (endpoint.as_deref(), model.as_deref()) { + (Some(e), Some(m)) if !e.trim().is_empty() && !m.trim().is_empty() => { + format!("on/{}", m.trim()) + } + _ => "off".to_string(), + } +} diff --git a/src/core/all.rs b/src/core/all.rs index c3be41260..c41a5f9ec 100644 --- a/src/core/all.rs +++ b/src/core/all.rs @@ -125,6 +125,8 @@ fn build_registered_controllers() -> Vec { controllers.extend(crate::openhuman::memory::all_memory_tree_registered_controllers()); // Memory tree retrieval layer (#710 — LLM-callable read tools over the tree) controllers.extend(crate::openhuman::memory::all_retrieval_registered_controllers()); + // Slack → memory-tree ingestion engine (backfill + poll + 6hr bucket flush) + controllers.extend(crate::openhuman::memory::all_slack_ingestion_registered_controllers()); // Link shortener for long tracking URLs — saves LLM tokens controllers .extend(crate::openhuman::redirect_links::all_redirect_links_registered_controllers()); @@ -200,6 +202,7 @@ fn build_declared_controller_schemas() -> Vec { schemas.extend(crate::openhuman::memory::all_memory_controller_schemas()); schemas.extend(crate::openhuman::memory::all_memory_tree_controller_schemas()); schemas.extend(crate::openhuman::memory::all_retrieval_controller_schemas()); + schemas.extend(crate::openhuman::memory::all_slack_ingestion_controller_schemas()); schemas.extend(crate::openhuman::redirect_links::all_redirect_links_controller_schemas()); schemas.extend(crate::openhuman::referral::all_referral_controller_schemas()); schemas.extend(crate::openhuman::billing::all_billing_controller_schemas()); diff --git a/src/openhuman/about_app/catalog.rs b/src/openhuman/about_app/catalog.rs index 088bab6ca..9034ff00e 100644 --- a/src/openhuman/about_app/catalog.rs +++ b/src/openhuman/about_app/catalog.rs @@ -186,6 +186,16 @@ const CAPABILITIES: &[Capability] = &[ status: CapabilityStatus::Beta, privacy: None, }, + Capability { + id: "intelligence.slack_memory_ingest", + name: "Slack Memory Ingestion", + domain: "intelligence", + category: CapabilityCategory::Intelligence, + description: "Backfill the last 6 days of Slack history into the memory tree and keep it up to date by flushing each closed 6-hour UTC bucket. Driven by an authenticated Slack connection (OAuth via Composio).", + how_to: "Settings > Messaging Channels > Slack", + status: CapabilityStatus::Beta, + privacy: LOCAL_RAW, + }, Capability { id: "intelligence.notifications_dismiss", name: "Dismiss Notifications", diff --git a/src/openhuman/channels/runtime/startup.rs b/src/openhuman/channels/runtime/startup.rs index 310a4af7f..e4c62b151 100644 --- a/src/openhuman/channels/runtime/startup.rs +++ b/src/openhuman/channels/runtime/startup.rs @@ -270,6 +270,10 @@ pub async fn start_channels(config: Config) -> Result<()> { sl.channel_id.clone(), sl.allowed_users.clone(), ))); + // Memory-tree ingestion is handled by the Composio-backed + // `SlackProvider`, which runs inside `composio::periodic` and + // fires per-connection on its own 15-minute cadence. No spawn + // required here. } if let Some(ref mm) = config.channels_config.mattermost { diff --git a/src/openhuman/composio/providers/mod.rs b/src/openhuman/composio/providers/mod.rs index 445602a1f..de9ec1b57 100644 --- a/src/openhuman/composio/providers/mod.rs +++ b/src/openhuman/composio/providers/mod.rs @@ -45,6 +45,7 @@ pub mod gmail; pub mod notion; pub mod profile; pub mod registry; +pub mod slack; pub mod sync_state; /// Static toolkit → curated catalog map. diff --git a/src/openhuman/composio/providers/registry.rs b/src/openhuman/composio/providers/registry.rs index ba1856129..689edc823 100644 --- a/src/openhuman/composio/providers/registry.rs +++ b/src/openhuman/composio/providers/registry.rs @@ -80,6 +80,7 @@ pub fn all_providers() -> Vec { pub fn init_default_providers() { register_provider(Arc::new(super::gmail::GmailProvider::new())); register_provider(Arc::new(super::notion::NotionProvider::new())); + register_provider(Arc::new(super::slack::SlackProvider::new())); tracing::info!( count = all_providers().len(), "[composio:registry] default providers initialised" diff --git a/src/openhuman/composio/providers/slack/mod.rs b/src/openhuman/composio/providers/slack/mod.rs new file mode 100644 index 000000000..4dd34414a --- /dev/null +++ b/src/openhuman/composio/providers/slack/mod.rs @@ -0,0 +1,16 @@ +//! Composio-backed Slack provider. +//! +//! The provider is wired into the periodic-sync scheduler (see +//! [`super::registry::init_default_providers`]) and fires +//! `SLACK_LIST_CONVERSATIONS` + `SLACK_FETCH_CONVERSATION_HISTORY` +//! against the user's Composio-authorized Slack connection. Messages +//! are grouped into 6-hour UTC buckets by +//! [`crate::openhuman::memory::slack_ingestion::bucketer`] and ingested +//! into the memory tree via +//! [`crate::openhuman::memory::slack_ingestion::ops::ingest_bucket`]. + +mod provider; +mod sync; +mod users; + +pub use provider::{run_backfill_via_search, SlackProvider, BACKFILL_DAYS}; diff --git a/src/openhuman/composio/providers/slack/provider.rs b/src/openhuman/composio/providers/slack/provider.rs new file mode 100644 index 000000000..bb6bff25f --- /dev/null +++ b/src/openhuman/composio/providers/slack/provider.rs @@ -0,0 +1,920 @@ +//! Composio-backed Slack provider. +//! +//! Drives Slack history ingestion **without** a user-managed bot token +//! — authorization lives in the user's Composio Slack connection, and +//! the actual API calls fan out through [`ComposioClient::execute_tool`] +//! against Composio's action catalog (`SLACK_LIST_CONVERSATIONS`, +//! `SLACK_FETCH_CONVERSATION_HISTORY`, `SLACK_FETCH_TEAM_INFO`, …). +//! +//! ## Per-sync lifecycle +//! +//! 1. Load [`SyncState`] for `(slack, connection_id)`. `state.cursor` is +//! a JSON-encoded [`sync::ChannelCursors`] map — Slack needs a cursor +//! per channel, not one global watermark (Gmail's single `cursor` +//! string wouldn't cut it). Parse failures degrade to an empty map, +//! i.e. full backfill, which is safe because chunk IDs are +//! deterministic. +//! 2. Enumerate every channel the bot can read via +//! [`ACTION_LIST_CONVERSATIONS`] with pagination. +//! 3. For each channel, pull messages since the per-channel cursor (or +//! `now - BACKFILL_DAYS` if no cursor yet) via +//! [`ACTION_FETCH_HISTORY`], paginated. +//! 4. Hand every collected message to +//! [`slack_ingestion::bucketer::split_closed`] — produces closed +//! 6-hour UTC buckets + a "still open" remainder (discarded; the +//! next sync will re-fetch that window, which is cheap because we +//! advance the cursor **only** after a bucket flushes). +//! 5. Ingest each closed bucket via +//! [`slack_ingestion::ops::ingest_bucket`] — canonicalise → chunk → +//! score (with the config's Ollama entity extractor, if set) → +//! persist → seal cascade (with the config's Ollama summariser). +//! 6. Advance per-channel cursor to the latest flushed bucket's end +//! timestamp; save [`SyncState`]. +//! +//! ## Idempotency +//! +//! - `source_id = "slack::"` is stable +//! across runs, so chunk IDs are deterministic and re-ingest is an +//! UPSERT — no duplicates. +//! - The cursor advances **only** after a bucket's `ingest_bucket` +//! call returns `Ok`. A crash mid-fetch means the next run re-walks +//! that range; a crash mid-ingest re-fetches the range too. Both are +//! safe by the chunk-id property above. + +use async_trait::async_trait; +use serde_json::{json, Value}; +use std::collections::{BTreeMap, HashMap}; +use std::path::PathBuf; +use std::time::Duration; + +use super::sync; +use super::users::SlackUsers; +use crate::openhuman::composio::client::ComposioClient; +use crate::openhuman::composio::providers::sync_state::SyncState; +use crate::openhuman::composio::providers::{ + pick_str, ComposioProvider, CuratedTool, ProviderContext, ProviderUserProfile, SyncOutcome, + SyncReason, +}; +use crate::openhuman::composio::types::ComposioExecuteResponse; +use crate::openhuman::memory::slack_ingestion::bucketer::{ + bucket_end_for, split_closed, GRACE_PERIOD, +}; +use crate::openhuman::memory::slack_ingestion::ops::ingest_bucket; +use crate::openhuman::memory::slack_ingestion::types::{SlackChannel, SlackMessage}; + +/// Composio action slug for channel listing. +const ACTION_LIST_CONVERSATIONS: &str = "SLACK_LIST_CONVERSATIONS"; +/// Composio action slug for message history. +const ACTION_FETCH_HISTORY: &str = "SLACK_FETCH_CONVERSATION_HISTORY"; +/// Composio action slug for team/workspace profile fetch. +const ACTION_FETCH_TEAM_INFO: &str = "SLACK_FETCH_TEAM_INFO"; + +/// Default backfill window (days) applied when a channel has no +/// cursor yet. Override via `OPENHUMAN_SLACK_BACKFILL_DAYS` env var +/// when a different window is needed (e.g. 30 days for a fresh +/// workspace, or 1 day for fast smoke tests). +pub const BACKFILL_DAYS: i64 = 6; + +/// Resolve the active backfill window in days. Reads +/// `OPENHUMAN_SLACK_BACKFILL_DAYS` env var if set and parseable as a +/// positive integer; falls back to [`BACKFILL_DAYS`] otherwise. +fn backfill_days() -> i64 { + match std::env::var("OPENHUMAN_SLACK_BACKFILL_DAYS") { + Ok(s) => match s.trim().parse::() { + Ok(n) if n >= 1 => n, + _ => { + log::warn!( + "[composio:slack] OPENHUMAN_SLACK_BACKFILL_DAYS={s:?} not a positive integer; \ + falling back to default {BACKFILL_DAYS}" + ); + BACKFILL_DAYS + } + }, + Err(_) => BACKFILL_DAYS, + } +} + +/// Max channels listed per `SLACK_LIST_CONVERSATIONS` page. Slack caps +/// this at 1000; 200 is a safe default. +const LIST_PAGE_SIZE: u32 = 200; + +/// Max messages per `SLACK_FETCH_CONVERSATION_HISTORY` page. With +/// `INTER_CALL_PACING` clamping us to 3 req/min, the marginal cost of +/// asking for 1000 vs 200 per call is just larger response payloads — +/// and we want to drain the 30-day window in as few calls as possible +/// to minimise total quota burn. +const HISTORY_PAGE_SIZE: u32 = 1000; + +/// Stop paginating any single channel's history after this many pages +/// so one dormant backfill can't consume the whole daily budget. With +/// `HISTORY_PAGE_SIZE=200`, this yields ≤ 4000 messages per channel per +/// sync — the next tick picks up the rest incrementally. +const MAX_HISTORY_PAGES_PER_CHANNEL: u32 = 20; + +/// Stop paginating channel listings after this many pages. +const MAX_LIST_PAGES: u32 = 10; + +/// Sync cadence — matches Gmail (15 minutes). Bucket flush granularity +/// is 6 hours anyway, so tighter cadences just burn API calls. +const SYNC_INTERVAL_SECS: u64 = 15 * 60; + +/// Initial backoff for rate-limit retries. Slack tier-2 endpoints +/// (`conversations.history`, `users.list`) advertise ~50 req/min but +/// Composio's quota appears stricter; 2s gives the bucket time to +/// refill on a `ratelimited` response. +const RATELIMIT_INITIAL_BACKOFF: Duration = Duration::from_secs(2); + +/// Cap on per-retry backoff. Without a ceiling, exponential growth +/// would push individual retries into the multi-minute range and stall +/// the whole sync. +const RATELIMIT_MAX_BACKOFF: Duration = Duration::from_secs(30); + +/// Total retries for a single rate-limited call before giving up. With +/// 2s → 4 → 8 → 16 → 30 → 30 backoff this gives ≤ 90s of grace per +/// call; longer than the worst Composio rate-window we've observed. +const RATELIMIT_MAX_ATTEMPTS: u32 = 6; + +/// Fixed inter-call sleep applied after every successful execute_tool. +/// At 20s per call we stay at 3 req/min — well below Slack's tier-2 +/// limit of 50/min and conservative enough to ride out Composio's +/// stricter staging-tier quota replenishment. Trade-off: a full +/// 30-day backfill across 11 channels takes 20-30 min wall-clock. +const INTER_CALL_PACING: Duration = Duration::from_secs(20); + +/// Resolve the JSON dump directory from `OPENHUMAN_SLACK_DUMP_DIR`. +/// When unset, dumping is disabled. When set, every successful Composio +/// response is mirrored to `//-.json` so the +/// raw payload can be replayed into `ingest_chat` later without +/// re-burning quota. +fn dump_dir() -> Option { + std::env::var_os("OPENHUMAN_SLACK_DUMP_DIR").map(PathBuf::from) +} + +/// Write a Composio response payload to disk under the dump dir. Best +/// effort — failures are logged at warn level and never fail the sync. +pub(super) fn dump_response(scope: &str, kind: &str, idx: u32, data: &Value) { + let Some(base) = dump_dir() else { + return; + }; + let path = base.join(scope).join(format!("{kind}-{idx:04}.json")); + if let Some(parent) = path.parent() { + if let Err(e) = std::fs::create_dir_all(parent) { + tracing::warn!( + error = %e, + path = %parent.display(), + "[composio:slack] dump_response: create_dir_all failed (skipping dump)" + ); + return; + } + } + match serde_json::to_string_pretty(data) { + Ok(json) => { + if let Err(e) = std::fs::write(&path, json) { + tracing::warn!( + error = %e, + path = %path.display(), + "[composio:slack] dump_response: write failed" + ); + } else { + tracing::debug!(path = %path.display(), "[composio:slack] dumped response"); + } + } + Err(e) => { + tracing::warn!(error = %e, "[composio:slack] dump_response: serialize failed"); + } + } +} + +/// Wrap [`ComposioClient::execute_tool`] with rate-limit-aware retry + +/// inter-call pacing. +/// +/// Returns `(response, attempts_made)` on first success so callers can +/// charge the daily quota meter for **every** attempt that hit Composio, +/// not just the one that succeeded — under throttling each retry is a +/// real billable call. The retry only fires for `ok=false, +/// error=ratelimited` responses; other failures pass through unchanged. +pub(super) async fn execute_with_retry( + client: &ComposioClient, + slug: &str, + args: serde_json::Value, + description: &str, +) -> Result<(ComposioExecuteResponse, u32), String> { + let mut delay = RATELIMIT_INITIAL_BACKOFF; + for attempt in 1..=RATELIMIT_MAX_ATTEMPTS { + let resp = client + .execute_tool(slug, Some(args.clone())) + .await + .map_err(|e| format!("{description}: {e:#}"))?; + if resp.successful { + tokio::time::sleep(INTER_CALL_PACING).await; + return Ok((resp, attempt)); + } + let err_str = resp.error.as_deref().unwrap_or("provider failure"); + let is_ratelimit = err_str.contains("ratelimited") + || err_str.contains("rate_limit") + || err_str.contains("rate limit"); + if is_ratelimit && attempt < RATELIMIT_MAX_ATTEMPTS { + tracing::warn!( + slug, + attempt, + max_attempts = RATELIMIT_MAX_ATTEMPTS, + sleep_ms = delay.as_millis() as u64, + "[composio:slack] rate-limited; backing off and retrying" + ); + tokio::time::sleep(delay).await; + delay = (delay * 2).min(RATELIMIT_MAX_BACKOFF); + continue; + } + return Err(format!("{description}: {err_str}")); + } + Err(format!( + "{description}: rate-limited after {RATELIMIT_MAX_ATTEMPTS} retries" + )) +} + +pub struct SlackProvider; + +impl SlackProvider { + pub fn new() -> Self { + Self + } +} + +impl Default for SlackProvider { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl ComposioProvider for SlackProvider { + fn toolkit_slug(&self) -> &'static str { + "slack" + } + + fn curated_tools(&self) -> Option<&'static [CuratedTool]> { + Some(crate::openhuman::composio::providers::catalogs::SLACK_CURATED) + } + + fn sync_interval_secs(&self) -> Option { + Some(SYNC_INTERVAL_SECS) + } + + async fn fetch_user_profile( + &self, + ctx: &ProviderContext, + ) -> Result { + tracing::debug!( + connection_id = ?ctx.connection_id, + "[composio:slack] fetch_user_profile via {ACTION_FETCH_TEAM_INFO}" + ); + + let resp = ctx + .client + .execute_tool(ACTION_FETCH_TEAM_INFO, Some(json!({}))) + .await + .map_err(|e| format!("[composio:slack] {ACTION_FETCH_TEAM_INFO} failed: {e:#}"))?; + + if !resp.successful { + let err = resp + .error + .clone() + .unwrap_or_else(|| "provider reported failure".to_string()); + return Err(format!("[composio:slack] {ACTION_FETCH_TEAM_INFO}: {err}")); + } + + let data = &resp.data; + let display_name = pick_str(data, &["data.team.name", "data.name", "team.name", "name"]); + let profile_url = pick_str(data, &["data.team.url", "data.url", "team.url", "url"]); + let email_domain = pick_str( + data, + &[ + "data.team.email_domain", + "data.email_domain", + "team.email_domain", + "email_domain", + ], + ); + let avatar_url = pick_str( + data, + &[ + "data.team.icon.image_132", + "data.team.icon.image_68", + "team.icon.image_132", + ], + ); + + let profile = ProviderUserProfile { + toolkit: "slack".to_string(), + connection_id: ctx.connection_id.clone(), + display_name, + email: None, // Slack team_info is workspace-scoped, not user-scoped + username: None, + avatar_url, + profile_url, + extras: json!({ "email_domain": email_domain, "raw": data }), + }; + + tracing::info!( + connection_id = ?profile.connection_id, + display_name = ?profile.display_name, + "[composio:slack] fetched team info" + ); + Ok(profile) + } + + async fn sync(&self, ctx: &ProviderContext, reason: SyncReason) -> Result { + let started_at_ms = sync::now_ms(); + let connection_id = ctx + .connection_id + .clone() + .unwrap_or_else(|| "default".to_string()); + + tracing::info!( + connection_id = %connection_id, + reason = reason.as_str(), + "[composio:slack] sync starting" + ); + + let Some(memory) = ctx.memory_client() else { + return Err("[composio:slack] memory client not ready".to_string()); + }; + let mut state = SyncState::load(&memory, "slack", &connection_id).await?; + + if state.budget_exhausted() { + tracing::info!( + connection_id = %connection_id, + "[composio:slack] daily request budget exhausted, skipping sync" + ); + return Ok(SyncOutcome { + toolkit: "slack".to_string(), + connection_id: Some(connection_id), + reason: reason.as_str().to_string(), + items_ingested: 0, + started_at_ms, + finished_at_ms: sync::now_ms(), + summary: "slack sync skipped: daily budget exhausted".to_string(), + details: json!({ "budget_exhausted": true }), + }); + } + + let mut cursors = sync::decode_cursors(state.cursor.as_deref()); + let now = chrono::Utc::now(); + + // Pull the workspace user directory once per sync so author + // ids and `<@…>` mentions in message text canonicalise to + // human-readable names. Soft-fails to an empty cache; raw ids + // simply pass through in that case. + let (users, user_call_count) = SlackUsers::fetch(&ctx.client).await; + state.record_requests(user_call_count); + tracing::info!( + connection_id = %connection_id, + user_count = users.len(), + "[composio:slack] users cached for this sync" + ); + + // 1. Enumerate channels ──────────────────────────────────────── + let channels = list_all_channels(ctx, &mut state) + .await + .map_err(|e| format!("[composio:slack] list_channels: {e:#}"))?; + + tracing::info!( + connection_id = %connection_id, + channel_count = channels.len(), + "[composio:slack] channels discovered" + ); + + // Save budget state early so a panic mid-fetch doesn't leak the + // request counter. Cheap. + let _ = state.save(&memory).await; + + let mut total_flushed_buckets: usize = 0; + let mut channels_processed: usize = 0; + let mut channels_errored: usize = 0; + + // 2. Per-channel: fetch → bucket → ingest → advance cursor ──── + for channel in &channels { + if state.budget_exhausted() { + tracing::warn!( + connection_id = %connection_id, + channel = %channel.id, + "[composio:slack] budget exhausted mid-sync, remaining channels deferred" + ); + break; + } + + match process_channel(ctx, &mut state, channel, &mut cursors, now, &users).await { + Ok(n) => { + total_flushed_buckets += n; + channels_processed += 1; + } + Err(err) => { + channels_errored += 1; + tracing::warn!( + connection_id = %connection_id, + channel = %channel.id, + error = %err, + "[composio:slack] channel sync failed (continuing with next channel)" + ); + } + } + + // Save after every channel — a crash between channels + // shouldn't lose already-advanced cursors. + state.advance_cursor(sync::encode_cursors(&cursors)); + if let Err(err) = state.save(&memory).await { + tracing::warn!( + error = %err, + "[composio:slack] state save failed after channel (non-fatal)" + ); + } + } + + let finished_at_ms = sync::now_ms(); + let summary = format!( + "slack sync: channels_processed={channels_processed} \ + channels_errored={channels_errored} \ + buckets_flushed={total_flushed_buckets}" + ); + tracing::info!( + connection_id = %connection_id, + elapsed_ms = finished_at_ms.saturating_sub(started_at_ms), + "{summary}" + ); + + Ok(SyncOutcome { + toolkit: "slack".to_string(), + connection_id: Some(connection_id), + reason: reason.as_str().to_string(), + items_ingested: total_flushed_buckets, + started_at_ms, + finished_at_ms, + summary, + details: json!({ + "channels_processed": channels_processed, + "channels_errored": channels_errored, + }), + }) + } + + async fn on_trigger( + &self, + ctx: &ProviderContext, + trigger: &str, + _payload: &Value, + ) -> Result<(), String> { + // Slack trigger names use SLACK_RECEIVE_MESSAGE / similar — + // match loosely so future slug changes don't silently drop this. + if trigger.to_ascii_uppercase().contains("MESSAGE") { + if let Err(e) = self.sync(ctx, SyncReason::Manual).await { + tracing::warn!( + error = %e, + "[composio:slack] trigger-driven sync failed (non-fatal)" + ); + } + } + Ok(()) + } +} + +/// Paginate through `SLACK_LIST_CONVERSATIONS` and flatten into a +/// single `Vec`. Aborts early if the daily budget runs out +/// partway through — callers handle the partial list gracefully. +async fn list_all_channels( + ctx: &ProviderContext, + state: &mut SyncState, +) -> Result, String> { + let mut out: Vec = Vec::new(); + let mut cursor: Option = None; + + for page_num in 0..MAX_LIST_PAGES { + if state.budget_exhausted() { + tracing::warn!( + page = page_num, + "[composio:slack] budget exhausted during channel listing" + ); + break; + } + let mut args = json!({ + "types": "public_channel,private_channel", + "exclude_archived": true, + "limit": LIST_PAGE_SIZE, + }); + if let Some(ref c) = cursor { + args["cursor"] = json!(c); + } + + let (resp, attempts) = execute_with_retry( + &ctx.client, + ACTION_LIST_CONVERSATIONS, + args, + &format!("{ACTION_LIST_CONVERSATIONS} page {page_num}"), + ) + .await?; + state.record_requests(attempts); + dump_response("_meta", "channels", page_num, &resp.data); + + out.extend(sync::extract_channels(&resp.data)); + cursor = sync::extract_next_cursor(&resp.data); + if cursor.is_none() { + break; + } + } + Ok(out) +} + +/// Pull one channel's history since its cursor, bucket it, and ingest +/// every closed bucket. Returns the number of buckets actually flushed. +async fn process_channel( + ctx: &ProviderContext, + state: &mut SyncState, + channel: &SlackChannel, + cursors: &mut sync::ChannelCursors, + now: chrono::DateTime, + users: &SlackUsers, +) -> Result { + // Derive `oldest` — per-channel cursor if we've synced before, + // else `now - backfill_days()` for a fresh channel. + let oldest_secs = cursors + .get(&channel.id) + .copied() + .unwrap_or_else(|| (now - chrono::Duration::days(backfill_days())).timestamp()); + + let mut all_messages: Vec = Vec::new(); + let mut cursor: Option = None; + + for page_num in 0..MAX_HISTORY_PAGES_PER_CHANNEL { + if state.budget_exhausted() { + tracing::warn!( + channel = %channel.id, + page = page_num, + "[composio:slack] budget exhausted during history fetch" + ); + break; + } + + let mut args = json!({ + "channel": channel.id, + "oldest": format!("{oldest_secs}.000000"), + "inclusive": false, + "limit": HISTORY_PAGE_SIZE, + }); + if let Some(ref c) = cursor { + args["cursor"] = json!(c); + } + + let (resp, attempts) = execute_with_retry( + &ctx.client, + ACTION_FETCH_HISTORY, + args, + &format!( + "{ACTION_FETCH_HISTORY} channel={} page {page_num}", + channel.id + ), + ) + .await?; + state.record_requests(attempts); + dump_response(&channel.id, "history", page_num, &resp.data); + + let msgs = sync::extract_messages(&resp.data, &channel.id, users); + tracing::debug!( + channel = %channel.id, + page = page_num, + fetched = msgs.len(), + "[composio:slack] history page" + ); + if msgs.is_empty() { + break; + } + all_messages.extend(msgs); + cursor = sync::extract_next_cursor(&resp.data); + if cursor.is_none() { + break; + } + } + + if all_messages.is_empty() { + tracing::debug!( + channel = %channel.id, + "[composio:slack] no new messages" + ); + return Ok(0); + } + + // Bucket — closed buckets ingest now; still-open ones get discarded + // and will be re-fetched on the next tick (cursor only advances on + // successful flush). + let (closed, remaining) = split_closed(all_messages, now, GRACE_PERIOD); + tracing::debug!( + channel = %channel.id, + closed_buckets = closed.len(), + remaining_msgs = remaining.len(), + "[composio:slack] bucket split" + ); + + let mut flushed = 0usize; + let mut latest_end: Option> = None; + let connection_id = ctx.connection_id.as_deref().unwrap_or("default"); + + for bucket in closed { + match ingest_bucket(&ctx.config, channel, &bucket, "", connection_id).await { + Ok(res) => { + flushed += 1; + latest_end = Some(bucket.end); + tracing::info!( + channel = %channel.id, + bucket_start = %bucket.start.to_rfc3339(), + chunks_written = res.chunks_written, + "[composio:slack] ingested bucket" + ); + } + Err(err) => { + tracing::warn!( + channel = %channel.id, + bucket_start = %bucket.start.to_rfc3339(), + error = %err, + "[composio:slack] ingest failed (cursor not advanced for this bucket)" + ); + // Stop processing more buckets in this channel — the + // next run will re-fetch from the current cursor. + break; + } + } + } + + if let Some(end) = latest_end { + cursors.insert(channel.id.clone(), end.timestamp()); + } + // Ensure bucket_end_for stays referenced (future-proofing for + // cascade-aware flushes) without dead-code. + let _ = bucket_end_for; + + Ok(flushed) +} + +// ── Search-based backfill (one-shot) ──────────────────────────────── + +/// Composio action slug for workspace-wide message search. +const ACTION_SEARCH_MESSAGES: &str = "SLACK_SEARCH_MESSAGES"; + +/// Max matches per `SLACK_SEARCH_MESSAGES` page (Slack's documented cap). +const SEARCH_PAGE_SIZE: u32 = 100; + +/// Hard cap on pages walked per backfill run. With 100 matches/page +/// that's 5000 messages — plenty for typical 30-day windows on small +/// workspaces. Larger backfills can run multiple times against +/// successive sub-windows. +const MAX_SEARCH_PAGES: u32 = 50; + +/// Run a one-shot historical backfill via `SLACK_SEARCH_MESSAGES` — +/// workspace-wide paginated search instead of per-channel +/// `conversations.history`. Better quota efficiency: each successful +/// call returns matches across many channels at once, so partial +/// progress translates to real coverage instead of one channel's +/// worth. +/// +/// Designed for the `slack-backfill` bin specifically — the periodic +/// `SlackProvider::sync()` keeps the per-channel incremental path so +/// it stays cheap on each tick. +/// +/// Lifecycle: +/// 1. Cache the channel directory (one `SLACK_LIST_CONVERSATIONS` call, +/// paginated) so canonicalisation can label channels by name + know +/// private-vs-public. +/// 2. Cache the user directory (one `SLACK_LIST_ALL_USERS` paginated +/// walk via [`SlackUsers::fetch`]). +/// 3. Paginate `SLACK_SEARCH_MESSAGES` with +/// `query = "after:"` until exhausted or page cap. +/// 4. Group every message by `(channel_id, 6hr_bucket_start)`. +/// 5. For each closed bucket, hand off to `ingest_bucket` (same +/// canonicalise → chunk → score → seal-cascade path the periodic +/// sync uses). +pub async fn run_backfill_via_search( + ctx: &ProviderContext, + backfill_days: i64, +) -> Result { + let started_at_ms = sync::now_ms(); + let connection_id = ctx + .connection_id + .clone() + .unwrap_or_else(|| "default".to_string()); + + tracing::info!( + connection_id = %connection_id, + backfill_days, + "[composio:slack] search-based backfill starting" + ); + + let memory = ctx + .memory_client() + .ok_or_else(|| "[composio:slack] memory client not ready".to_string())?; + let mut state = SyncState::load(&memory, "slack", &connection_id).await?; + + if state.budget_exhausted() { + return Ok(SyncOutcome { + toolkit: "slack".to_string(), + connection_id: Some(connection_id), + reason: SyncReason::Manual.as_str().to_string(), + items_ingested: 0, + started_at_ms, + finished_at_ms: sync::now_ms(), + summary: "slack search-backfill skipped: daily budget exhausted".to_string(), + details: json!({ "budget_exhausted": true }), + }); + } + + // 1. Channel directory — needed for canonicalisation (name + + // is_private flag come off `SlackChannel`). + let channels = list_all_channels(ctx, &mut state) + .await + .map_err(|e| format!("[composio:slack] list_channels: {e:#}"))?; + let channel_map: HashMap = + channels.into_iter().map(|c| (c.id.clone(), c)).collect(); + + // 2. User directory — for ID → display-name resolution + mention + // rewrites. + let (users, user_call_count) = SlackUsers::fetch(&ctx.client).await; + state.record_requests(user_call_count); + tracing::info!( + connection_id = %connection_id, + user_count = users.len(), + channel_count = channel_map.len(), + "[composio:slack] caches ready" + ); + let _ = state.save(&memory).await; + + // 3. Paginated workspace-wide search. + let now = chrono::Utc::now(); + let after = (now - chrono::Duration::days(backfill_days)) + .format("%Y-%m-%d") + .to_string(); + let query = format!("after:{after}"); + let mut all_messages: Vec = Vec::new(); + let mut page: u32 = 1; + let mut total_pages: u32 = 1; + + loop { + if state.budget_exhausted() { + tracing::warn!( + page, + "[composio:slack] budget exhausted mid-search, halting" + ); + break; + } + let args = json!({ + "query": query, + "count": SEARCH_PAGE_SIZE, + "sort": "timestamp", + "sort_dir": "asc", + "page": page, + }); + let (resp, attempts) = execute_with_retry( + &ctx.client, + ACTION_SEARCH_MESSAGES, + args, + &format!("{ACTION_SEARCH_MESSAGES} page {page}"), + ) + .await?; + state.record_requests(attempts); + dump_response("_meta", "search", page, &resp.data); + + let msgs = sync::extract_search_messages(&resp.data, &users); + if page == 1 { + total_pages = sync::extract_search_total_pages(&resp.data).min(MAX_SEARCH_PAGES); + tracing::info!( + connection_id = %connection_id, + total_pages, + first_page_msgs = msgs.len(), + "[composio:slack] search pagination plan" + ); + } + let fetched = msgs.len(); + all_messages.extend(msgs); + if fetched == 0 || page >= total_pages { + break; + } + page += 1; + } + let _ = state.save(&memory).await; + + // 4. Group messages by (channel, 6hr-bucket-start). + let mut by_bucket: BTreeMap<(String, chrono::DateTime), Vec> = + BTreeMap::new(); + for msg in all_messages { + let bucket_start = + crate::openhuman::memory::slack_ingestion::bucketer::bucket_start_for(msg.timestamp); + by_bucket + .entry((msg.channel_id.clone(), bucket_start)) + .or_default() + .push(msg); + } + + tracing::info!( + connection_id = %connection_id, + total_buckets = by_bucket.len(), + "[composio:slack] grouped messages into buckets" + ); + + // 5. Ingest each closed bucket. + let mut buckets_flushed = 0usize; + let mut buckets_skipped_open = 0usize; + let mut buckets_skipped_unknown_channel = 0usize; + let mut buckets_failed = 0usize; + + for ((channel_id, bucket_start), mut msgs) in by_bucket { + let Some(channel) = channel_map.get(&channel_id) else { + buckets_skipped_unknown_channel += 1; + continue; + }; + msgs.sort_by_key(|m| m.timestamp); + let bucket = crate::openhuman::memory::slack_ingestion::types::Bucket { + start: bucket_start, + end: bucket_end_for(bucket_start), + messages: msgs, + }; + // Skip still-open buckets — same closed-bucket invariant the + // periodic sync enforces. + if bucket.end + crate::openhuman::memory::slack_ingestion::bucketer::GRACE_PERIOD > now { + buckets_skipped_open += 1; + continue; + } + match ingest_bucket(&ctx.config, channel, &bucket, "", &connection_id).await { + Ok(res) => { + buckets_flushed += 1; + tracing::info!( + channel = %channel.id, + bucket_start = %bucket.start.to_rfc3339(), + messages = bucket.messages.len(), + chunks_written = res.chunks_written, + "[composio:slack] ingested bucket" + ); + } + Err(err) => { + buckets_failed += 1; + tracing::warn!( + channel = %channel.id, + bucket_start = %bucket.start.to_rfc3339(), + error = %err, + "[composio:slack] ingest_bucket failed" + ); + } + } + } + + let finished_at_ms = sync::now_ms(); + let summary = format!( + "slack search-backfill: pages={page} buckets_flushed={buckets_flushed} \ + buckets_open={buckets_skipped_open} \ + unknown_channel={buckets_skipped_unknown_channel} \ + failed={buckets_failed}" + ); + tracing::info!( + connection_id = %connection_id, + elapsed_ms = finished_at_ms.saturating_sub(started_at_ms), + "{summary}" + ); + + Ok(SyncOutcome { + toolkit: "slack".to_string(), + connection_id: Some(connection_id), + reason: SyncReason::Manual.as_str().to_string(), + items_ingested: buckets_flushed, + started_at_ms, + finished_at_ms, + summary, + details: json!({ + "pages_walked": page, + "buckets_flushed": buckets_flushed, + "buckets_open": buckets_skipped_open, + "buckets_unknown_channel": buckets_skipped_unknown_channel, + "buckets_failed": buckets_failed, + }), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn toolkit_slug_is_stable() { + assert_eq!(SlackProvider::new().toolkit_slug(), "slack"); + } + + #[test] + fn sync_interval_matches_constant() { + assert_eq!( + SlackProvider::new().sync_interval_secs(), + Some(SYNC_INTERVAL_SECS) + ); + } + + #[test] + fn curated_tools_returns_slack_catalog() { + let tools = SlackProvider::new().curated_tools().unwrap(); + assert!(tools + .iter() + .any(|t| t.slug == "SLACK_FETCH_CONVERSATION_HISTORY")); + assert!(tools.iter().any(|t| t.slug == "SLACK_LIST_CONVERSATIONS")); + } +} diff --git a/src/openhuman/composio/providers/slack/sync.rs b/src/openhuman/composio/providers/slack/sync.rs new file mode 100644 index 000000000..2f6d92d4d --- /dev/null +++ b/src/openhuman/composio/providers/slack/sync.rs @@ -0,0 +1,397 @@ +//! Helpers for the Composio-backed Slack provider. +//! +//! Split out from `provider.rs` so the response-shape parsing code can +//! evolve independently of the sync orchestration — Composio (and Slack +//! beneath it) periodically widens response envelopes, and keeping the +//! JSON-pointer walks in one file makes adding new paths cheap. + +use std::collections::BTreeMap; +use std::time::{SystemTime, UNIX_EPOCH}; + +use chrono::{DateTime, TimeZone, Utc}; +use serde_json::Value; + +use super::users::SlackUsers; +use crate::openhuman::memory::slack_ingestion::types::{SlackChannel, SlackMessage}; + +/// Walk the Composio response envelope and pull out the channel array +/// from a `SLACK_LIST_CONVERSATIONS` call. Composio often wraps the raw +/// upstream shape one or two levels deeper, so we try multiple pointers. +pub(crate) fn extract_channels(data: &Value) -> Vec { + let candidates = [ + data.pointer("/data/channels"), + data.pointer("/channels"), + data.pointer("/data/data/channels"), + data.pointer("/data/conversations"), + data.pointer("/conversations"), + ]; + let arr = candidates + .into_iter() + .flatten() + .find_map(|v| v.as_array()) + .cloned() + .unwrap_or_default(); + + arr.into_iter().filter_map(parse_channel).collect() +} + +fn parse_channel(raw: Value) -> Option { + let id = raw + .get("id") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + if id.is_empty() { + return None; + } + let name = raw + .get("name") + .and_then(|v| v.as_str()) + .unwrap_or(&id) + .to_string(); + let is_private = raw + .get("is_private") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + Some(SlackChannel { + id, + name, + is_private, + }) +} + +/// Walk the Composio response envelope and pull out the `messages` array +/// from a `SLACK_FETCH_CONVERSATION_HISTORY` call. +/// +/// `users` resolves Slack user ids both as the message author and inline +/// `<@…>` mentions in the text. Pass [`SlackUsers::empty`] to skip +/// resolution — raw ids will pass through unchanged. +pub(crate) fn extract_messages( + data: &Value, + channel_id: &str, + users: &SlackUsers, +) -> Vec { + let candidates = [ + data.pointer("/data/messages"), + data.pointer("/messages"), + data.pointer("/data/data/messages"), + ]; + let arr = candidates + .into_iter() + .flatten() + .find_map(|v| v.as_array()) + .cloned() + .unwrap_or_default(); + + arr.into_iter() + .filter_map(|raw| parse_message(channel_id, raw, users)) + .collect() +} + +fn parse_message(channel_id: &str, raw: Value, users: &SlackUsers) -> Option { + let ts_raw = raw.get("ts").and_then(|t| t.as_str())?.to_string(); + let timestamp = parse_ts(&ts_raw)?; + let raw_text = raw + .get("text") + .and_then(|t| t.as_str()) + .unwrap_or("") + .to_string(); + if raw_text.trim().is_empty() { + return None; + } + // Replace `<@Uxxx>` mentions with `@` so the canonical + // markdown reads naturally. + let text = users.replace_mentions(&raw_text); + let author_id = raw + .get("user") + .or_else(|| raw.get("bot_id")) + .and_then(|u| u.as_str()) + .unwrap_or("") + .to_string(); + let author = users.resolve(&author_id); + let thread_ts = raw + .get("thread_ts") + .and_then(|t| t.as_str()) + .map(String::from); + Some(SlackMessage { + channel_id: channel_id.to_string(), + author, + text, + timestamp, + ts_raw, + thread_ts, + }) +} + +/// Slack's `ts` is a decimal string `"."`. The +/// integer part is what we care about for bucketing. +fn parse_ts(ts_raw: &str) -> Option> { + let seconds_str = ts_raw.split('.').next()?; + let secs: i64 = seconds_str.parse().ok()?; + Utc.timestamp_opt(secs, 0).single() +} + +/// Walk a `SLACK_SEARCH_MESSAGES` response envelope and pull out every +/// matching message. Unlike `extract_messages` (which is per-channel), +/// search results carry a `channel.id` field on each match — so the +/// returned `SlackMessage`s span every channel that matched the query. +pub(crate) fn extract_search_messages(data: &Value, users: &SlackUsers) -> Vec { + let candidates = [ + data.pointer("/data/messages/matches"), + data.pointer("/messages/matches"), + data.pointer("/data/data/messages/matches"), + ]; + let arr = candidates + .into_iter() + .flatten() + .find_map(|v| v.as_array()) + .cloned() + .unwrap_or_default(); + + arr.into_iter() + .filter_map(|raw| parse_search_match(raw, users)) + .collect() +} + +/// Total page count from the search response. Slack's `search.messages` +/// uses page-number pagination (1-indexed) under +/// `messages.paging.pages`. +pub(crate) fn extract_search_total_pages(data: &Value) -> u32 { + let candidates = [ + data.pointer("/data/messages/paging/pages"), + data.pointer("/messages/paging/pages"), + ]; + candidates + .into_iter() + .flatten() + .find_map(|v| v.as_u64()) + .unwrap_or(1) as u32 +} + +fn parse_search_match(raw: Value, users: &SlackUsers) -> Option { + let ts_raw = raw.get("ts").and_then(|t| t.as_str())?.to_string(); + let timestamp = parse_ts(&ts_raw)?; + let raw_text = raw + .get("text") + .and_then(|t| t.as_str()) + .unwrap_or("") + .to_string(); + if raw_text.trim().is_empty() { + return None; + } + let text = users.replace_mentions(&raw_text); + let author_id = raw + .get("user") + .or_else(|| raw.get("bot_id")) + .and_then(|u| u.as_str()) + .unwrap_or("") + .to_string(); + let author = users.resolve(&author_id); + let channel_id = raw + .pointer("/channel/id") + .and_then(|c| c.as_str())? + .to_string(); + let thread_ts = raw + .get("thread_ts") + .and_then(|t| t.as_str()) + .map(String::from); + Some(SlackMessage { + channel_id, + author, + text, + timestamp, + ts_raw, + thread_ts, + }) +} + +/// Extract a pagination `next_cursor` from a `SLACK_LIST_CONVERSATIONS` +/// or `SLACK_FETCH_CONVERSATION_HISTORY` response. +pub(crate) fn extract_next_cursor(data: &Value) -> Option { + let candidates = [ + data.pointer("/data/response_metadata/next_cursor"), + data.pointer("/response_metadata/next_cursor"), + data.pointer("/data/next_cursor"), + data.pointer("/next_cursor"), + ]; + for cand in candidates.into_iter().flatten() { + if let Some(s) = cand.as_str() { + let trimmed = s.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + } + None +} + +/// Per-channel cursor map encoded into `SyncState.cursor`. We use +/// `BTreeMap` so serialization is deterministic (makes log diffs +/// readable and tests stable). +/// +/// Value is unix-seconds of the **end of the latest flushed bucket** for +/// that channel. Fetches for that channel use `oldest = value` so we +/// skip already-ingested ranges. +pub type ChannelCursors = BTreeMap; + +/// Deserialize the per-channel cursor map out of `SyncState.cursor`. +/// Returns an empty map on any parse failure — a "broken" cursor should +/// degrade to "start from the backfill window" rather than bail out. +pub(crate) fn decode_cursors(raw: Option<&str>) -> ChannelCursors { + let Some(raw) = raw else { + return ChannelCursors::new(); + }; + match serde_json::from_str::(raw) { + Ok(map) => map, + Err(err) => { + tracing::warn!( + error = %err, + "[composio:slack] cursor parse failed, resetting per-channel cursors" + ); + ChannelCursors::new() + } + } +} + +pub(crate) fn encode_cursors(map: &ChannelCursors) -> String { + serde_json::to_string(map).unwrap_or_else(|_| "{}".to_string()) +} + +pub(crate) fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn extract_channels_from_data_channels() { + let data = json!({ + "data": { + "channels": [ + {"id": "C1", "name": "eng", "is_private": false}, + {"id": "G1", "name": "ops", "is_private": true}, + {"id": "", "name": "empty-id-drops"}, + ] + } + }); + let out = extract_channels(&data); + assert_eq!(out.len(), 2); + assert_eq!(out[0].id, "C1"); + assert!(!out[0].is_private); + assert_eq!(out[1].id, "G1"); + assert!(out[1].is_private); + } + + #[test] + fn extract_channels_honors_nested_envelope() { + let data = json!({ + "data": { "data": { "channels": [{"id": "C1", "name": "eng"}] } } + }); + let out = extract_channels(&data); + assert_eq!(out.len(), 1); + assert_eq!(out[0].id, "C1"); + } + + #[test] + fn extract_messages_parses_fields() { + let data = json!({ + "data": { + "messages": [ + {"ts": "1714003200.000100", "user": "U1", "text": "hi"}, + {"ts": "1714003300.000200", "user": "U2", "text": "world"}, + {"ts": "1714003400.000300", "user": "U3", "text": " "} // dropped (blank) + ] + } + }); + let users = SlackUsers::empty(); + let out = extract_messages(&data, "C1", &users); + assert_eq!(out.len(), 2); + assert_eq!(out[0].channel_id, "C1"); + // Empty user-cache passes raw id through unchanged. + assert_eq!(out[0].author, "U1"); + assert_eq!(out[0].text, "hi"); + assert_eq!(out[0].timestamp.timestamp(), 1_714_003_200); + assert_eq!(out[0].ts_raw, "1714003200.000100"); + } + + #[test] + fn extract_messages_resolves_authors_and_mentions() { + let data = json!({ + "data": { + "messages": [ + {"ts": "1714003200.0", "user": "U1", "text": "ping <@U2> about the migration"} + ] + } + }); + let mut m = std::collections::HashMap::new(); + m.insert("U1".into(), "alice".into()); + m.insert("U2".into(), "bob".into()); + let users = SlackUsers::from_map(m); + let out = extract_messages(&data, "C1", &users); + assert_eq!(out.len(), 1); + assert_eq!(out[0].author, "alice"); + assert_eq!(out[0].text, "ping @bob about the migration"); + } + + #[test] + fn extract_messages_skips_missing_ts() { + let data = json!({"messages": [{"user": "U1", "text": "orphan"}]}); + let users = SlackUsers::empty(); + let out = extract_messages(&data, "C1", &users); + assert!(out.is_empty()); + } + + #[test] + fn extract_next_cursor_finds_response_metadata_path() { + let data = json!({ + "data": { + "response_metadata": { "next_cursor": "dXNlcjpVMDY..." } + } + }); + assert_eq!( + extract_next_cursor(&data), + Some("dXNlcjpVMDY...".to_string()) + ); + } + + #[test] + fn extract_next_cursor_none_when_blank() { + let data = json!({"data": {"response_metadata": {"next_cursor": " "}}}); + assert!(extract_next_cursor(&data).is_none()); + } + + #[test] + fn encode_decode_roundtrip() { + let mut map = ChannelCursors::new(); + map.insert("C1".into(), 1_714_003_200); + map.insert("C2".into(), 1_714_010_000); + let encoded = encode_cursors(&map); + let decoded = decode_cursors(Some(&encoded)); + assert_eq!(decoded, map); + } + + #[test] + fn decode_empty_cursor_returns_empty_map() { + assert!(decode_cursors(None).is_empty()); + assert!(decode_cursors(Some("")).is_empty()); + assert!(decode_cursors(Some("not json")).is_empty()); + } + + #[test] + fn parse_ts_accepts_slack_decimal_format() { + let dt = parse_ts("1714003200.000100").unwrap(); + assert_eq!(dt.timestamp(), 1_714_003_200); + } + + #[test] + fn parse_ts_rejects_garbage() { + assert!(parse_ts("").is_none()); + assert!(parse_ts("not.a.number").is_none()); + } +} diff --git a/src/openhuman/composio/providers/slack/users.rs b/src/openhuman/composio/providers/slack/users.rs new file mode 100644 index 000000000..ae7373721 --- /dev/null +++ b/src/openhuman/composio/providers/slack/users.rs @@ -0,0 +1,364 @@ +//! Slack user-id → display-name resolver. +//! +//! Slack's `conversations.history` payload references users by their +//! workspace-stable id (e.g. `U01Q1TBL20P`) in two places: +//! +//! 1. The `user` field on each message (the author). +//! 2. Inline `<@U01Q1TBL20P>` mention syntax inside `text`. +//! +//! Neither is human-readable. To make canonical chat transcripts useful +//! for retrieval (and for humans reading the seal-cascade summaries), +//! we fetch the workspace's user directory once per sync run, build an +//! id → display-name map, and apply it both to the author field and to +//! every `<@…>` mention in message bodies. +//! +//! ## Cache scope +//! +//! Per-sync only. Each `SlackProvider::sync()` invocation calls +//! [`SlackUsers::fetch`] once before walking channels. The map lives +//! in a local variable for the duration of the sync, then drops. +//! Slack's user list rarely changes within a 15-minute sync window, +//! and re-fetching per sync keeps stale-cache risk near zero without +//! adding persistence machinery. +//! +//! ## Soft-fallback contract +//! +//! Following the pattern of [`crate::openhuman::composio::providers::slack::sync::extract_messages`] +//! and the [`super::provider::SlackProvider::sync`] error handling, a +//! failure to fetch users is **not fatal**. The returned [`SlackUsers`] +//! is empty, and `resolve()` / `replace_mentions()` pass through raw +//! ids unchanged — same behaviour as before this module existed. + +use regex::Regex; +use serde_json::{json, Value}; +use std::collections::HashMap; +use std::sync::OnceLock; + +use crate::openhuman::composio::client::ComposioClient; + +/// Composio action slug for the bulk user listing. +const ACTION_LIST_USERS: &str = "SLACK_LIST_ALL_USERS"; + +/// Page size — Slack caps at 1000; 200 keeps each page small. +const PAGE_SIZE: u32 = 200; + +/// Maximum pages to walk per sync. With `PAGE_SIZE = 200` this covers +/// workspaces up to 4000 users without complaint. Beyond that the tail +/// is truncated and unresolved ids will pass through verbatim. +const MAX_PAGES: u32 = 20; + +/// Slack mention syntax: `<@U01Q1TBL20P>`. Captures the bare id so we +/// can drop the wrapper when substituting in a resolved name. +fn mention_re() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| Regex::new(r"<@(U[A-Z0-9]+)>").expect("static mention regex compiles")) +} + +/// Map of Slack user id → human-readable display name. +#[derive(Debug, Default, Clone)] +pub struct SlackUsers { + map: HashMap, +} + +impl SlackUsers { + /// Empty map — `resolve()` passes through raw ids verbatim. + pub fn empty() -> Self { + Self::default() + } + + /// Number of users in the cache. + pub fn len(&self) -> usize { + self.map.len() + } + + pub fn is_empty(&self) -> bool { + self.map.is_empty() + } + + /// Resolve a Slack user id to a display name. Returns the input id + /// unchanged when no mapping exists — matches the + /// resolve-or-passthrough contract of the parent provider. + pub fn resolve(&self, user_id: &str) -> String { + self.map + .get(user_id) + .cloned() + .unwrap_or_else(|| user_id.to_string()) + } + + /// Replace every `<@Uxxx>` mention in `text` with `@`. + /// Unknown ids stay as `@Uxxx` (the wrapper is removed but the id + /// is preserved so retrieval can still surface them). + pub fn replace_mentions(&self, text: &str) -> String { + mention_re() + .replace_all(text, |caps: ®ex::Captures| { + let id = &caps[1]; + let resolved = self.map.get(id).map(String::as_str).unwrap_or(id); + format!("@{resolved}") + }) + .into_owned() + } + + /// Pull the workspace user directory via Composio. Soft-fails to + /// [`SlackUsers::empty`] on transport, HTTP, JSON, or + /// provider-failure errors so the sync can continue with raw ids. + /// + /// Returns `(users, total_attempts)` where `total_attempts` sums every + /// real Composio call this fetch made across pages and rate-limit + /// retries, so the caller can charge the daily quota meter + /// accurately. Pages walked silently are tracked too — without this, + /// large workspaces under-report their request usage. + pub async fn fetch(client: &ComposioClient) -> (Self, u32) { + let mut map: HashMap = HashMap::new(); + let mut cursor: Option = None; + let mut total_attempts: u32 = 0; + + for page_num in 0..MAX_PAGES { + let mut args = json!({ "limit": PAGE_SIZE }); + if let Some(ref c) = cursor { + args["cursor"] = json!(c); + } + + // Going through `execute_with_retry` so a transient + // `ratelimited` page doesn't drop us into a half-built + // directory while the rest of the provider uses backoff. + // Soft-fall to whatever was collected so far on any failure. + let (resp, attempts) = match super::provider::execute_with_retry( + client, + ACTION_LIST_USERS, + args, + &format!("{ACTION_LIST_USERS} page {page_num}"), + ) + .await + { + Ok(t) => t, + Err(err) => { + // We don't know exactly how many attempts the helper + // burned before bailing, but at least one ran — count + // it so the budget meter doesn't silently undercount. + total_attempts = total_attempts.saturating_add(1); + log::warn!( + "[composio:slack:users] {ACTION_LIST_USERS} page {page_num} failed: {err} — \ + degrading to raw ids for the rest of this sync" + ); + return (Self { map }, total_attempts); + } + }; + total_attempts = total_attempts.saturating_add(attempts); + + super::provider::dump_response("_meta", "users", page_num, &resp.data); + absorb_page(&resp.data, &mut map); + + cursor = extract_next_cursor(&resp.data); + if cursor.is_none() { + break; + } + } + + log::info!( + "[composio:slack:users] resolved {} workspace users in {total_attempts} call(s)", + map.len() + ); + (Self { map }, total_attempts) + } + + /// Construct from a pre-built map. Test-only — production callers + /// should use [`Self::fetch`] or [`Self::empty`]. + #[cfg(test)] + pub fn from_map(map: HashMap) -> Self { + Self { map } + } +} + +/// Walk a Composio response envelope and absorb every user object's +/// `id` + best-available display name into `map`. +fn absorb_page(data: &Value, map: &mut HashMap) { + let candidates = [ + data.pointer("/data/members"), + data.pointer("/members"), + data.pointer("/data/users"), + data.pointer("/users"), + data.pointer("/data/data/members"), + ]; + let arr = candidates + .into_iter() + .flatten() + .find_map(|v| v.as_array()) + .cloned() + .unwrap_or_default(); + + for raw in arr { + let id = raw + .get("id") + .and_then(|v| v.as_str()) + .unwrap_or("") + .trim() + .to_string(); + if id.is_empty() { + continue; + } + if let Some(name) = pick_display_name(&raw) { + map.insert(id, name); + } + } +} + +/// Slack returns several name fields per user. Prefer the most +/// human-readable, fall back through real_name → name → display_name. +fn pick_display_name(raw: &Value) -> Option { + let candidates = [ + raw.pointer("/profile/display_name"), + raw.pointer("/profile/real_name"), + raw.get("real_name"), + raw.get("name"), + raw.pointer("/profile/display_name_normalized"), + raw.pointer("/profile/real_name_normalized"), + ]; + for cand in candidates.into_iter().flatten() { + if let Some(s) = cand.as_str() { + let trimmed = s.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + } + None +} + +fn extract_next_cursor(data: &Value) -> Option { + let candidates = [ + data.pointer("/data/response_metadata/next_cursor"), + data.pointer("/response_metadata/next_cursor"), + data.pointer("/data/next_cursor"), + data.pointer("/next_cursor"), + ]; + for cand in candidates.into_iter().flatten() { + if let Some(s) = cand.as_str() { + let trimmed = s.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample_users() -> SlackUsers { + let mut m = HashMap::new(); + m.insert("U001".to_string(), "alice".to_string()); + m.insert("U002".to_string(), "bob".to_string()); + SlackUsers::from_map(m) + } + + #[test] + fn resolve_known_id_returns_name() { + let u = sample_users(); + assert_eq!(u.resolve("U001"), "alice"); + } + + #[test] + fn resolve_unknown_id_passes_through() { + let u = sample_users(); + assert_eq!(u.resolve("U999"), "U999"); + } + + #[test] + fn empty_passes_through_every_id() { + let u = SlackUsers::empty(); + assert_eq!(u.resolve("U001"), "U001"); + assert_eq!(u.replace_mentions("hi <@U001>"), "hi @U001"); + } + + #[test] + fn replace_mentions_substitutes_known_ids() { + let u = sample_users(); + let out = u.replace_mentions("Hi <@U001>, please ping <@U002>."); + assert_eq!(out, "Hi @alice, please ping @bob."); + } + + #[test] + fn replace_mentions_strips_wrapper_for_unknown_id() { + let u = sample_users(); + // Unknown id keeps the raw id but loses the `<@...>` wrapper. + let out = u.replace_mentions("ping <@U999>"); + assert_eq!(out, "ping @U999"); + } + + #[test] + fn replace_mentions_leaves_non_mention_text_alone() { + let u = sample_users(); + let out = u.replace_mentions("no mentions here, just "); + assert_eq!(out, "no mentions here, just "); + } + + #[test] + fn replace_mentions_handles_multiple_in_one_line() { + let u = sample_users(); + let out = u.replace_mentions("<@U001> said hi to <@U001> and <@U002>"); + assert_eq!(out, "@alice said hi to @alice and @bob"); + } + + #[test] + fn absorb_page_reads_data_members_path() { + let data = json!({ + "data": { + "members": [ + { + "id": "U001", + "profile": { "display_name": "alice", "real_name": "Alice Smith" } + }, + { + "id": "U002", + "profile": { "display_name": "" , "real_name": "Bob Jones" } + }, + { + "id": "", + "profile": { "display_name": "skipped" } + } + ] + } + }); + let mut m = HashMap::new(); + absorb_page(&data, &mut m); + assert_eq!(m.get("U001").unwrap(), "alice"); + // Falls back to real_name when display_name is blank. + assert_eq!(m.get("U002").unwrap(), "Bob Jones"); + // Empty id row is dropped. + assert!(!m.contains_key("")); + } + + #[test] + fn pick_display_name_prefers_display_name_over_real_name() { + let raw = json!({ + "profile": { "display_name": "alice", "real_name": "Alice Smith" } + }); + assert_eq!(pick_display_name(&raw).as_deref(), Some("alice")); + } + + #[test] + fn pick_display_name_falls_back_to_name() { + let raw = json!({ "name": "alice", "profile": {} }); + assert_eq!(pick_display_name(&raw).as_deref(), Some("alice")); + } + + #[test] + fn pick_display_name_returns_none_when_all_blank() { + let raw = json!({ "profile": { "display_name": " " }, "name": "" }); + assert!(pick_display_name(&raw).is_none()); + } + + #[test] + fn extract_next_cursor_finds_response_metadata() { + let data = json!({"data": {"response_metadata": {"next_cursor": "abc123"}}}); + assert_eq!(extract_next_cursor(&data).as_deref(), Some("abc123")); + } + + #[test] + fn extract_next_cursor_none_when_blank() { + let data = json!({"response_metadata": {"next_cursor": " "}}); + assert!(extract_next_cursor(&data).is_none()); + } +} diff --git a/src/openhuman/config/schema/load.rs b/src/openhuman/config/schema/load.rs index a25bbf283..badc8c2e6 100644 --- a/src/openhuman/config/schema/load.rs +++ b/src/openhuman/config/schema/load.rs @@ -926,6 +926,60 @@ impl Config { } } + // LLM entity extractor overrides — set endpoint + model to route + // ingest scoring through Ollama NER (Phase 2 follow-up). Empty + // string explicitly clears (opts out). + if let Ok(endpoint) = std::env::var("OPENHUMAN_MEMORY_EXTRACT_ENDPOINT") { + let trimmed = endpoint.trim(); + self.memory_tree.llm_extractor_endpoint = if trimmed.is_empty() { + None + } else { + Some(trimmed.to_string()) + }; + } + if let Ok(model) = std::env::var("OPENHUMAN_MEMORY_EXTRACT_MODEL") { + let trimmed = model.trim(); + self.memory_tree.llm_extractor_model = if trimmed.is_empty() { + None + } else { + Some(trimmed.to_string()) + }; + } + if let Ok(val) = std::env::var("OPENHUMAN_MEMORY_EXTRACT_TIMEOUT_MS") { + if let Ok(ms) = val.trim().parse::() { + if ms > 0 { + self.memory_tree.llm_extractor_timeout_ms = Some(ms); + } + } + } + + // LLM summariser overrides — set endpoint + model to route + // bucket-seal summaries through Ollama instead of InertSummariser + // (Phase 3a real-summariser hook). + if let Ok(endpoint) = std::env::var("OPENHUMAN_MEMORY_SUMMARISE_ENDPOINT") { + let trimmed = endpoint.trim(); + self.memory_tree.llm_summariser_endpoint = if trimmed.is_empty() { + None + } else { + Some(trimmed.to_string()) + }; + } + if let Ok(model) = std::env::var("OPENHUMAN_MEMORY_SUMMARISE_MODEL") { + let trimmed = model.trim(); + self.memory_tree.llm_summariser_model = if trimmed.is_empty() { + None + } else { + Some(trimmed.to_string()) + }; + } + if let Ok(val) = std::env::var("OPENHUMAN_MEMORY_SUMMARISE_TIMEOUT_MS") { + if let Ok(ms) = val.trim().parse::() { + if ms > 0 { + self.memory_tree.llm_summariser_timeout_ms = Some(ms); + } + } + } + // Auto-update overrides if let Some(flag) = env.get("OPENHUMAN_AUTO_UPDATE_ENABLED") { let normalized = flag.trim().to_ascii_lowercase(); diff --git a/src/openhuman/config/schema/storage_memory.rs b/src/openhuman/config/schema/storage_memory.rs index f10a52c5b..790bccd92 100644 --- a/src/openhuman/config/schema/storage_memory.rs +++ b/src/openhuman/config/schema/storage_memory.rs @@ -90,6 +90,12 @@ impl Default for MemoryConfig { /// - `OPENHUMAN_MEMORY_EMBED_ENDPOINT` /// - `OPENHUMAN_MEMORY_EMBED_MODEL` /// - `OPENHUMAN_MEMORY_EMBED_TIMEOUT_MS` +/// - `OPENHUMAN_MEMORY_EXTRACT_ENDPOINT` +/// - `OPENHUMAN_MEMORY_EXTRACT_MODEL` +/// - `OPENHUMAN_MEMORY_EXTRACT_TIMEOUT_MS` +/// - `OPENHUMAN_MEMORY_SUMMARISE_ENDPOINT` +/// - `OPENHUMAN_MEMORY_SUMMARISE_MODEL` +/// - `OPENHUMAN_MEMORY_SUMMARISE_TIMEOUT_MS` #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct MemoryTreeConfig { /// Ollama endpoint for the embedder (e.g. `http://localhost:11434`). @@ -113,6 +119,41 @@ pub struct MemoryTreeConfig { /// rerank falls back to scope + recency ordering only. #[serde(default = "default_memory_tree_embedding_strict")] pub embedding_strict: bool, + + /// Ollama endpoint for the LLM entity extractor + /// (`memory::tree::score::extract::llm::LlmEntityExtractor`). When + /// unset, ingest uses the regex-only extractor — no LLM call. Soft + /// failures in the LLM path fall back to regex-only for that chunk. + #[serde(default = "default_memory_tree_llm_endpoint")] + pub llm_extractor_endpoint: Option, + + /// Model name for the entity extractor (e.g. `qwen2.5:0.5b`). + #[serde(default = "default_memory_tree_llm_endpoint")] + pub llm_extractor_model: Option, + + /// Per-request timeout for the LLM extractor, in milliseconds. + #[serde(default = "default_memory_tree_llm_extractor_timeout_ms")] + pub llm_extractor_timeout_ms: Option, + + /// Ollama endpoint for the summariser + /// (`memory::tree::source_tree::summariser::llm::LlmSummariser`). + /// When unset, bucket-seal cascades use `InertSummariser` — sealed + /// nodes contain concatenated+truncated child text instead of a + /// real LLM summary. Soft failures fall back to inert per seal. + #[serde(default = "default_memory_tree_llm_endpoint")] + pub llm_summariser_endpoint: Option, + + /// Model name for the summariser. Larger models produce better + /// summaries at higher latency; `llama3.1:8b` is a reasonable + /// default for production. + #[serde(default = "default_memory_tree_llm_endpoint")] + pub llm_summariser_model: Option, + + /// Per-request timeout for the summariser, in milliseconds. Default + /// is higher than the extractor because summarisation uses more + /// tokens and therefore takes longer to generate. + #[serde(default = "default_memory_tree_llm_summariser_timeout_ms")] + pub llm_summariser_timeout_ms: Option, } /// Returns `None` so that existing installs that never opted into Phase 4 @@ -139,6 +180,25 @@ fn default_memory_tree_embedding_strict() -> bool { false } +/// Shared `None` default for the LLM-path fields (extractor + summariser +/// endpoints + models). Keeping the same function for all of them makes +/// the intent explicit: nothing here auto-enables Ollama. +fn default_memory_tree_llm_endpoint() -> Option { + None +} + +fn default_memory_tree_llm_extractor_timeout_ms() -> Option { + Some(15_000) +} + +fn default_memory_tree_llm_summariser_timeout_ms() -> Option { + // 120s — large enough for small/medium local models to finish a + // seal-budget summary on a cold-loaded weight cache. Tighter + // values cause the LlmSummariser to time out and silently fall + // back to InertSummariser (no LLM signal in the resulting node). + Some(120_000) +} + impl Default for MemoryTreeConfig { fn default() -> Self { Self { @@ -146,6 +206,12 @@ impl Default for MemoryTreeConfig { embedding_model: default_memory_tree_embedding_model(), embedding_timeout_ms: default_memory_tree_embedding_timeout_ms(), embedding_strict: default_memory_tree_embedding_strict(), + llm_extractor_endpoint: default_memory_tree_llm_endpoint(), + llm_extractor_model: default_memory_tree_llm_endpoint(), + llm_extractor_timeout_ms: default_memory_tree_llm_extractor_timeout_ms(), + llm_summariser_endpoint: default_memory_tree_llm_endpoint(), + llm_summariser_model: default_memory_tree_llm_endpoint(), + llm_summariser_timeout_ms: default_memory_tree_llm_summariser_timeout_ms(), } } } diff --git a/src/openhuman/memory/mod.rs b/src/openhuman/memory/mod.rs index 58ecec35b..7cfe92e36 100644 --- a/src/openhuman/memory/mod.rs +++ b/src/openhuman/memory/mod.rs @@ -14,6 +14,7 @@ pub mod ingestion_queue; pub mod ops; pub mod rpc_models; pub mod schemas; +pub mod slack_ingestion; pub mod store; pub mod traits; pub mod tree; @@ -30,6 +31,9 @@ pub use schemas::{ all_controller_schemas as all_memory_controller_schemas, all_registered_controllers as all_memory_registered_controllers, }; +pub use slack_ingestion::{ + all_slack_ingestion_controller_schemas, all_slack_ingestion_registered_controllers, +}; pub use store::{ create_memory, create_memory_for_migration, create_memory_with_storage, create_memory_with_storage_and_routes, effective_memory_backend_name, MemoryClient, diff --git a/src/openhuman/memory/slack_ingestion/bucketer.rs b/src/openhuman/memory/slack_ingestion/bucketer.rs new file mode 100644 index 000000000..bae1f5fd9 --- /dev/null +++ b/src/openhuman/memory/slack_ingestion/bucketer.rs @@ -0,0 +1,261 @@ +//! 6-hour UTC-aligned bucketing for Slack messages. +//! +//! Buckets have fixed wall-clock starts at `00:00`, `06:00`, `12:00`, and +//! `18:00` UTC each day. A message's bucket is determined solely by its +//! timestamp — the engine never decides bucketing based on order of +//! arrival or polling cadence, so the same message always lands in the +//! same bucket regardless of when it's seen. +//! +//! ## Why closed-bucket-only? +//! +//! Each ingested chunk becomes a *leaf* in the source tree via +//! `append_leaf` (see `memory::tree::source_tree::bucket_seal`). Leaves +//! participate in a token-budget seal cascade — once appended, they +//! cannot be cleanly retracted from already-sealed summary nodes +//! upstream. So we only ingest a bucket once, *after* it has closed +//! (window end + grace period is past). +//! +//! The `source_id` convention (`"slack::"`) +//! plus deterministic chunk IDs give us idempotent re-ingest for late +//! edits in a future iteration without breaking that invariant. + +use chrono::{DateTime, Datelike, Duration, TimeZone, Timelike, Utc}; +use std::collections::BTreeMap; + +use super::types::{Bucket, SlackMessage}; + +/// Width of each bucket. Changing this changes the ingest granularity +/// *and* invalidates existing `source_id`s, so treat it as a schema +/// constant — not a runtime tunable. +pub const BUCKET_HOURS: u32 = 6; + +/// Safety window after a bucket's nominal end before we consider it +/// closed. Protects against clock skew, slow API polls, or messages +/// that arrive via `conversations.history` a few minutes after they +/// were sent. +pub const GRACE_PERIOD: Duration = Duration::minutes(15); + +/// Round a timestamp down to its 6-hour UTC bucket start. +/// +/// For `2026-04-25T13:42:07Z` this returns `2026-04-25T12:00:00Z`. +pub fn bucket_start_for(ts: DateTime) -> DateTime { + let hour = ts.hour(); + let bucket_hour = (hour / BUCKET_HOURS) * BUCKET_HOURS; + Utc.with_ymd_and_hms(ts.year(), ts.month(), ts.day(), bucket_hour, 0, 0) + .single() + .expect("00/06/12/18 are always valid wall-clock hours in UTC") +} + +/// Exclusive bucket end — `start + BUCKET_HOURS`. +pub fn bucket_end_for(start: DateTime) -> DateTime { + start + Duration::hours(BUCKET_HOURS as i64) +} + +/// Stable identifier for a channel's bucket, used as the memory-tree +/// `source_id`. Uses Unix seconds so the id is compact, comparable, and +/// independent of the caller's locale. +pub fn source_id_for(channel_id: &str, bucket_start: DateTime) -> String { + format!("slack:{channel_id}:{}", bucket_start.timestamp()) +} + +/// Partition an in-memory buffer into closed buckets (ready to ingest) +/// and the messages that remain in still-open buckets. +/// +/// A bucket is *closed* when `bucket_end + GRACE_PERIOD <= now`. Messages +/// in closed buckets are grouped into [`Bucket`] values ordered by start. +/// Messages in open buckets flow back into `remaining` and will be +/// re-evaluated on a later tick when their window closes. +/// +/// Empty buckets are never produced — if no messages fall into a range +/// there is no [`Bucket`] for it. +pub fn split_closed( + buffer: Vec, + now: DateTime, + grace: Duration, +) -> (Vec, Vec) { + let mut by_bucket: BTreeMap, Vec> = BTreeMap::new(); + for msg in buffer { + let start = bucket_start_for(msg.timestamp); + by_bucket.entry(start).or_default().push(msg); + } + + let mut closed = Vec::new(); + let mut remaining = Vec::new(); + for (start, mut messages) in by_bucket { + let end = bucket_end_for(start); + if end + grace <= now { + // Preserve chronological order within the bucket. + messages.sort_by_key(|m| m.timestamp); + closed.push(Bucket { + start, + end, + messages, + }); + } else { + remaining.extend(messages); + } + } + (closed, remaining) +} + +/// Enumerate every 6-hour bucket start in `[from, until)` — used by the +/// backfill path to walk the last N days in order. +pub fn buckets_between(from: DateTime, until: DateTime) -> Vec> { + if until <= from { + return Vec::new(); + } + let mut starts = Vec::new(); + let mut cursor = bucket_start_for(from); + while cursor < until { + starts.push(cursor); + cursor = bucket_end_for(cursor); + } + starts +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ts(year: i32, month: u32, day: u32, hour: u32, min: u32) -> DateTime { + Utc.with_ymd_and_hms(year, month, day, hour, min, 0) + .single() + .unwrap() + } + + fn msg(channel: &str, timestamp: DateTime, body: &str) -> SlackMessage { + SlackMessage { + channel_id: channel.into(), + author: "U1".into(), + text: body.into(), + timestamp, + ts_raw: format!("{}.000000", timestamp.timestamp()), + thread_ts: None, + } + } + + #[test] + fn bucket_start_rounds_down_to_six_hour_boundary() { + assert_eq!( + bucket_start_for(ts(2026, 4, 25, 0, 0)), + ts(2026, 4, 25, 0, 0) + ); + assert_eq!( + bucket_start_for(ts(2026, 4, 25, 5, 59)), + ts(2026, 4, 25, 0, 0) + ); + assert_eq!( + bucket_start_for(ts(2026, 4, 25, 6, 0)), + ts(2026, 4, 25, 6, 0) + ); + assert_eq!( + bucket_start_for(ts(2026, 4, 25, 13, 42)), + ts(2026, 4, 25, 12, 0) + ); + assert_eq!( + bucket_start_for(ts(2026, 4, 25, 23, 59)), + ts(2026, 4, 25, 18, 0) + ); + } + + #[test] + fn bucket_end_is_start_plus_six_hours() { + let start = ts(2026, 4, 25, 6, 0); + assert_eq!(bucket_end_for(start), ts(2026, 4, 25, 12, 0)); + } + + #[test] + fn source_id_is_deterministic_and_channel_scoped() { + let start = ts(2026, 4, 25, 12, 0); + assert_eq!( + source_id_for("C0123456", start), + format!("slack:C0123456:{}", start.timestamp()) + ); + assert_ne!( + source_id_for("C0123456", start), + source_id_for("C9999999", start), + ); + } + + #[test] + fn split_closed_keeps_open_bucket_in_remaining() { + let now = ts(2026, 4, 25, 7, 0); // bucket [06:00, 12:00) still open + let buf = vec![msg("C1", ts(2026, 4, 25, 6, 30), "hi")]; + let (closed, remaining) = split_closed(buf, now, GRACE_PERIOD); + assert!(closed.is_empty()); + assert_eq!(remaining.len(), 1); + } + + #[test] + fn split_closed_emits_fully_closed_bucket() { + // Bucket [00:00, 06:00) closes at 06:15 with grace; now is 07:00. + let now = ts(2026, 4, 25, 7, 0); + let buf = vec![ + msg("C1", ts(2026, 4, 25, 1, 0), "a"), + msg("C1", ts(2026, 4, 25, 3, 30), "b"), + ]; + let (closed, remaining) = split_closed(buf, now, GRACE_PERIOD); + assert_eq!(closed.len(), 1); + assert_eq!(remaining.len(), 0); + assert_eq!(closed[0].start, ts(2026, 4, 25, 0, 0)); + assert_eq!(closed[0].end, ts(2026, 4, 25, 6, 0)); + assert_eq!(closed[0].messages.len(), 2); + // Sorted chronologically inside the bucket. + assert!(closed[0].messages[0].timestamp < closed[0].messages[1].timestamp); + } + + #[test] + fn split_closed_respects_grace_boundary() { + // Bucket ends at 06:00. With 15-min grace, it closes at 06:15. + let just_before = ts(2026, 4, 25, 6, 14); + let just_after = ts(2026, 4, 25, 6, 15); + let buf = || vec![msg("C1", ts(2026, 4, 25, 3, 0), "x")]; + + let (closed_before, _) = split_closed(buf(), just_before, GRACE_PERIOD); + assert!(closed_before.is_empty(), "still within grace window"); + + let (closed_after, remaining) = split_closed(buf(), just_after, GRACE_PERIOD); + assert_eq!(closed_after.len(), 1, "grace expired exactly at 06:15"); + assert!(remaining.is_empty()); + } + + #[test] + fn split_closed_handles_mixed_buckets() { + let now = ts(2026, 4, 25, 13, 0); + let buf = vec![ + msg("C1", ts(2026, 4, 25, 1, 0), "closed bucket 1"), + msg("C1", ts(2026, 4, 25, 7, 0), "closed bucket 2"), + msg("C1", ts(2026, 4, 25, 12, 5), "open bucket"), + ]; + let (closed, remaining) = split_closed(buf, now, GRACE_PERIOD); + assert_eq!(closed.len(), 2); + assert_eq!(remaining.len(), 1); + assert_eq!(closed[0].start, ts(2026, 4, 25, 0, 0)); + assert_eq!(closed[1].start, ts(2026, 4, 25, 6, 0)); + } + + #[test] + fn split_closed_empty_buffer_is_noop() { + let (closed, remaining) = split_closed(vec![], ts(2026, 4, 25, 12, 0), GRACE_PERIOD); + assert!(closed.is_empty()); + assert!(remaining.is_empty()); + } + + #[test] + fn buckets_between_walks_six_hour_starts() { + let from = ts(2026, 4, 25, 2, 0); // snaps to 00:00 + let until = ts(2026, 4, 26, 0, 0); + let starts = buckets_between(from, until); + assert_eq!(starts.len(), 4); + assert_eq!(starts[0], ts(2026, 4, 25, 0, 0)); + assert_eq!(starts[1], ts(2026, 4, 25, 6, 0)); + assert_eq!(starts[2], ts(2026, 4, 25, 12, 0)); + assert_eq!(starts[3], ts(2026, 4, 25, 18, 0)); + } + + #[test] + fn buckets_between_empty_when_range_reversed() { + let starts = buckets_between(ts(2026, 4, 26, 0, 0), ts(2026, 4, 25, 0, 0)); + assert!(starts.is_empty()); + } +} diff --git a/src/openhuman/memory/slack_ingestion/mod.rs b/src/openhuman/memory/slack_ingestion/mod.rs new file mode 100644 index 000000000..510e8498a --- /dev/null +++ b/src/openhuman/memory/slack_ingestion/mod.rs @@ -0,0 +1,46 @@ +//! Slack → memory-tree ingestion support. +//! +//! This module is the "Slack-specific plumbing the memory tree needs": +//! +//! - [`bucketer`] — 6-hour UTC-aligned window grouping + grace-period +//! closed-bucket extraction (`split_closed`) + stable `source_id` +//! generation. +//! - [`ops`] — bucket → [`memory::tree::canonicalize::chat::ChatBatch`] +//! conversion + wrapper around +//! [`memory::tree::ingest::ingest_chat`] so the caller just says +//! "ingest this bucket for this channel". +//! - [`types`] — `SlackMessage`, `SlackChannel`, `Bucket`. Used by +//! both the Composio-backed [`crate::openhuman::composio::providers::slack`] +//! provider and the RPC/observability surfaces below. +//! - [`rpc`] / [`schemas`] — JSON-RPC surface for manually triggering a +//! Slack sync (`openhuman.slack_memory_sync_trigger`) + inspecting +//! per-connection state (`openhuman.slack_memory_sync_status`). +//! +//! Auth + scheduling live elsewhere: +//! +//! - OAuth is delegated to Composio (the user's Slack connection in +//! Composio's hosted flow is what authorises the API calls). +//! - Periodic scheduling comes from `composio::periodic` — every 15 +//! minutes the scheduler fires `SlackProvider::sync()` for every +//! active Slack connection. +//! +//! What this module does NOT contain (removed when we pivoted from +//! direct-bot-token to Composio): +//! +//! - A Slack Web API HTTP client. Calls go through `ctx.client.execute_tool()`. +//! - A custom per-channel cursor SQLite table. State is persisted via +//! `composio::providers::sync_state::SyncState` in the memory KV store, +//! keyed by `(toolkit="slack", connection_id)`. +//! - A long-running engine task. `composio::periodic::run_one_tick` is +//! the scheduler. + +pub mod bucketer; +pub mod ops; +pub mod rpc; +pub mod schemas; +pub mod types; + +pub use schemas::{ + all_controller_schemas as all_slack_ingestion_controller_schemas, + all_registered_controllers as all_slack_ingestion_registered_controllers, +}; diff --git a/src/openhuman/memory/slack_ingestion/ops.rs b/src/openhuman/memory/slack_ingestion/ops.rs new file mode 100644 index 000000000..fd85b166e --- /dev/null +++ b/src/openhuman/memory/slack_ingestion/ops.rs @@ -0,0 +1,234 @@ +//! Bucket → memory-tree ingest plumbing. +//! +//! This module owns the one-way conversion from our internal +//! [`Bucket`] type to [`ChatBatch`] + the call into +//! [`memory::tree::ingest::ingest_chat`]. It is intentionally free of +//! HTTP, timers, or state — those belong to [`super::engine`] — so the +//! canonicalisation path is easy to unit test. + +use anyhow::{Context, Result}; +use chrono::{DateTime, Utc}; + +use super::types::{Bucket, SlackChannel}; +use crate::openhuman::config::Config; +use crate::openhuman::memory::tree::canonicalize::chat::{ChatBatch, ChatMessage}; +use crate::openhuman::memory::tree::ingest::{ingest_chat, IngestResult}; + +/// Platform identifier embedded in the canonical chat transcript header. +/// Matches the entry in `memory::tree::retrieval::source::PLATFORM_KINDS`. +pub const SLACK_PLATFORM: &str = "slack"; + +/// Tags attached to every Slack-ingested chunk. Keep this list stable — +/// retrieval callers filter on them. +pub const DEFAULT_TAGS: &[&str] = &["slack", "ingested"]; + +/// Convert a closed [`Bucket`] into a [`ChatBatch`] ready for +/// `memory::tree::ingest::ingest_chat`. +/// +/// Empty buckets return `None` — upstream code should skip them rather +/// than ingest an empty payload. Bucket authorship is preserved by +/// using the Slack user ID (`author`) directly; the tree will not +/// attempt to resolve it to a display name (future enhancement could +/// plug in a `users.info` cache). +pub fn bucket_to_chat_batch(bucket: &Bucket, channel: &SlackChannel) -> Option { + if bucket.messages.is_empty() { + return None; + } + let messages = bucket + .messages + .iter() + .map(|m| ChatMessage { + author: if m.author.is_empty() { + "unknown".to_string() + } else { + m.author.clone() + }, + timestamp: m.timestamp, + text: m.text.clone(), + source_ref: Some(slack_archive_url(&channel.id, &m.ts_raw)), + }) + .collect(); + + Some(ChatBatch { + platform: SLACK_PLATFORM.to_string(), + channel_label: channel_label(channel), + messages, + }) +} + +/// Render a channel label for the canonical transcript header — +/// `"#eng"` for public channels, `"private:ops"` for private ones so +/// the retrieval side can distinguish them at a glance. +pub fn channel_label(channel: &SlackChannel) -> String { + if channel.is_private { + format!("private:{}", channel.name) + } else { + format!("#{}", channel.name) + } +} + +/// Permalink-shaped pointer to a Slack message. Not a real HTTPS URL — +/// we don't know the workspace hostname from the bot token alone — but +/// retains enough info for humans to reconstruct one. +fn slack_archive_url(channel_id: &str, ts_raw: &str) -> String { + // Example: slack://archives/C012345/1714003200.000100 + format!("slack://archives/{channel_id}/{ts_raw}") +} + +/// Ingest one bucket. Returns the tree's [`IngestResult`] so callers +/// can surface chunk counts in logs / RPC responses. +/// +/// `connection_id` is the Composio connection (typically one Slack +/// workspace per connection). It becomes the `source_id` so all +/// channels' all buckets accumulate into ONE source tree per +/// workspace — letting the L0 buffer fill across many ingest calls +/// and eventually trigger a seal cascade. Chunk-id uniqueness across +/// buckets is preserved by `chunk_id` now hashing content. +pub async fn ingest_bucket( + config: &Config, + channel: &SlackChannel, + bucket: &Bucket, + owner: &str, + connection_id: &str, +) -> Result { + let source_id = format!("slack:{connection_id}"); + let batch = match bucket_to_chat_batch(bucket, channel) { + Some(b) => b, + None => { + log::debug!( + "[slack_ingest][ops] skip empty bucket channel={} start={}", + channel.id, + bucket.start.to_rfc3339() + ); + return Ok(IngestResult { + source_id: source_id.clone(), + chunks_written: 0, + chunks_dropped: 0, + chunk_ids: Vec::new(), + }); + } + }; + let tags = DEFAULT_TAGS.iter().map(|s| (*s).to_string()).collect(); + log::info!( + "[slack_ingest][ops] ingest bucket channel={} start={} messages={} source_id={}", + channel.id, + bucket.start.to_rfc3339(), + batch.messages.len(), + source_id + ); + ingest_chat(config, &source_id, owner, tags, batch) + .await + .with_context(|| { + format!( + "[slack_ingest][ops] ingest_chat failed channel={} source_id={}", + channel.id, source_id + ) + }) +} + +/// Unix seconds for an `end`-of-bucket cursor advance — used by the +/// engine to persist `last_synced_ts` after a successful ingest. +pub fn cursor_ts_for_flushed_bucket(end: DateTime) -> i64 { + end.timestamp() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::memory::slack_ingestion::types::SlackMessage; + use chrono::TimeZone; + + fn sample_channel() -> SlackChannel { + SlackChannel { + id: "C0123456".into(), + name: "eng".into(), + is_private: false, + } + } + + fn sample_message(ts_secs: i64, author: &str, text: &str) -> SlackMessage { + SlackMessage { + channel_id: "C0123456".into(), + author: author.into(), + text: text.into(), + timestamp: Utc.timestamp_opt(ts_secs, 0).single().unwrap(), + ts_raw: format!("{ts_secs}.000000"), + thread_ts: None, + } + } + + #[test] + fn bucket_to_chat_batch_builds_expected_payload() { + let channel = sample_channel(); + let start = Utc.with_ymd_and_hms(2026, 4, 25, 6, 0, 0).unwrap(); + let end = start + chrono::Duration::hours(6); + let bucket = Bucket { + start, + end, + messages: vec![ + sample_message(start.timestamp() + 60, "U1", "hello"), + sample_message(start.timestamp() + 120, "U2", "world"), + ], + }; + let batch = bucket_to_chat_batch(&bucket, &channel).unwrap(); + assert_eq!(batch.platform, "slack"); + assert_eq!(batch.channel_label, "#eng"); + assert_eq!(batch.messages.len(), 2); + assert_eq!(batch.messages[0].author, "U1"); + assert_eq!(batch.messages[0].text, "hello"); + assert!(batch.messages[0] + .source_ref + .as_deref() + .unwrap() + .starts_with("slack://archives/C0123456/")); + } + + #[test] + fn bucket_to_chat_batch_none_for_empty_messages() { + let channel = sample_channel(); + let start = Utc.with_ymd_and_hms(2026, 4, 25, 6, 0, 0).unwrap(); + let bucket = Bucket { + start, + end: start + chrono::Duration::hours(6), + messages: vec![], + }; + assert!(bucket_to_chat_batch(&bucket, &channel).is_none()); + } + + #[test] + fn channel_label_distinguishes_private() { + let pub_ch = sample_channel(); + let priv_ch = SlackChannel { + id: "G1".into(), + name: "ops".into(), + is_private: true, + }; + assert_eq!(channel_label(&pub_ch), "#eng"); + assert_eq!(channel_label(&priv_ch), "private:ops"); + } + + #[test] + fn missing_author_defaults_to_unknown() { + let channel = sample_channel(); + let start = Utc.with_ymd_and_hms(2026, 4, 25, 6, 0, 0).unwrap(); + let bucket = Bucket { + start, + end: start + chrono::Duration::hours(6), + messages: vec![sample_message(start.timestamp() + 30, "", "anon")], + }; + let batch = bucket_to_chat_batch(&bucket, &channel).unwrap(); + assert_eq!(batch.messages[0].author, "unknown"); + } + + #[test] + fn slack_archive_url_format_is_stable() { + let url = slack_archive_url("C1", "1714003200.000100"); + assert_eq!(url, "slack://archives/C1/1714003200.000100"); + } + + #[test] + fn cursor_ts_uses_bucket_end() { + let end = Utc.with_ymd_and_hms(2026, 4, 25, 12, 0, 0).unwrap(); + assert_eq!(cursor_ts_for_flushed_bucket(end), end.timestamp()); + } +} diff --git a/src/openhuman/memory/slack_ingestion/rpc.rs b/src/openhuman/memory/slack_ingestion/rpc.rs new file mode 100644 index 000000000..148b8fde2 --- /dev/null +++ b/src/openhuman/memory/slack_ingestion/rpc.rs @@ -0,0 +1,177 @@ +//! JSON-RPC handler functions for the Composio-backed Slack provider. +//! +//! Public JSON-RPC surface: +//! - `openhuman.slack_memory_sync_trigger` — run `SlackProvider::sync()` +//! once for each active Slack connection (or just one, if +//! `connection_id` is supplied). +//! - `openhuman.slack_memory_sync_status` — list the per-connection +//! sync cursors + last-synced timestamps. + +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; + +use crate::openhuman::composio::client::build_composio_client; +use crate::openhuman::composio::providers::registry::get_provider; +use crate::openhuman::composio::providers::sync_state::SyncState; +use crate::openhuman::composio::providers::{ProviderContext, SyncOutcome, SyncReason}; +use crate::openhuman::config::Config; +use crate::openhuman::memory::global::client_if_ready; +use crate::rpc::RpcOutcome; + +/// Optional connection-id override for the trigger. When absent, all +/// active Slack connections are synced (serially, one-by-one). +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct SyncTriggerRequest { + #[serde(default)] + pub connection_id: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SyncTriggerResponse { + pub outcomes: Vec, + pub connections_considered: usize, + pub connections_synced: usize, +} + +/// Run `SlackProvider::sync()` once for every active Slack connection +/// (or exactly one, if `connection_id` is provided). Fails if the +/// user is not signed in (no Composio JWT available). +pub async fn sync_trigger_rpc( + config: &Config, + req: SyncTriggerRequest, +) -> Result, String> { + let provider = get_provider("slack") + .ok_or_else(|| "[slack_ingest] SlackProvider not registered".to_string())?; + + let client = build_composio_client(config).ok_or_else(|| { + "[slack_ingest] Composio client unavailable (user not signed in?)".to_string() + })?; + + // Discover connections via the backend; filter for slack ones. + let connections = client + .list_connections() + .await + .map_err(|e| format!("[slack_ingest] list_connections failed: {e:#}"))?; + + let mut candidates: Vec<_> = connections + .connections + .into_iter() + .filter(|c| { + c.toolkit.eq_ignore_ascii_case("slack") + && matches!(c.status.as_str(), "ACTIVE" | "CONNECTED") + }) + .collect(); + + if let Some(ref wanted) = req.connection_id { + candidates.retain(|c| &c.id == wanted); + if candidates.is_empty() { + return Err(format!( + "[slack_ingest] no active Slack connection with id={wanted}" + )); + } + } + + let considered = candidates.len(); + let config_arc = Arc::new(config.clone()); + let mut outcomes: Vec = Vec::with_capacity(considered); + + for conn in candidates { + let ctx = ProviderContext { + config: Arc::clone(&config_arc), + client: client.clone(), + toolkit: conn.toolkit.clone(), + connection_id: Some(conn.id.clone()), + }; + match provider.sync(&ctx, SyncReason::Manual).await { + Ok(o) => outcomes.push(o), + Err(err) => { + log::warn!( + "[slack_ingest] connection={} sync failed: {err:#} (continuing)", + conn.id + ); + } + } + } + + let synced = outcomes.len(); + Ok(RpcOutcome::single_log( + SyncTriggerResponse { + outcomes, + connections_considered: considered, + connections_synced: synced, + }, + format!("slack_ingest: trigger considered={considered} synced={synced}"), + )) +} + +#[derive(Clone, Debug, Serialize, Deserialize, Default)] +pub struct SyncStatusRequest {} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SyncStatusResponse { + pub connections: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ConnectionStatus { + pub connection_id: String, + /// JSON-encoded per-channel cursors (see + /// `composio::providers::slack::sync::ChannelCursors`). Empty map + /// when no channels have been flushed yet. + pub per_channel_cursors: String, + pub synced_ids_count: usize, + pub requests_used_today: u32, + pub daily_request_limit: u32, +} + +/// Report one row per active Slack Composio connection, pulled from +/// the Composio sync-state KV store. +pub async fn sync_status_rpc( + config: &Config, + _req: SyncStatusRequest, +) -> Result, String> { + let client = build_composio_client(config).ok_or_else(|| { + "[slack_ingest] Composio client unavailable (user not signed in?)".to_string() + })?; + let memory = + client_if_ready().ok_or_else(|| "[slack_ingest] memory client not ready".to_string())?; + + let connections = client + .list_connections() + .await + .map_err(|e| format!("[slack_ingest] list_connections failed: {e:#}"))?; + + let mut rows = Vec::new(); + for conn in connections.connections { + if !conn.toolkit.eq_ignore_ascii_case("slack") { + continue; + } + if !matches!(conn.status.as_str(), "ACTIVE" | "CONNECTED") { + continue; + } + let state = match SyncState::load(&memory, "slack", &conn.id).await { + Ok(s) => s, + Err(err) => { + log::warn!( + "[slack_ingest] load_state connection={} failed: {err:#}", + conn.id + ); + continue; + } + }; + rows.push(ConnectionStatus { + connection_id: conn.id.clone(), + per_channel_cursors: state.cursor.clone().unwrap_or_else(|| "{}".to_string()), + synced_ids_count: state.synced_ids.len(), + requests_used_today: state.daily_budget.requests_used, + daily_request_limit: state.daily_budget.limit, + }); + } + + let count = rows.len(); + Ok(RpcOutcome::single_log( + SyncStatusResponse { connections: rows }, + format!("slack_ingest: status connections={count}"), + )) +} diff --git a/src/openhuman/memory/slack_ingestion/schemas.rs b/src/openhuman/memory/slack_ingestion/schemas.rs new file mode 100644 index 000000000..c1e9b6acc --- /dev/null +++ b/src/openhuman/memory/slack_ingestion/schemas.rs @@ -0,0 +1,128 @@ +//! Controller schemas + JSON-RPC handler dispatch for the Slack +//! ingestion path. +//! +//! Registered JSON-RPC methods (namespace `slack_memory`): +//! - `openhuman.slack_memory_sync_trigger` — run the Composio-backed +//! `SlackProvider::sync()` once per active Slack connection. +//! - `openhuman.slack_memory_sync_status` — list per-connection +//! cursor + dedup + budget state. + +use serde::de::DeserializeOwned; +use serde_json::{Map, Value}; + +use crate::core::all::{ControllerFuture, RegisteredController}; +use crate::core::{ControllerSchema, FieldSchema, TypeSchema}; +use crate::openhuman::config::rpc as config_rpc; +use crate::openhuman::memory::slack_ingestion::rpc as slack_rpc; +use crate::rpc::RpcOutcome; + +const NAMESPACE: &str = "slack_memory"; + +pub fn all_controller_schemas() -> Vec { + vec![schemas("sync_trigger"), schemas("sync_status")] +} + +pub fn all_registered_controllers() -> Vec { + vec![ + RegisteredController { + schema: schemas("sync_trigger"), + handler: handle_sync_trigger, + }, + RegisteredController { + schema: schemas("sync_status"), + handler: handle_sync_status, + }, + ] +} + +pub fn schemas(function: &str) -> ControllerSchema { + match function { + "sync_trigger" => ControllerSchema { + namespace: NAMESPACE, + function: "sync_trigger", + description: "Run the Composio-backed Slack provider sync once per active \ + Slack connection. When `connection_id` is provided, only that one \ + connection is synced.", + inputs: vec![FieldSchema { + name: "connection_id", + ty: TypeSchema::String, + comment: "Optional — restrict the trigger to one Composio connection id.", + required: false, + }], + outputs: vec![ + FieldSchema { + name: "outcomes", + ty: TypeSchema::Array(Box::new(TypeSchema::Ref("SyncOutcome"))), + comment: "Per-connection SyncOutcome records returned by SlackProvider::sync.", + required: true, + }, + FieldSchema { + name: "connections_considered", + ty: TypeSchema::I64, + comment: "Number of active Slack connections evaluated in this call.", + required: true, + }, + FieldSchema { + name: "connections_synced", + ty: TypeSchema::I64, + comment: "Number of connections whose sync completed without error.", + required: true, + }, + ], + }, + "sync_status" => ControllerSchema { + namespace: NAMESPACE, + function: "sync_status", + description: "List per-connection Slack ingestion state (cursors, synced-id \ + count, daily budget).", + inputs: vec![], + outputs: vec![FieldSchema { + name: "connections", + ty: TypeSchema::Array(Box::new(TypeSchema::Ref("ConnectionStatus"))), + comment: "One row per active Slack Composio connection.", + required: true, + }], + }, + _ => ControllerSchema { + namespace: NAMESPACE, + function: "unknown", + description: "Unknown slack_memory controller function.", + inputs: vec![FieldSchema { + name: "function", + ty: TypeSchema::String, + comment: "Unknown function requested for schema lookup.", + required: true, + }], + outputs: vec![FieldSchema { + name: "error", + ty: TypeSchema::String, + comment: "Lookup error details.", + required: true, + }], + }, + } +} + +fn handle_sync_trigger(params: Map) -> ControllerFuture { + Box::pin(async move { + let config = config_rpc::load_config_with_timeout().await?; + let req = parse_value::(Value::Object(params))?; + to_json(slack_rpc::sync_trigger_rpc(&config, req).await?) + }) +} + +fn handle_sync_status(params: Map) -> ControllerFuture { + Box::pin(async move { + let config = config_rpc::load_config_with_timeout().await?; + let req = parse_value::(Value::Object(params))?; + to_json(slack_rpc::sync_status_rpc(&config, req).await?) + }) +} + +fn parse_value(v: Value) -> Result { + serde_json::from_value(v).map_err(|e| format!("invalid params: {e}")) +} + +fn to_json(outcome: RpcOutcome) -> Result { + outcome.into_cli_compatible_json() +} diff --git a/src/openhuman/memory/slack_ingestion/types.rs b/src/openhuman/memory/slack_ingestion/types.rs new file mode 100644 index 000000000..99d33b352 --- /dev/null +++ b/src/openhuman/memory/slack_ingestion/types.rs @@ -0,0 +1,69 @@ +//! Shared types for the Slack ingestion path. +//! +//! These are intentionally independent of the Slack Web API payload +//! shape. Parsing of Composio-wrapped JSON into these structs happens in +//! [`crate::openhuman::composio::providers::slack::sync`]; everything +//! downstream only deals with the canonical types below. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +/// A single message fetched from Slack's `conversations.history`. +/// +/// The Slack API represents `ts` as a decimal string like `"1714003200.123456"` +/// where the integer part is Unix seconds and the fractional part is a +/// per-workspace message sequence. We retain the original string in `ts_raw` +/// so it can round-trip back to the API (e.g. as the `oldest` cursor on the +/// next poll, and as the permalink suffix for provenance). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SlackMessage { + /// Channel ID this message belongs to (e.g. `"C0123456"`). + pub channel_id: String, + /// Slack user ID of the author (e.g. `"U01234"`). Empty if the message + /// was a bot/system event we still want to retain (rare). + pub author: String, + /// Message body (plain text; may contain Slack-flavoured markdown). + pub text: String, + /// Canonical bucketing timestamp derived from `ts_raw` — Unix millis. + pub timestamp: DateTime, + /// Raw Slack `ts` string (used for API cursors + permalinks). + pub ts_raw: String, + /// Root thread `ts` if this message is a reply; `None` for channel-level + /// messages. Retained for future thread-aware ingestion (v2). + pub thread_ts: Option, +} + +/// A Slack channel visible to the bot, as returned by `conversations.list`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SlackChannel { + /// Channel ID (stable across renames). + pub id: String, + /// Human-readable name (e.g. `"eng"` → rendered as `"#eng"` in headers). + /// May change if admins rename the channel. + pub name: String, + /// `true` if this is a private channel the bot has been invited to. + pub is_private: bool, +} + +/// A closed time window of messages ready for ingest. +/// +/// Created by [`super::bucketer::split_closed`] when every 6-hour UTC +/// window older than `now - GRACE_PERIOD` is extracted from the buffer. +/// `messages` is non-empty by construction — empty buckets are dropped +/// before they reach this type. +#[derive(Clone, Debug)] +pub struct Bucket { + /// Inclusive bucket start (wall-clock UTC aligned to 00/06/12/18). + pub start: DateTime, + /// Exclusive bucket end (start + 6 hours). + pub end: DateTime, + /// Messages that fall in `[start, end)`, in arrival order. + pub messages: Vec, +} + +// Per-channel cursors are stored by the Composio-backed SlackProvider +// via `composio::providers::sync_state::SyncState` (a JSON-encoded +// `BTreeMap` in the `cursor` field). The +// standalone `SyncCursor` struct that previously lived here has been +// removed along with the SQLite cursor table — retention happens in +// the memory KV store now, keyed by (toolkit="slack", connection_id). diff --git a/src/openhuman/memory/tree/chunker.rs b/src/openhuman/memory/tree/chunker.rs index 44fedee66..4e0c13b38 100644 --- a/src/openhuman/memory/tree/chunker.rs +++ b/src/openhuman/memory/tree/chunker.rs @@ -66,8 +66,9 @@ pub fn chunk_markdown(input: &ChunkerInput, opts: &ChunkerOptions) -> Vec .map(|(idx, content)| { let seq = idx as u32; let token_count = approx_token_count(&content); + let id = super::types::chunk_id(input.source_kind, &input.source_id, seq, &content); Chunk { - id: super::types::chunk_id(input.source_kind, &input.source_id, seq), + id, content, metadata: input.metadata.clone(), token_count, diff --git a/src/openhuman/memory/tree/global_tree/digest_tests.rs b/src/openhuman/memory/tree/global_tree/digest_tests.rs index 8b29b422d..0db1ca750 100644 --- a/src/openhuman/memory/tree/global_tree/digest_tests.rs +++ b/src/openhuman/memory/tree/global_tree/digest_tests.rs @@ -25,7 +25,7 @@ async fn seed_source_tree_with_sealed_l1(cfg: &Config, scope: &str, ts: DateTime let summariser = InertSummariser::new(); let c1 = Chunk { - id: chunk_id(SourceKind::Chat, scope, 0), + id: chunk_id(SourceKind::Chat, scope, 0, "test-content"), content: format!("chunk 1 in {scope}"), metadata: Metadata { source_kind: SourceKind::Chat, @@ -41,7 +41,7 @@ async fn seed_source_tree_with_sealed_l1(cfg: &Config, scope: &str, ts: DateTime created_at: ts, }; let c2 = Chunk { - id: chunk_id(SourceKind::Chat, scope, 1), + id: chunk_id(SourceKind::Chat, scope, 1, "test-content"), content: format!("chunk 2 in {scope}"), metadata: Metadata { source_kind: SourceKind::Chat, diff --git a/src/openhuman/memory/tree/global_tree/recap.rs b/src/openhuman/memory/tree/global_tree/recap.rs index ed8067367..ddbccfde7 100644 --- a/src/openhuman/memory/tree/global_tree/recap.rs +++ b/src/openhuman/memory/tree/global_tree/recap.rs @@ -202,7 +202,7 @@ mod tests { let tree = get_or_create_source_tree(cfg, scope).unwrap(); let summariser = InertSummariser::new(); let c1 = Chunk { - id: chunk_id(SourceKind::Chat, scope, 0), + id: chunk_id(SourceKind::Chat, scope, 0, "test-content"), content: format!("c1-{scope}"), metadata: Metadata { source_kind: SourceKind::Chat, @@ -218,7 +218,7 @@ mod tests { created_at: ts, }; let c2 = Chunk { - id: chunk_id(SourceKind::Chat, scope, 1), + id: chunk_id(SourceKind::Chat, scope, 1, "test-content"), content: format!("c2-{scope}"), metadata: Metadata { source_kind: SourceKind::Chat, diff --git a/src/openhuman/memory/tree/ingest.rs b/src/openhuman/memory/tree/ingest.rs index 19e6311a7..91386ee3d 100644 --- a/src/openhuman/memory/tree/ingest.rs +++ b/src/openhuman/memory/tree/ingest.rs @@ -22,7 +22,7 @@ use crate::openhuman::memory::tree::chunker::{chunk_markdown, ChunkerInput, Chun use crate::openhuman::memory::tree::score::embed::{build_embedder_from_config, pack_checked}; use crate::openhuman::memory::tree::score::{self, ScoreResult, ScoringConfig}; use crate::openhuman::memory::tree::source_tree::{ - append_leaf, get_or_create_source_tree, InertSummariser, LeafRef, + append_leaf, get_or_create_source_tree, LeafRef, }; use crate::openhuman::memory::tree::store; use crate::openhuman::memory::tree::topic_tree::route_leaf_to_topic_trees; @@ -134,8 +134,11 @@ async fn persist( return Ok(IngestResult::empty(source_id)); } - // 2. Score (async; uses configured extractor) - let scoring_cfg = ScoringConfig::default_regex_only(); + // 2. Score (async; extractor chain driven by config.memory_tree — + // regex-only unless llm_extractor_endpoint + model are set, in + // which case LlmEntityExtractor runs as second-pass on borderline + // chunks with soft-fallback to regex on LLM failure). + let scoring_cfg = ScoringConfig::from_config(config); let scores = score::score_chunks(&chunks, &scoring_cfg).await?; // Fail fast on scorer length mismatch — silently truncating via zip would @@ -252,8 +255,10 @@ async fn persist( /// Push every kept chunk into its source tree. Scoped to Phase 3a — all /// chunks from one ingest batch share the same `source_id`, so they share -/// one tree lookup. The inert summariser is the default; future wiring -/// can swap in an LLM summariser here. +/// one tree lookup. The summariser comes from `build_summariser(config)`: +/// Ollama-backed [`LlmSummariser`] when `memory_tree.llm_summariser_*` is +/// set, else deterministic [`InertSummariser`]. Both share the same trait, +/// so the seal cascade doesn't notice the difference. async fn append_leaves_to_tree( config: &Config, source_id: &str, @@ -264,7 +269,7 @@ async fn append_leaves_to_tree( return Ok(()); } let tree = get_or_create_source_tree(config, source_id)?; - let summariser = InertSummariser::new(); + let summariser = crate::openhuman::memory::tree::source_tree::build_summariser(config); // Build a chunk_id → (score, entities, topics) map for quick lookup. use std::collections::HashMap; @@ -294,12 +299,14 @@ async fn append_leaves_to_tree( topics, score: score_value, }; - append_leaf(config, &tree, &leaf, &summariser).await?; + append_leaf(config, &tree, &leaf, summariser.as_ref()).await?; // Phase 3c (#709): route the leaf to every matching topic tree // and tick the curator for each entity. Non-fatal on error — // the source-tree append has already succeeded above. - if let Err(e) = route_leaf_to_topic_trees(config, &leaf, &entities, &summariser).await { + if let Err(e) = + route_leaf_to_topic_trees(config, &leaf, &entities, summariser.as_ref()).await + { log::warn!( "[memory_tree::ingest] topic_tree routing failed chunk_id={} err={:#}", chunk.id, diff --git a/src/openhuman/memory/tree/retrieval/drill_down.rs b/src/openhuman/memory/tree/retrieval/drill_down.rs index 26247fa68..9a307bfb1 100644 --- a/src/openhuman/memory/tree/retrieval/drill_down.rs +++ b/src/openhuman/memory/tree/retrieval/drill_down.rs @@ -243,7 +243,7 @@ mod tests { let mut leaf_ids: Vec = Vec::new(); for seq in 0..2u32 { let c = Chunk { - id: chunk_id(SourceKind::Chat, "slack:#eng", seq), + id: chunk_id(SourceKind::Chat, "slack:#eng", seq, "test-content"), content: format!("content-{seq}"), metadata: Metadata { source_kind: SourceKind::Chat, diff --git a/src/openhuman/memory/tree/retrieval/fetch.rs b/src/openhuman/memory/tree/retrieval/fetch.rs index 1989dedf9..3eced1209 100644 --- a/src/openhuman/memory/tree/retrieval/fetch.rs +++ b/src/openhuman/memory/tree/retrieval/fetch.rs @@ -99,7 +99,7 @@ mod tests { fn sample_chunk(source: &str, seq: u32) -> Chunk { let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(); Chunk { - id: chunk_id(SourceKind::Chat, source, seq), + id: chunk_id(SourceKind::Chat, source, seq, "test-content"), content: format!("content-{source}-{seq}"), metadata: Metadata { source_kind: SourceKind::Chat, diff --git a/src/openhuman/memory/tree/retrieval/global.rs b/src/openhuman/memory/tree/retrieval/global.rs index a93b5ac64..fd5199f29 100644 --- a/src/openhuman/memory/tree/retrieval/global.rs +++ b/src/openhuman/memory/tree/retrieval/global.rs @@ -120,7 +120,7 @@ mod tests { let summariser = InertSummariser::new(); for seq in 0..2u32 { let c = Chunk { - id: chunk_id(SourceKind::Chat, scope, seq), + id: chunk_id(SourceKind::Chat, scope, seq, "test-content"), content: format!("daily-{scope}-{seq}"), metadata: Metadata { source_kind: SourceKind::Chat, diff --git a/src/openhuman/memory/tree/retrieval/integration_test.rs b/src/openhuman/memory/tree/retrieval/integration_test.rs index 07860eca9..95cfe3c6b 100644 --- a/src/openhuman/memory/tree/retrieval/integration_test.rs +++ b/src/openhuman/memory/tree/retrieval/integration_test.rs @@ -202,7 +202,7 @@ async fn seal_populates_summary_embedding() { let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(); let mk_chunk = |seq: u32, tokens: u32| Chunk { - id: chunk_id(SourceKind::Chat, "slack:#seal-test", seq), + id: chunk_id(SourceKind::Chat, "slack:#seal-test", seq, "test-content"), content: format!("substantive chunk content {seq}"), metadata: Metadata { source_kind: SourceKind::Chat, diff --git a/src/openhuman/memory/tree/retrieval/rpc.rs b/src/openhuman/memory/tree/retrieval/rpc.rs index d5e54b51f..171ff0419 100644 --- a/src/openhuman/memory/tree/retrieval/rpc.rs +++ b/src/openhuman/memory/tree/retrieval/rpc.rs @@ -302,7 +302,7 @@ mod tests { fn sample_chunk(source: &str, seq: u32) -> Chunk { let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(); Chunk { - id: chunk_id(SourceKind::Chat, source, seq), + id: chunk_id(SourceKind::Chat, source, seq, "test-content"), content: format!("content-{source}-{seq}"), metadata: Metadata { source_kind: SourceKind::Chat, diff --git a/src/openhuman/memory/tree/retrieval/source.rs b/src/openhuman/memory/tree/retrieval/source.rs index d783519bc..46d5bc473 100644 --- a/src/openhuman/memory/tree/retrieval/source.rs +++ b/src/openhuman/memory/tree/retrieval/source.rs @@ -317,7 +317,7 @@ mod tests { let summariser = InertSummariser::new(); for seq in 0..2u32 { let c = Chunk { - id: chunk_id(SourceKind::Chat, scope, seq), + id: chunk_id(SourceKind::Chat, scope, seq, "test-content"), content: format!("payload-{scope}-{seq}"), metadata: Metadata { source_kind: SourceKind::Chat, diff --git a/src/openhuman/memory/tree/score/mod.rs b/src/openhuman/memory/tree/score/mod.rs index b3ba26893..15f17be76 100644 --- a/src/openhuman/memory/tree/score/mod.rs +++ b/src/openhuman/memory/tree/score/mod.rs @@ -104,6 +104,64 @@ impl ScoringConfig { definite_drop_threshold: DEFAULT_DEFINITE_DROP, } } + + /// Build a [`ScoringConfig`] from the workspace [`Config`]. When + /// `memory_tree.llm_extractor_endpoint` and `llm_extractor_model` + /// are both set, wires [`extract::LlmEntityExtractor`] as the + /// second-pass extractor. Otherwise falls back to + /// [`Self::default_regex_only`]. Construction errors in the LLM + /// extractor (rare — only client-builder failures) also fall back + /// to regex-only with a warn log; scoring never blocks on LLM + /// availability. + pub fn from_config(config: &crate::openhuman::config::Config) -> Self { + let endpoint = config + .memory_tree + .llm_extractor_endpoint + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()); + let model = config + .memory_tree + .llm_extractor_model + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()); + + let (Some(endpoint), Some(model)) = (endpoint, model) else { + log::debug!("[memory_tree::score] llm_extractor not configured — using regex-only"); + return Self::default_regex_only(); + }; + + let timeout_ms = config + .memory_tree + .llm_extractor_timeout_ms + .unwrap_or(15_000); + + let cfg = extract::LlmExtractorConfig { + endpoint: endpoint.to_string(), + model: model.to_string(), + timeout: std::time::Duration::from_millis(timeout_ms), + ..extract::LlmExtractorConfig::default() + }; + match extract::LlmEntityExtractor::new(cfg) { + Ok(llm) => { + log::info!( + "[memory_tree::score] using LlmEntityExtractor endpoint={} model={} timeout_ms={}", + endpoint, + model, + timeout_ms + ); + Self::with_llm_extractor(Arc::new(llm)) + } + Err(err) => { + log::warn!( + "[memory_tree::score] LlmEntityExtractor construction failed: {err:#} — \ + falling back to regex-only" + ); + Self::default_regex_only() + } + } + } } /// Compute the score for one chunk. diff --git a/src/openhuman/memory/tree/score/mod_tests.rs b/src/openhuman/memory/tree/score/mod_tests.rs index eab862f6b..ac46d0083 100644 --- a/src/openhuman/memory/tree/score/mod_tests.rs +++ b/src/openhuman/memory/tree/score/mod_tests.rs @@ -5,7 +5,7 @@ use chrono::Utc; fn test_chunk(content: &str) -> Chunk { let meta = Metadata::point_in_time(SourceKind::Email, "t1", "alice", Utc::now()); Chunk { - id: chunk_id(SourceKind::Email, "t1", 0), + id: chunk_id(SourceKind::Email, "t1", 0, "test-content"), content: content.to_string(), token_count: crate::openhuman::memory::tree::types::approx_token_count(content), metadata: meta, diff --git a/src/openhuman/memory/tree/source_tree/bucket_seal.rs b/src/openhuman/memory/tree/source_tree/bucket_seal.rs index 652fb8995..0f9f2b2c0 100644 --- a/src/openhuman/memory/tree/source_tree/bucket_seal.rs +++ b/src/openhuman/memory/tree/source_tree/bucket_seal.rs @@ -1,11 +1,20 @@ //! Append + cascade-seal for summary trees (#709). //! -//! `append_leaf` pushes a persisted chunk into the L0 buffer of a tree. If -//! the buffer's running `token_sum` crosses `TOKEN_BUDGET`, the buffer -//! seals into a level-1 summary, its items move into the summary's +//! `append_leaf` pushes a persisted chunk into the L0 buffer of a tree. +//! Seal gates differ by level: +//! +//! - **L0 (leaves → L1)**: seal when `token_sum >= TOKEN_BUDGET`. Bounds +//! the summariser's raw input. +//! - **L≥1 (summaries → next level)**: seal when `item_ids.len() >= +//! SUMMARY_FANOUT`. Per-summary token size depends on summariser +//! quality, so a token-based gate collapses to a 1:1:1 chain when the +//! summariser is weak. Counting siblings keeps the tree's fan-in +//! stable regardless. +//! +//! When a buffer seals, its items move into the new summary's //! `child_ids`, the buffer clears, and the new summary id is queued at -//! level 2. The cascade continues upward until a buffer stays under the -//! token budget. +//! the next level. The cascade continues upward until a buffer fails its +//! gate. //! //! Concurrency: Phase 3a assumes a single-process SQLite workspace. All //! writes in one seal step run in a single transaction; the async @@ -29,7 +38,7 @@ use crate::openhuman::memory::tree::source_tree::summariser::{ Summariser, SummaryContext, SummaryInput, }; use crate::openhuman::memory::tree::source_tree::types::{ - Buffer, SummaryNode, Tree, TreeKind, TOKEN_BUDGET, + Buffer, SummaryNode, Tree, TreeKind, SUMMARY_FANOUT, TOKEN_BUDGET, }; use crate::openhuman::memory::tree::store::with_connection; @@ -168,8 +177,23 @@ pub async fn cascade_all_from( Ok(sealed_ids) } +/// Level-aware seal gate. +/// +/// L0 buffers (raw leaves) gate on `token_sum` so the summariser's input +/// stays bounded. L≥1 buffers gate on sibling count so the tree's +/// fan-in is independent of per-summary token size — without this, +/// summarisers that emit at the full token budget (e.g. the inert +/// fallback) collapse the cascade into a 1:1:1 chain instead of a real +/// tree. fn should_seal(buf: &Buffer) -> bool { - !buf.is_empty() && buf.token_sum >= TOKEN_BUDGET as i64 + if buf.is_empty() { + return false; + } + if buf.level == 0 { + buf.token_sum >= TOKEN_BUDGET as i64 + } else { + (buf.item_ids.len() as u32) >= SUMMARY_FANOUT + } } /// Seal `buf` at `level` into one summary at `level + 1`. Returns the new @@ -227,8 +251,26 @@ async fn seal_one_level( // the buffer stays intact, and a retry re-embeds from scratch. The // tx below would otherwise commit a summary with no embedding, // polluting retrieval's semantic rerank. + // + // Embedder context-window guard: `nomic-embed-text-v1.5` accepts + // up to 8192 tokens of input. Summary content is bounded by + // `ctx.token_budget = TOKEN_BUDGET = 10_000` so a worst-case + // summary overshoots the embedder. We truncate the input passed + // to `embed()` to fit (the persisted summary content stays full; + // only the embedding's "view" of it is clamped). 6000 tokens + // leaves headroom for tokenizer differences across embedders. let embedder = build_embedder_from_config(config).context("build embedder during seal")?; - let embedding = embedder.embed(&output.content).await.with_context(|| { + // Conservative cap. Slack-style chat content (URLs, mentions, + // emoji) tokenizes 2-4× higher than the 4-chars/token heuristic. + // 1000 approx-tokens (~4000 chars) is comfortably under 8192 + // even at 4× tokenizer ratio. + let embed_input = truncate_for_embed(&output.content, 1_000); + log::info!( + "[source_tree::bucket_seal] embed input: original_chars={} truncated_chars={}", + output.content.len(), + embed_input.len() + ); + let embedding = embedder.embed(&embed_input).await.with_context(|| { format!( "embed summary during seal tree_id={} level={}", tree.id, level @@ -365,6 +407,21 @@ async fn seal_one_level( Ok(summary_id) } +/// Clamp `text` to roughly `max_tokens` tokens before passing to the +/// embedder. Uses the same ~4 chars/token heuristic as +/// `approx_token_count`. Embedders have hard input-size limits (e.g. +/// `nomic-embed-text-v1.5` = 8192 tokens) and an overshoot returns +/// HTTP 500 from Ollama rather than auto-truncating, which would +/// abort the seal transaction. +fn truncate_for_embed(text: &str, max_tokens: u32) -> String { + let approx = crate::openhuman::memory::tree::types::approx_token_count(text); + if approx <= max_tokens { + return text.to_string(); + } + let char_ceiling = (max_tokens as usize).saturating_mul(4); + text.chars().take(char_ceiling).collect() +} + fn refresh_last_sealed_tx( tx: &Transaction<'_>, tree_id: &str, diff --git a/src/openhuman/memory/tree/source_tree/bucket_seal_tests.rs b/src/openhuman/memory/tree/source_tree/bucket_seal_tests.rs index 41cda15a8..a39524ba4 100644 --- a/src/openhuman/memory/tree/source_tree/bucket_seal_tests.rs +++ b/src/openhuman/memory/tree/source_tree/bucket_seal_tests.rs @@ -58,7 +58,7 @@ async fn crossing_budget_triggers_seal() { // Persist two chunks that the hydrator can load during seal. let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(); let mk_chunk = |seq: u32, tokens: u32| Chunk { - id: chunk_id(SourceKind::Chat, "slack:#eng", seq), + id: chunk_id(SourceKind::Chat, "slack:#eng", seq, "test-content"), content: format!("substantive chunk content {seq}"), metadata: Metadata { source_kind: SourceKind::Chat, @@ -136,3 +136,148 @@ async fn crossing_budget_triggers_seal() { .unwrap(); assert_eq!(parent.as_deref(), Some(summary_id.as_str())); } + +#[tokio::test] +async fn fanout_at_l1_triggers_l2_seal() { + use crate::openhuman::memory::tree::source_tree::types::SUMMARY_FANOUT; + use crate::openhuman::memory::tree::store::upsert_chunks; + use crate::openhuman::memory::tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; + use chrono::TimeZone; + + let (_tmp, cfg) = test_config(); + let tree = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); + let summariser = InertSummariser::new(); + + let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(); + let mk_chunk = |seq: u32| { + let content = format!("substantive chunk content {seq}"); + Chunk { + id: chunk_id(SourceKind::Chat, "slack:#eng", seq, &content), + content, + metadata: Metadata { + source_kind: SourceKind::Chat, + source_id: "slack:#eng".into(), + owner: "alice".into(), + timestamp: ts, + time_range: (ts, ts), + tags: vec![], + source_ref: Some(SourceRef::new("slack://x")), + }, + // Each leaf alone busts TOKEN_BUDGET so the L0→L1 seal + // fires on every append. After SUMMARY_FANOUT seals, the + // L1 buffer's count-based gate trips and cascades to L2. + token_count: 10_000, + seq_in_source: seq, + created_at: ts, + } + }; + + let fanout = SUMMARY_FANOUT; + let mut all_sealed: Vec = Vec::new(); + for seq in 0..fanout { + let chunk = mk_chunk(seq); + upsert_chunks(&cfg, &[chunk.clone()]).unwrap(); + let leaf = LeafRef { + chunk_id: chunk.id.clone(), + token_count: chunk.token_count, + timestamp: ts, + content: chunk.content.clone(), + entities: vec![], + topics: vec![], + score: 0.5, + }; + let sealed = append_leaf(&cfg, &tree, &leaf, &summariser).await.unwrap(); + all_sealed.extend(sealed); + } + + // First (fanout-1) appends each emit one L1 seal. The final + // append emits an L1 seal AND cascades into one L2 seal. + assert_eq!( + all_sealed.len() as u32, + fanout + 1, + "expected {} L1 seals + 1 L2 seal, got {}", + fanout, + all_sealed.len() + ); + + let t = store::get_tree(&cfg, &tree.id).unwrap().unwrap(); + assert_eq!(t.max_level, 2, "tree should have climbed to L2"); + + let l1 = store::get_buffer(&cfg, &tree.id, 1).unwrap(); + assert!( + l1.is_empty(), + "L1 buffer should clear when the fanout seal fires" + ); + + let l2 = store::get_buffer(&cfg, &tree.id, 2).unwrap(); + assert_eq!(l2.item_ids.len(), 1, "exactly one L2 summary queued"); + + let l2_summary = store::get_summary(&cfg, &l2.item_ids[0]).unwrap().unwrap(); + assert_eq!(l2_summary.level, 2); + assert_eq!( + l2_summary.child_ids.len() as u32, + fanout, + "L2 summary should fold all {fanout} L1 children" + ); +} + +#[tokio::test] +async fn upper_level_does_not_seal_below_fanout() { + use crate::openhuman::memory::tree::source_tree::types::SUMMARY_FANOUT; + use crate::openhuman::memory::tree::store::upsert_chunks; + use crate::openhuman::memory::tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; + use chrono::TimeZone; + + let (_tmp, cfg) = test_config(); + let tree = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); + let summariser = InertSummariser::new(); + + let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(); + // Emit (fanout - 1) L1 summaries — should leave the L1 buffer + // populated but BELOW the count gate, so no L2 seal. + let stop_before = SUMMARY_FANOUT.saturating_sub(1); + for seq in 0..stop_before { + let content = format!("c{seq}"); + let chunk = Chunk { + id: chunk_id(SourceKind::Chat, "slack:#eng", seq, &content), + content, + metadata: Metadata { + source_kind: SourceKind::Chat, + source_id: "slack:#eng".into(), + owner: "alice".into(), + timestamp: ts, + time_range: (ts, ts), + tags: vec![], + source_ref: Some(SourceRef::new("slack://x")), + }, + token_count: 10_000, + seq_in_source: seq, + created_at: ts, + }; + upsert_chunks(&cfg, &[chunk.clone()]).unwrap(); + let leaf = LeafRef { + chunk_id: chunk.id, + token_count: chunk.token_count, + timestamp: ts, + content: chunk.content, + entities: vec![], + topics: vec![], + score: 0.5, + }; + let _ = append_leaf(&cfg, &tree, &leaf, &summariser).await.unwrap(); + } + + let t = store::get_tree(&cfg, &tree.id).unwrap().unwrap(); + assert_eq!(t.max_level, 1, "should plateau at L1 below fanout"); + + let l1 = store::get_buffer(&cfg, &tree.id, 1).unwrap(); + assert_eq!( + l1.item_ids.len() as u32, + stop_before, + "L1 buffer should hold the unsealed siblings" + ); + assert_eq!( + store::count_summaries(&cfg, &tree.id).unwrap(), + stop_before as u64 + ); +} diff --git a/src/openhuman/memory/tree/source_tree/flush.rs b/src/openhuman/memory/tree/source_tree/flush.rs index 7b7e4fdc7..920c47061 100644 --- a/src/openhuman/memory/tree/source_tree/flush.rs +++ b/src/openhuman/memory/tree/source_tree/flush.rs @@ -110,7 +110,7 @@ mod tests { // Persist one chunk with an old timestamp (10 days ago). let old_ts = Utc::now() - Duration::days(10); let c = Chunk { - id: chunk_id(SourceKind::Chat, "slack:#eng", 0), + id: chunk_id(SourceKind::Chat, "slack:#eng", 0, "test-content"), content: "old content that should get sealed".into(), metadata: Metadata { source_kind: SourceKind::Chat, @@ -158,7 +158,7 @@ mod tests { // Persist a leaf stamped now so it's NOT stale. let now = Utc::now(); let c = Chunk { - id: chunk_id(SourceKind::Chat, "slack:#eng", 0), + id: chunk_id(SourceKind::Chat, "slack:#eng", 0, "test-content"), content: "fresh".into(), metadata: Metadata { source_kind: SourceKind::Chat, diff --git a/src/openhuman/memory/tree/source_tree/mod.rs b/src/openhuman/memory/tree/source_tree/mod.rs index aca71e990..f97d07995 100644 --- a/src/openhuman/memory/tree/source_tree/mod.rs +++ b/src/openhuman/memory/tree/source_tree/mod.rs @@ -25,5 +25,5 @@ pub mod types; pub use bucket_seal::{append_leaf, LeafRef}; pub use registry::get_or_create_source_tree; pub use store::{get_summary_embedding, set_summary_embedding}; -pub use summariser::{inert::InertSummariser, Summariser}; +pub use summariser::{build_summariser, inert::InertSummariser, llm::LlmSummariser, Summariser}; pub use types::{Buffer, SummaryNode, Tree, TreeKind, TreeStatus, TOKEN_BUDGET}; diff --git a/src/openhuman/memory/tree/source_tree/summariser/llm.rs b/src/openhuman/memory/tree/source_tree/summariser/llm.rs new file mode 100644 index 000000000..09c8e6ba4 --- /dev/null +++ b/src/openhuman/memory/tree/source_tree/summariser/llm.rs @@ -0,0 +1,607 @@ +//! LLM-backed summariser — Ollama `/api/chat` peer of +//! [`crate::openhuman::memory::tree::score::extract::llm::LlmEntityExtractor`]. +//! +//! ## Responsibility +//! +//! When the source / topic / global tree's bucket-seal cascade decides to +//! fold N contributions (raw leaves at L0→L1, or lower-level summaries at +//! L_n→L_{n+1}), this summariser is asked to produce the parent node's +//! `content` + derived `entities` + `topics`. The seal machinery itself +//! (bucket budgeting, level promotion, `mem_tree_summaries` persistence) +//! is unchanged — only the text inside the summary row differs from +//! [`super::inert::InertSummariser`]. +//! +//! ## Soft-fallback contract +//! +//! A summariser that returns `Err` would abort the seal cascade and leave +//! the tree in an inconsistent state — a half-sealed buffer with no +//! parent row. We therefore promise **never** to return `Err`: every +//! failure (transport, HTTP status, JSON shape) falls back to the same +//! deterministic concat-and-truncate behaviour as `InertSummariser` and +//! logs a warn. Callers distinguish "LLM ran fine" from "we fell back" +//! only by observing whether the returned `entities`/`topics` are +//! populated — the inert branch emits empty vecs. +//! +//! ## Prompt shape +//! +//! The system prompt commits the model to returning JSON with the shape +//! `{ summary, entities, topics }`. We use Ollama's `format: "json"` + +//! `temperature: 0.0` to maximise determinism — same knobs the entity +//! extractor already uses with success. + +use std::time::Duration; + +use anyhow::Result; +use async_trait::async_trait; +use reqwest::Client; +use serde::{Deserialize, Serialize}; + +use super::inert::InertSummariser; +use super::{Summariser, SummaryContext, SummaryInput, SummaryOutput}; +use crate::openhuman::memory::tree::types::approx_token_count; + +/// Hard cap on summary OUTPUT tokens, regardless of the seal's +/// `ctx.token_budget`. Driven by the embedder's context window — +/// `nomic-embed-text-v1.5` accepts up to 8192 tokens and Phase 4 +/// (`source_tree::bucket_seal`) embeds the summary right after we +/// produce it. If our summary overshoots, the embedder returns 500 +/// and the whole seal transaction rolls back → no summary persists. +/// 6000 leaves a safety margin for tokenizer differences and the +/// JSON wrapper. +const MAX_SUMMARY_OUTPUT_TOKENS: u32 = 6_000; + +/// Context window we ask Ollama for. Must match the value below in +/// [`OllamaOptions::num_ctx`] so the per-input clamp computed in +/// [`LlmSummariser::summarise`] sizes inputs against the same window +/// the model actually sees. +const NUM_CTX_TOKENS: u32 = 16_384; + +/// Tokens reserved for the system prompt, JSON wrapper, and tokenizer +/// drift between our 4-chars/token heuristic and the model's tokenizer. +/// Trades a small loss of input capacity for a guarantee that the +/// prompt body + output budget never exceeds `num_ctx`. +const OVERHEAD_RESERVE_TOKENS: u32 = 512; + +/// Configuration for [`LlmSummariser`]. Endpoint + model defaults match +/// [`crate::openhuman::memory::tree::score::extract::llm::LlmExtractorConfig`] +/// so a workspace configured for one LLM path also satisfies the other +/// by default. +#[derive(Clone, Debug)] +pub struct LlmSummariserConfig { + /// Base URL of the Ollama-compatible endpoint (e.g. + /// `http://localhost:11434`). Do NOT include `/api/chat` — the + /// summariser appends it. + pub endpoint: String, + /// Model identifier (e.g. `qwen2.5:0.5b` or `llama3.1:8b`). + pub model: String, + /// Per-request timeout. Generous default because first-call weight + /// loading can be slow. + pub timeout: Duration, +} + +impl Default for LlmSummariserConfig { + fn default() -> Self { + Self { + endpoint: "http://localhost:11434".to_string(), + model: "qwen2.5:0.5b".to_string(), + // 120s — generous enough for small/medium models (1B-8B + // params) summarising the seal cascade's full token + // budget on first invocation, when Ollama may also be + // loading model weights into VRAM. Large models on CPU + // can still time out; bump via config for those. + timeout: Duration::from_secs(120), + } + } +} + +/// LLM-backed summariser. Delegates to [`InertSummariser`] on any +/// failure so seal cascades never fail. +pub struct LlmSummariser { + cfg: LlmSummariserConfig, + http: Client, + fallback: InertSummariser, +} + +impl LlmSummariser { + pub fn new(cfg: LlmSummariserConfig) -> Result { + let http = Client::builder() + .timeout(cfg.timeout) + .build() + .map_err(anyhow::Error::from)?; + Ok(Self { + cfg, + http, + fallback: InertSummariser::new(), + }) + } + + fn build_request(&self, prompt_body: &str, budget: u32) -> OllamaChatRequest { + OllamaChatRequest { + model: self.cfg.model.clone(), + messages: vec![ + OllamaMessage { + role: "system".to_string(), + content: system_prompt(budget), + }, + OllamaMessage { + role: "user".to_string(), + content: prompt_body.to_string(), + }, + ], + format: "json".to_string(), + stream: false, + options: OllamaOptions { + temperature: 0.0, + // 16k context window. Sized so that the per-input + // clamp in `summarise` (NUM_CTX - output_budget - + // overhead, divided by `inputs.len()`) keeps the + // joined prompt body inside this window even at + // upper-level seals where SUMMARY_FANOUT children + // each near MAX_SUMMARY_OUTPUT_TOKENS would otherwise + // overflow. Keeping `num_ctx` modest also keeps the + // kv-cache small enough to fit on consumer GPUs + // alongside the model weights. + num_ctx: NUM_CTX_TOKENS, + }, + } + } +} + +#[async_trait] +impl Summariser for LlmSummariser { + async fn summarise( + &self, + inputs: &[SummaryInput], + ctx: &SummaryContext<'_>, + ) -> Result { + // Clamp the model-side output budget so the summary fits the + // downstream embedder. The seal-cascade hands us + // `ctx.token_budget = 10k` by default but `nomic-embed-text` + // only accepts ≤ 8k tokens of input. Producing a smaller + // summary upfront avoids the embed-fails-after-summary + // dead end. + let effective_budget = ctx.token_budget.min(MAX_SUMMARY_OUTPUT_TOKENS); + + // Per-input clamp scaled by fanout. Without this, an upper-level + // seal feeding `SUMMARY_FANOUT=4` children each near + // `MAX_SUMMARY_OUTPUT_TOKENS` would push the prompt body alone + // past `num_ctx` and Ollama would silently truncate (or error). + // Divide the input budget evenly across contributors. + let per_input_cap = if inputs.is_empty() { + 0 + } else { + NUM_CTX_TOKENS + .saturating_sub(effective_budget) + .saturating_sub(OVERHEAD_RESERVE_TOKENS) + / inputs.len() as u32 + }; + + // Assemble the user-side prompt. We prefix each contribution with + // its id so the model can weigh them and so log diffs are + // traceable to source rows if anything looks odd. + let body = build_user_prompt(inputs, per_input_cap); + if body.trim().is_empty() { + log::debug!( + "[source_tree::summariser::llm] empty prompt body (no non-blank inputs) \ + tree_id={} level={} — returning empty summary", + ctx.tree_id, + ctx.target_level + ); + return Ok(SummaryOutput { + content: String::new(), + token_count: 0, + entities: Vec::new(), + topics: Vec::new(), + }); + } + + let url = format!("{}/api/chat", self.cfg.endpoint.trim_end_matches('/')); + let req = self.build_request(&body, effective_budget); + + log::debug!( + "[source_tree::summariser::llm] POST {url} model={} tree_id={} level={} \ + inputs={} budget={}", + self.cfg.model, + ctx.tree_id, + ctx.target_level, + inputs.len(), + ctx.token_budget + ); + + let resp = match self.http.post(&url).json(&req).send().await { + Ok(r) => r, + Err(e) => { + log::warn!( + "[source_tree::summariser::llm] transport failure to {url}: {e} — \ + falling back to inert summariser for tree_id={} level={}", + ctx.tree_id, + ctx.target_level + ); + return self.fallback.summarise(inputs, ctx).await; + } + }; + + if !resp.status().is_success() { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + log::warn!( + "[source_tree::summariser::llm] ollama non-success status {status} \ + tree_id={} level={}: {} — falling back to inert", + ctx.tree_id, + ctx.target_level, + truncate_for_log(&body, 200) + ); + return self.fallback.summarise(inputs, ctx).await; + } + + let envelope: OllamaChatResponse = match resp.json().await { + Ok(v) => v, + Err(e) => { + log::warn!( + "[source_tree::summariser::llm] response not Ollama-shaped JSON: {e} — \ + falling back to inert for tree_id={} level={}", + ctx.tree_id, + ctx.target_level + ); + return self.fallback.summarise(inputs, ctx).await; + } + }; + + let parsed: LlmSummaryOutput = match serde_json::from_str(&envelope.message.content) { + Ok(v) => v, + Err(e) => { + log::warn!( + "[source_tree::summariser::llm] model returned non-JSON or wrong-shape \ + body: {e}; content was: {} — falling back to inert", + truncate_for_log(&envelope.message.content, 400) + ); + return self.fallback.summarise(inputs, ctx).await; + } + }; + + let (content, token_count) = clamp_to_budget(&parsed.summary, effective_budget); + log::debug!( + "[source_tree::summariser::llm] sealed tree_id={} level={} inputs={} tokens={} \ + surface_entities_dropped={} topics={}", + ctx.tree_id, + ctx.target_level, + inputs.len(), + token_count, + parsed.entities.len(), + parsed.topics.len() + ); + + // Drop LLM-emitted entities. The model returns surface forms + // ("Alice", "she"), but `SummaryNode.entities` is indexed via + // `index_summary_entity_ids_tx` as canonical ids. Surface forms + // would silently corrupt that index — searches by canonical id + // would not find these summaries. Canonicalisation is the + // entity extractor's job, not the summariser's. Topics stay + // because they're free-form labels, not indexed as canonical + // ids. The `entities` field stays in the prompt to nudge the + // model toward entity-aware summarisation; we just don't + // persist its output. + Ok(SummaryOutput { + content, + token_count, + entities: Vec::new(), + topics: dedupe_sorted(parsed.topics), + }) + } +} + +/// Build the user-message body that precedes the model call. Each +/// contribution is prefixed with a short id header and separated by a +/// blank line — matches the layout the model is instructed to +/// summarise. Each input's content is clamped to +/// `per_input_cap_tokens` so the joined body fits inside `num_ctx` even +/// at upper-level seals where many large summaries fold together. A +/// `0` cap means "don't include any content" (used when there are no +/// inputs); pass `u32::MAX` to disable clamping. +fn build_user_prompt(inputs: &[SummaryInput], per_input_cap_tokens: u32) -> String { + let mut out = String::new(); + for inp in inputs { + let trimmed = inp.content.trim(); + if trimmed.is_empty() { + continue; + } + let (clamped, _) = clamp_to_budget(trimmed, per_input_cap_tokens); + if !out.is_empty() { + out.push_str("\n\n"); + } + out.push_str(&format!("[{}]\n{clamped}", inp.id)); + } + out +} + +/// System prompt. Token budget is templated in so the model aims under it. +fn system_prompt(budget: u32) -> String { + format!( + "You are a precise summariser. Summarise the user-provided contributions into a \ + single cohesive passage that preserves concrete facts, decisions, named entities, \ + and temporal ordering. Do not invent facts. Stay well under {budget} tokens.\n\ + \n\ + Return JSON only — no prose, no markdown, no commentary. Schema:\n\ + {{\n\ + \x20 \"summary\": \"\",\n\ + \x20 \"entities\": [\"\"],\n\ + \x20 \"topics\": [\"\"]\n\ + }}" + ) +} + +/// Truncate to the caller's token budget using the same ~4 chars/token +/// heuristic as [`InertSummariser`]. +fn clamp_to_budget(text: &str, budget: u32) -> (String, u32) { + let initial = approx_token_count(text); + if initial <= budget { + return (text.to_string(), initial); + } + let char_ceiling = (budget as usize).saturating_mul(4); + let truncated: String = text.chars().take(char_ceiling).collect(); + let tokens = approx_token_count(&truncated); + (truncated, tokens) +} + +fn dedupe_sorted(mut items: Vec) -> Vec { + for item in items.iter_mut() { + *item = item.trim().to_string(); + } + items.retain(|s| !s.is_empty()); + items.sort(); + items.dedup(); + items +} + +fn truncate_for_log(s: &str, max_chars: usize) -> String { + if s.chars().count() <= max_chars { + return s.to_string(); + } + let truncated: String = s.chars().take(max_chars).collect(); + format!("{truncated}…") +} + +// ── Wire types (Ollama API) ────────────────────────────────────────────── + +#[derive(Debug, Serialize)] +struct OllamaChatRequest { + model: String, + messages: Vec, + format: String, + stream: bool, + options: OllamaOptions, +} + +#[derive(Debug, Serialize)] +struct OllamaMessage { + role: String, + content: String, +} + +#[derive(Debug, Serialize)] +struct OllamaOptions { + temperature: f32, + /// Override Ollama's default 32k context window — large local + /// models inflate kv-cache to >15 GiB at 32k which won't fit on + /// consumer GPUs. 16k is plenty for one bucket summary (input is + /// bounded by the source-tree's ~10k-token bucket budget) while + /// keeping kv-cache reasonable. + num_ctx: u32, +} + +#[derive(Debug, Deserialize)] +struct OllamaChatResponse { + message: OllamaResponseMessage, +} + +#[derive(Debug, Deserialize)] +struct OllamaResponseMessage { + content: String, +} + +// ── LLM JSON output ────────────────────────────────────────────────────── + +#[derive(Debug, Deserialize)] +struct LlmSummaryOutput { + #[serde(default)] + summary: String, + #[serde(default)] + entities: Vec, + #[serde(default)] + topics: Vec, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::memory::tree::source_tree::types::TreeKind; + use chrono::Utc; + + fn sample_input(id: &str, content: &str) -> SummaryInput { + let ts = Utc::now(); + SummaryInput { + id: id.to_string(), + content: content.to_string(), + token_count: approx_token_count(content), + entities: Vec::new(), + topics: Vec::new(), + time_range_start: ts, + time_range_end: ts, + score: 0.5, + } + } + + fn test_ctx() -> SummaryContext<'static> { + SummaryContext { + tree_id: "tree-1", + tree_kind: TreeKind::Source, + target_level: 1, + token_budget: 10_000, + } + } + + #[test] + fn build_user_prompt_includes_ids_and_content() { + let inputs = vec![ + sample_input("a", "hello world"), + sample_input("b", "second contribution"), + ]; + let out = build_user_prompt(&inputs, u32::MAX); + assert!(out.contains("[a]")); + assert!(out.contains("hello world")); + assert!(out.contains("[b]")); + assert!(out.contains("second contribution")); + } + + #[test] + fn build_user_prompt_skips_blank_contributions() { + let inputs = vec![sample_input("a", " "), sample_input("b", "kept")]; + let out = build_user_prompt(&inputs, u32::MAX); + assert!(!out.contains("[a]")); + assert!(out.contains("[b]")); + assert!(out.contains("kept")); + } + + #[test] + fn build_user_prompt_clamps_each_input_to_per_input_cap() { + // Regression guard for upper-level context overflow: at L2 with + // SUMMARY_FANOUT=4 and large child summaries, the joined body + // would otherwise blow past NUM_CTX_TOKENS. The clamp keeps + // each contribution under per_input_cap_tokens regardless of + // how big the original content is. + let long = "x".repeat(2_000); // ~500 approx-tokens + let inputs = vec![ + sample_input("a", &long), + sample_input("b", &long), + sample_input("c", &long), + sample_input("d", &long), + ]; + let cap_tokens: u32 = 50; // ~200 chars per input + let out = build_user_prompt(&inputs, cap_tokens); + + // Each input contributes at most cap_tokens*4 chars of content, + // plus a small id header. Total stays well under the unclamped + // 4 * 2_000 = 8_000 chars baseline. + let unclamped_baseline = 4 * 2_000; + assert!( + out.len() < unclamped_baseline / 2, + "expected clamp to halve the body or better, got {} chars", + out.len() + ); + assert!(out.contains("[a]")); + assert!(out.contains("[d]")); + } + + #[test] + fn system_prompt_templates_budget() { + let p = system_prompt(4096); + assert!(p.contains("4096")); + assert!(p.contains("\"summary\"")); + assert!(p.contains("\"entities\"")); + assert!(p.contains("\"topics\"")); + } + + #[test] + fn clamp_to_budget_no_op_when_under() { + let (out, t) = clamp_to_budget("short", 1000); + assert_eq!(out, "short"); + assert_eq!(t, approx_token_count("short")); + } + + #[test] + fn clamp_to_budget_truncates_when_over() { + let long = "a".repeat(1000); + let (out, t) = clamp_to_budget(&long, 5); + assert!(out.len() < long.len()); + assert!(t <= 6); + } + + #[test] + fn dedupe_sorted_trims_and_dedupes() { + let out = dedupe_sorted(vec![ + "Bob".into(), + " Alice ".into(), + "Bob".into(), + "".into(), + " ".into(), + "Alice".into(), + ]); + assert_eq!(out, vec!["Alice", "Bob"]); + } + + #[test] + fn truncate_for_log_short_input_unchanged() { + assert_eq!(truncate_for_log("hi", 10), "hi"); + } + + #[test] + fn truncate_for_log_long_input_appends_ellipsis() { + let long = "x".repeat(500); + let out = truncate_for_log(&long, 10); + assert_eq!(out.chars().count(), 11); + assert!(out.ends_with('…')); + } + + #[tokio::test] + async fn empty_inputs_yield_empty_summary_without_network_call() { + // All inputs are blank → prompt body is empty → the summariser + // short-circuits and returns an empty output. Importantly, this + // path must work even if Ollama is unreachable. + let cfg = LlmSummariserConfig { + endpoint: "http://127.0.0.1:1".to_string(), + timeout: Duration::from_millis(50), + ..LlmSummariserConfig::default() + }; + let s = LlmSummariser::new(cfg).unwrap(); + let inputs = vec![sample_input("a", " "), sample_input("b", "")]; + let out = s.summarise(&inputs, &test_ctx()).await.unwrap(); + assert!(out.content.is_empty()); + assert_eq!(out.token_count, 0); + } + + #[tokio::test] + async fn transport_failure_falls_back_to_inert() { + // Unreachable endpoint → transport error → must NOT return Err; + // must fall through to InertSummariser's concatenate+truncate + // behaviour (content present, entities empty). + let cfg = LlmSummariserConfig { + endpoint: "http://127.0.0.1:1".to_string(), + timeout: Duration::from_millis(100), + ..LlmSummariserConfig::default() + }; + let s = LlmSummariser::new(cfg).unwrap(); + let inputs = vec![sample_input("a", "alice decided to ship friday")]; + let out = s.summarise(&inputs, &test_ctx()).await.unwrap(); + assert!(out.content.contains("alice decided to ship")); + // Inert branch emits empty entities/topics — this is how callers + // can distinguish fallback from a real LLM success with no entities. + assert!(out.entities.is_empty()); + assert!(out.topics.is_empty()); + } + + #[test] + fn build_request_uses_configured_model_and_json_format() { + let cfg = LlmSummariserConfig { + model: "llama3.1:8b".into(), + ..LlmSummariserConfig::default() + }; + let s = LlmSummariser::new(cfg).unwrap(); + let req = s.build_request("body", 2048); + assert_eq!(req.model, "llama3.1:8b"); + assert_eq!(req.format, "json"); + assert!(!req.stream); + assert_eq!(req.options.temperature, 0.0); + assert_eq!(req.messages[0].role, "system"); + assert!(req.messages[0].content.contains("2048")); + assert_eq!(req.messages[1].role, "user"); + assert_eq!(req.messages[1].content, "body"); + } + + #[test] + fn llm_output_deserialises_with_missing_fields() { + let v: LlmSummaryOutput = serde_json::from_str(r#"{"summary":"hi"}"#).unwrap(); + assert_eq!(v.summary, "hi"); + assert!(v.entities.is_empty()); + assert!(v.topics.is_empty()); + } +} diff --git a/src/openhuman/memory/tree/source_tree/summariser/mod.rs b/src/openhuman/memory/tree/source_tree/summariser/mod.rs index a273643b2..8dd510083 100644 --- a/src/openhuman/memory/tree/source_tree/summariser/mod.rs +++ b/src/openhuman/memory/tree/source_tree/summariser/mod.rs @@ -10,9 +10,13 @@ use anyhow::Result; use async_trait::async_trait; use chrono::{DateTime, Utc}; +use std::sync::Arc; + +use crate::openhuman::config::Config; use crate::openhuman::memory::tree::source_tree::types::TreeKind; pub mod inert; +pub mod llm; /// One contribution being folded — either a raw leaf (chunk) at L0→L1, or /// a lower-level summary at L_n→L_{n+1}. @@ -60,3 +64,66 @@ pub trait Summariser: Send + Sync { ctx: &SummaryContext<'_>, ) -> Result; } + +/// Build the summariser implementation driven by the workspace's +/// [`Config`]. When `memory_tree.llm_summariser_endpoint` and +/// `llm_summariser_model` are both set, return the Ollama-backed +/// [`llm::LlmSummariser`] (which itself soft-falls-back to inert on +/// transport failure). Otherwise return [`inert::InertSummariser`]. +/// +/// Returned as `Arc` so the ingest pipeline can pass it +/// by reference to `append_leaf` and `route_leaf_to_topic_trees` +/// without threading a generic type parameter through every caller. +pub fn build_summariser(config: &Config) -> Arc { + let endpoint = config + .memory_tree + .llm_summariser_endpoint + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()); + let model = config + .memory_tree + .llm_summariser_model + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()); + + let (Some(endpoint), Some(model)) = (endpoint, model) else { + log::debug!( + "[source_tree::summariser] llm_summariser not configured — using InertSummariser" + ); + return Arc::new(inert::InertSummariser::new()); + }; + + // 120s default — matches `LlmSummariserConfig::default()`. Lets a + // small/medium local model finish the seal-budget summary on a + // cold-loaded weight cache without spurious timeouts. + let timeout_ms = config + .memory_tree + .llm_summariser_timeout_ms + .unwrap_or(120_000); + + let cfg = llm::LlmSummariserConfig { + endpoint: endpoint.to_string(), + model: model.to_string(), + timeout: std::time::Duration::from_millis(timeout_ms), + }; + match llm::LlmSummariser::new(cfg) { + Ok(s) => { + log::info!( + "[source_tree::summariser] using LlmSummariser endpoint={} model={} timeout_ms={}", + endpoint, + model, + timeout_ms + ); + Arc::new(s) + } + Err(err) => { + log::warn!( + "[source_tree::summariser] LlmSummariser construction failed: {err:#} — \ + falling back to InertSummariser" + ); + Arc::new(inert::InertSummariser::new()) + } + } +} diff --git a/src/openhuman/memory/tree/source_tree/types.rs b/src/openhuman/memory/tree/source_tree/types.rs index 23e34e4ef..4fded7db8 100644 --- a/src/openhuman/memory/tree/source_tree/types.rs +++ b/src/openhuman/memory/tree/source_tree/types.rs @@ -174,8 +174,22 @@ impl Buffer { /// Token ceiling for one summariser invocation — aligned with the Phase 1 /// chunker ceiling so a single leaf never busts a seal on its own. +/// +/// Gates only the L0 → L1 seal: leaves are fan-in by raw token volume so +/// the summariser input stays bounded. Summaries above L0 use +/// [`SUMMARY_FANOUT`] instead — see `bucket_seal::should_seal`. pub const TOKEN_BUDGET: u32 = 10_000; +/// Sibling count that triggers a seal at level ≥ 1 (summaries → next level). +/// +/// Decouples upper-level seals from per-summary token size so the tree's +/// fan-in stays stable regardless of summariser quality. With a real +/// summariser each L1 might be ~500 tokens; with the inert fallback each +/// L1 fills the full [`TOKEN_BUDGET`]. Token-based gating collapses the +/// inert case into a 1:1:1 chain — count-based gating gives a real tree +/// shape (`SUMMARY_FANOUT` children per parent) in both cases. +pub const SUMMARY_FANOUT: u32 = 4; + /// Default age at which a non-empty buffer is force-sealed even under the /// token budget. Keeps recent activity from stalling waiting for more /// leaves that may never arrive. diff --git a/src/openhuman/memory/tree/store_tests.rs b/src/openhuman/memory/tree/store_tests.rs index 64045b9bb..698c9e243 100644 --- a/src/openhuman/memory/tree/store_tests.rs +++ b/src/openhuman/memory/tree/store_tests.rs @@ -13,7 +13,7 @@ fn test_config() -> (TempDir, Config) { fn sample_chunk(source_id: &str, seq: u32, ts_ms: i64) -> Chunk { let ts = Utc.timestamp_millis_opt(ts_ms).unwrap(); Chunk { - id: chunk_id(SourceKind::Chat, source_id, seq), + id: chunk_id(SourceKind::Chat, source_id, seq, "test-content"), content: format!("content {source_id} {seq}"), metadata: Metadata { source_kind: SourceKind::Chat, diff --git a/src/openhuman/memory/tree/topic_tree/backfill.rs b/src/openhuman/memory/tree/topic_tree/backfill.rs index e84c6b77b..e98a53e53 100644 --- a/src/openhuman/memory/tree/topic_tree/backfill.rs +++ b/src/openhuman/memory/tree/topic_tree/backfill.rs @@ -144,7 +144,7 @@ mod tests { fn mk_chunk(source_id: &str, seq: u32, ts_ms: i64, tokens: u32) -> Chunk { let ts = Utc.timestamp_millis_opt(ts_ms).unwrap(); Chunk { - id: chunk_id(SourceKind::Chat, source_id, seq), + id: chunk_id(SourceKind::Chat, source_id, seq, "test-content"), content: format!("substantive chunk mentioning alice {source_id}#{seq}"), metadata: Metadata { source_kind: SourceKind::Chat, diff --git a/src/openhuman/memory/tree/topic_tree/curator.rs b/src/openhuman/memory/tree/topic_tree/curator.rs index f5d69a2e3..ae4006272 100644 --- a/src/openhuman/memory/tree/topic_tree/curator.rs +++ b/src/openhuman/memory/tree/topic_tree/curator.rs @@ -189,7 +189,7 @@ mod tests { let ts_ms = 1_700_000_000_000 + (seq as i64) * 1_000; let ts = Utc.timestamp_millis_opt(ts_ms).unwrap(); let c = Chunk { - id: chunk_id(SourceKind::Chat, source_tree, seq), + id: chunk_id(SourceKind::Chat, source_tree, seq, "test-content"), content: format!("mentioning entity in {source_tree}#{seq}"), metadata: Metadata { source_kind: SourceKind::Chat, diff --git a/src/openhuman/memory/tree/topic_tree/routing.rs b/src/openhuman/memory/tree/topic_tree/routing.rs index 5ea491873..686756125 100644 --- a/src/openhuman/memory/tree/topic_tree/routing.rs +++ b/src/openhuman/memory/tree/topic_tree/routing.rs @@ -148,7 +148,7 @@ mod tests { fn persist_chunk(cfg: &Config, source_id: &str, seq: u32, ts_ms: i64, tokens: u32) -> String { let ts = Utc.timestamp_millis_opt(ts_ms).unwrap(); let c = Chunk { - id: chunk_id(SourceKind::Chat, source_id, seq), + id: chunk_id(SourceKind::Chat, source_id, seq, "test-content"), content: format!("chunk content {source_id} {seq}"), metadata: Metadata { source_kind: SourceKind::Chat, diff --git a/src/openhuman/memory/tree/types.rs b/src/openhuman/memory/tree/types.rs index 2b880e3e3..1d266da82 100644 --- a/src/openhuman/memory/tree/types.rs +++ b/src/openhuman/memory/tree/types.rs @@ -236,16 +236,30 @@ pub struct Chunk { /// Deterministic chunk id. /// -/// `sha256(source_kind | "\0" | source_id | "\0" | seq)` hex-encoded, first -/// 32 chars (128 bits of collision resistance). Short enough for human -/// inspection, long enough for global uniqueness in a single-user workspace. -pub fn chunk_id(source_kind: SourceKind, source_id: &str, seq_in_source: u32) -> String { +/// `sha256(source_kind | "\0" | source_id | "\0" | seq | "\0" | content)` +/// hex-encoded, first 32 chars (128 bits of collision resistance). Short +/// enough for human inspection, long enough for global uniqueness in a +/// single-user workspace. +/// +/// Content is included so multiple ingest calls that share a `source_id` +/// (e.g. successive Slack 6-hour buckets all flowing into one +/// per-connection source tree) don't collide on `seq=0,1,2,…`. Re-ingesting +/// the same canonical content under the same `(source_id, seq)` still +/// produces the same id, so upserts stay idempotent. +pub fn chunk_id( + source_kind: SourceKind, + source_id: &str, + seq_in_source: u32, + content: &str, +) -> String { let mut hasher = Sha256::new(); hasher.update(source_kind.as_str().as_bytes()); hasher.update([0u8]); hasher.update(source_id.as_bytes()); hasher.update([0u8]); hasher.update(seq_in_source.to_be_bytes()); + hasher.update([0u8]); + hasher.update(content.as_bytes()); let digest = hasher.finalize(); let hex = digest.iter().fold(String::with_capacity(64), |mut acc, b| { use std::fmt::Write; @@ -308,30 +322,40 @@ mod tests { #[test] fn chunk_id_is_deterministic() { - let a = chunk_id(SourceKind::Chat, "slack:#eng", 0); - let b = chunk_id(SourceKind::Chat, "slack:#eng", 0); + let a = chunk_id(SourceKind::Chat, "slack:#eng", 0, "hello"); + let b = chunk_id(SourceKind::Chat, "slack:#eng", 0, "hello"); assert_eq!(a, b); assert_eq!(a.len(), 32); } #[test] fn chunk_id_varies_with_seq() { - let a = chunk_id(SourceKind::Chat, "slack:#eng", 0); - let b = chunk_id(SourceKind::Chat, "slack:#eng", 1); + let a = chunk_id(SourceKind::Chat, "slack:#eng", 0, "hello"); + let b = chunk_id(SourceKind::Chat, "slack:#eng", 1, "hello"); assert_ne!(a, b); } #[test] fn chunk_id_varies_with_source_kind() { - let a = chunk_id(SourceKind::Chat, "foo", 0); - let b = chunk_id(SourceKind::Email, "foo", 0); + let a = chunk_id(SourceKind::Chat, "foo", 0, "hello"); + let b = chunk_id(SourceKind::Email, "foo", 0, "hello"); assert_ne!(a, b); } #[test] fn chunk_id_varies_with_source_id() { - let a = chunk_id(SourceKind::Chat, "x", 0); - let b = chunk_id(SourceKind::Chat, "y", 0); + let a = chunk_id(SourceKind::Chat, "x", 0, "hello"); + let b = chunk_id(SourceKind::Chat, "y", 0, "hello"); + assert_ne!(a, b); + } + + #[test] + fn chunk_id_varies_with_content() { + // Critical for the per-connection source_id design: two ingests + // sharing source_id but different content (e.g. different 6-hour + // Slack buckets) must produce distinct ids at seq=0,1,2,… + let a = chunk_id(SourceKind::Chat, "slack:c1", 0, "bucket A content"); + let b = chunk_id(SourceKind::Chat, "slack:c1", 0, "bucket B content"); assert_ne!(a, b); }