mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-27 21:08:00 +00:00
feat(slack): backfill ingestion + LLM summariser + tree fanout gate (#934)
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
sanil-23
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
cc53614747
commit
238a9a5ad9
@@ -9,6 +9,10 @@ autobins = false
|
||||
name = "openhuman-core"
|
||||
path = "src/main.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "slack-backfill"
|
||||
path = "src/bin/slack_backfill.rs"
|
||||
|
||||
[lib]
|
||||
name = "openhuman_core"
|
||||
crate-type = ["rlib"]
|
||||
|
||||
@@ -0,0 +1,409 @@
|
||||
//! Manual smoke/backfill trigger for the Composio-backed Slack
|
||||
//! provider.
|
||||
//!
|
||||
//! Invokes the same path the 15-minute periodic scheduler uses —
|
||||
//! `SlackProvider::sync()` for each active Slack Composio connection —
|
||||
//! but runs exactly **once** so operators can observe results end to
|
||||
//! end before trusting the scheduler.
|
||||
//!
|
||||
//! # Prerequisites
|
||||
//!
|
||||
//! - A working openhuman install (same workspace dir the desktop app
|
||||
//! uses) with a signed-in session JWT.
|
||||
//! - A Slack connection created via Composio's OAuth flow (e.g. from
|
||||
//! the desktop app's Integrations screen). No self-hosted Slack App
|
||||
//! or bot token is needed — authorization lives in Composio.
|
||||
//! - Ollama pulled with whatever models you want the ingest pipeline to
|
||||
//! use (embedder, LLM NER, LLM summariser). Any of these can be left
|
||||
//! unconfigured — `memory/tree/ingest` soft-falls-back per call.
|
||||
//!
|
||||
//! # Usage
|
||||
//!
|
||||
//! ```sh
|
||||
//! export OPENHUMAN_WORKSPACE=/path/to/workspace # must match desktop app
|
||||
//! export OPENHUMAN_MEMORY_EMBED_ENDPOINT=http://localhost:11434
|
||||
//! export OPENHUMAN_MEMORY_EMBED_MODEL=nomic-embed-text
|
||||
//! export OPENHUMAN_MEMORY_EXTRACT_ENDPOINT=http://localhost:11434
|
||||
//! export OPENHUMAN_MEMORY_EXTRACT_MODEL=qwen2.5:0.5b
|
||||
//! export OPENHUMAN_MEMORY_SUMMARISE_ENDPOINT=http://localhost:11434
|
||||
//! export OPENHUMAN_MEMORY_SUMMARISE_MODEL=llama3.1:8b
|
||||
//! export RUST_LOG=info,openhuman_core::openhuman::composio::providers::slack=debug,openhuman_core::openhuman::memory=debug
|
||||
//!
|
||||
//! cargo run --bin slack-backfill # all active slack connections
|
||||
//! cargo run --bin slack-backfill -- --connection conn_abc # one specific connection
|
||||
//! ```
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use anyhow::{bail, Context, Result};
|
||||
use clap::Parser;
|
||||
|
||||
use openhuman_core::openhuman::composio::client::build_composio_client;
|
||||
use openhuman_core::openhuman::composio::providers::registry::{
|
||||
get_provider, init_default_providers,
|
||||
};
|
||||
use openhuman_core::openhuman::composio::providers::slack::run_backfill_via_search;
|
||||
use openhuman_core::openhuman::composio::providers::{ProviderContext, SyncReason};
|
||||
use openhuman_core::openhuman::config::Config;
|
||||
use openhuman_core::openhuman::memory;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(
|
||||
name = "slack-backfill",
|
||||
about = "Run SlackProvider::sync() once against the user's Composio-authorized Slack connection(s)."
|
||||
)]
|
||||
struct Cli {
|
||||
/// Optional Composio connection id. When omitted, every active
|
||||
/// Slack connection is synced.
|
||||
#[arg(long = "connection")]
|
||||
connection_id: Option<String>,
|
||||
|
||||
/// Reset the per-connection SyncState before syncing — wipes the
|
||||
/// per-channel cursor map + dedup set + daily budget. The next
|
||||
/// sync re-walks the full backfill window. Useful when you've
|
||||
/// changed canonicalisation logic and want to overwrite existing
|
||||
/// chunks (chunk-id determinism makes the rewrite an UPSERT).
|
||||
#[arg(long = "reset-state", default_value_t = false)]
|
||||
reset_state: bool,
|
||||
|
||||
/// One-shot: invoke `SLACK_SEARCH_MESSAGES` with a small query and
|
||||
/// print the raw response, then exit. Probe to see if the
|
||||
/// workspace's Slack plan supports `search.messages` (paid plans
|
||||
/// only) before we consider rebuilding the provider around it.
|
||||
/// Skips the normal backfill flow.
|
||||
#[arg(long = "probe-search", default_value_t = false)]
|
||||
probe_search: bool,
|
||||
|
||||
/// Use the workspace-wide `SLACK_SEARCH_MESSAGES` path instead of
|
||||
/// per-channel `conversations.history`. Better quota efficiency
|
||||
/// (each successful call returns matches across many channels)
|
||||
/// but requires the workspace to be on a paid Slack plan.
|
||||
/// `--days` controls the backfill window.
|
||||
#[arg(long = "use-search", default_value_t = false)]
|
||||
use_search: bool,
|
||||
|
||||
/// Backfill window in days when `--use-search` is set. Defaults to
|
||||
/// 30 unless `OPENHUMAN_SLACK_BACKFILL_DAYS` overrides.
|
||||
#[arg(long = "days", default_value_t = 30)]
|
||||
days: i64,
|
||||
|
||||
/// Synthesise a tiny single-message `ChatBatch` and ingest it
|
||||
/// under the existing per-connection `source_id` to trigger a
|
||||
/// seal cascade against the existing L0 buffer (without
|
||||
/// re-fetching from Slack/Composio). Useful after fixing a seal-
|
||||
/// downstream bug — the existing 15k-token buffer immediately
|
||||
/// re-attempts cascade on the next append.
|
||||
#[arg(long = "seal-probe", default_value_t = false)]
|
||||
seal_probe: bool,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
// env_logger captures `log::*` events (used by reqwest, the
|
||||
// memory-tree pipeline, the slack ingestion ops layer, …).
|
||||
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info"))
|
||||
.format_timestamp_secs()
|
||||
.try_init()
|
||||
.ok(); // ignore double-init in test harness scenarios.
|
||||
|
||||
// tracing-subscriber captures `tracing::*` events (used by the
|
||||
// composio-side providers, including SlackProvider). Without this,
|
||||
// channel-level warn logs from `process_channel` are silent and
|
||||
// backfill failures look like silent zeros. Filter respects
|
||||
// `RUST_LOG` (e.g. `RUST_LOG=info,openhuman_core=debug`).
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
|
||||
)
|
||||
.with_target(true)
|
||||
.try_init()
|
||||
.ok();
|
||||
|
||||
let cli = Cli::parse();
|
||||
|
||||
// Load real on-disk config — same path the full core uses — so
|
||||
// `memory_tree.embedding_*`, `llm_extractor_*`, and
|
||||
// `llm_summariser_*` settings apply automatically.
|
||||
let config = Config::load_or_init()
|
||||
.await
|
||||
.context("[slack_backfill] Config::load_or_init failed")?;
|
||||
std::fs::create_dir_all(&config.workspace_dir).with_context(|| {
|
||||
format!(
|
||||
"failed to create workspace dir: {}",
|
||||
config.workspace_dir.display()
|
||||
)
|
||||
})?;
|
||||
let config = Arc::new(config);
|
||||
|
||||
// Bootstrap the memory global so `SyncState` KV reads/writes work
|
||||
// from inside `SlackProvider::sync()`. `init` is idempotent and
|
||||
// returns the (possibly pre-existing) client.
|
||||
memory::global::init(config.workspace_dir.clone())
|
||||
.map_err(|e| anyhow::anyhow!("[slack_backfill] memory::global::init failed: {e}"))?;
|
||||
|
||||
// Register the default Composio providers (gmail, notion, slack).
|
||||
// Idempotent — safe even if called twice.
|
||||
init_default_providers();
|
||||
|
||||
let provider = get_provider("slack").ok_or_else(|| {
|
||||
anyhow::anyhow!("SlackProvider not registered after init_default_providers")
|
||||
})?;
|
||||
|
||||
let client = build_composio_client(&config).ok_or_else(|| {
|
||||
anyhow::anyhow!(
|
||||
"No Composio client — user not signed in (no JWT). \
|
||||
Sign in via the desktop app first, then re-run this binary."
|
||||
)
|
||||
})?;
|
||||
|
||||
if cli.seal_probe {
|
||||
use chrono::{Duration, Utc};
|
||||
use openhuman_core::openhuman::memory::tree::canonicalize::chat::{ChatBatch, ChatMessage};
|
||||
use openhuman_core::openhuman::memory::tree::ingest::ingest_chat;
|
||||
|
||||
let connection_id = cli.connection_id.clone().ok_or_else(|| {
|
||||
anyhow::anyhow!(
|
||||
"--seal-probe requires --connection <connection_id> so the probe message \
|
||||
lands on a real Slack source tree (no implicit default)"
|
||||
)
|
||||
})?;
|
||||
let source_id = format!("slack:{connection_id}");
|
||||
let batch = ChatBatch {
|
||||
platform: "slack".into(),
|
||||
channel_label: "#seal-probe".into(),
|
||||
messages: vec![ChatMessage {
|
||||
author: "seal-probe".into(),
|
||||
timestamp: Utc::now() - Duration::days(2),
|
||||
text: format!(
|
||||
"Seal-cascade probe message at {} — triggers append_leaf \
|
||||
against the existing per-connection source tree's L0 \
|
||||
buffer (already over 10k tokens) so cascade_seals fires \
|
||||
immediately. Used to verify the LlmSummariser→embedder \
|
||||
fix without re-fetching from Composio.",
|
||||
Utc::now().to_rfc3339()
|
||||
),
|
||||
source_ref: Some("probe://seal-cascade".into()),
|
||||
}],
|
||||
};
|
||||
log::info!(
|
||||
"[slack_backfill] seal-probe: ingesting 1 message under source_id={}",
|
||||
source_id
|
||||
);
|
||||
let result = ingest_chat(
|
||||
&config,
|
||||
&source_id,
|
||||
"",
|
||||
vec!["probe".into(), "seal-cascade".into()],
|
||||
batch,
|
||||
)
|
||||
.await
|
||||
.context("[slack_backfill] seal-probe ingest_chat failed")?;
|
||||
println!(
|
||||
"seal-probe done — chunks_written={} chunks_dropped={} chunk_ids={:?}",
|
||||
result.chunks_written, result.chunks_dropped, result.chunk_ids
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if cli.probe_search {
|
||||
// Probe whether the workspace's Slack plan supports
|
||||
// `search.messages` (paid plans only). One small query, print
|
||||
// raw response, exit. Lets us decide whether to rebuild the
|
||||
// provider around SEARCH_MESSAGES (1 paginated call workspace-
|
||||
// wide) instead of per-channel `conversations.history` calls.
|
||||
let now = chrono::Utc::now();
|
||||
let after = (now - chrono::Duration::days(7))
|
||||
.format("%Y-%m-%d")
|
||||
.to_string();
|
||||
let args = serde_json::json!({
|
||||
"query": format!("after:{after}"),
|
||||
"count": 5,
|
||||
"sort": "timestamp",
|
||||
"sort_dir": "desc",
|
||||
});
|
||||
log::info!(
|
||||
"[slack_backfill] probing SLACK_SEARCH_MESSAGES with query={}",
|
||||
args["query"]
|
||||
);
|
||||
let resp = client
|
||||
.execute_tool("SLACK_SEARCH_MESSAGES", Some(args))
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("SLACK_SEARCH_MESSAGES failed: {e:#}"))?;
|
||||
println!("=== SLACK_SEARCH_MESSAGES probe ===");
|
||||
println!("successful: {}", resp.successful);
|
||||
println!("error: {:?}", resp.error);
|
||||
println!("cost_usd: {}", resp.cost_usd);
|
||||
println!("data:");
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&resp.data).unwrap_or_default()
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let connections = client
|
||||
.list_connections()
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("list_connections failed: {e:#}"))?;
|
||||
|
||||
if cli.use_search {
|
||||
let mut slack_conns: Vec<_> = connections
|
||||
.connections
|
||||
.iter()
|
||||
.filter(|c| {
|
||||
c.toolkit.eq_ignore_ascii_case("slack")
|
||||
&& matches!(c.status.as_str(), "ACTIVE" | "CONNECTED")
|
||||
})
|
||||
.cloned()
|
||||
.collect();
|
||||
if let Some(ref wanted) = cli.connection_id {
|
||||
slack_conns.retain(|c| &c.id == wanted);
|
||||
}
|
||||
if slack_conns.is_empty() {
|
||||
bail!("no active Slack connection found");
|
||||
}
|
||||
let started = Instant::now();
|
||||
let mut total_buckets = 0usize;
|
||||
for conn in &slack_conns {
|
||||
let ctx = ProviderContext {
|
||||
config: Arc::clone(&config),
|
||||
client: client.clone(),
|
||||
toolkit: conn.toolkit.clone(),
|
||||
connection_id: Some(conn.id.clone()),
|
||||
};
|
||||
match run_backfill_via_search(&ctx, cli.days).await {
|
||||
Ok(outcome) => {
|
||||
total_buckets += outcome.items_ingested;
|
||||
println!(
|
||||
"connection={} buckets={} elapsed_ms={} summary={:?}",
|
||||
conn.id,
|
||||
outcome.items_ingested,
|
||||
outcome.elapsed_ms(),
|
||||
outcome.summary,
|
||||
);
|
||||
}
|
||||
Err(err) => {
|
||||
eprintln!("connection={} search-backfill failed: {err:#}", conn.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
println!(
|
||||
"slack-backfill (search) done in {:.1}s — total_buckets={}",
|
||||
started.elapsed().as_secs_f64(),
|
||||
total_buckets
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut candidates: Vec<_> = connections
|
||||
.connections
|
||||
.into_iter()
|
||||
.filter(|c| {
|
||||
c.toolkit.eq_ignore_ascii_case("slack")
|
||||
&& matches!(c.status.as_str(), "ACTIVE" | "CONNECTED")
|
||||
})
|
||||
.collect();
|
||||
|
||||
if let Some(ref wanted) = cli.connection_id {
|
||||
candidates.retain(|c| &c.id == wanted);
|
||||
if candidates.is_empty() {
|
||||
bail!("no active Slack connection found with id={wanted}");
|
||||
}
|
||||
}
|
||||
|
||||
if candidates.is_empty() {
|
||||
bail!(
|
||||
"no active Slack connections in Composio. \
|
||||
Connect Slack from the desktop app's Integrations screen first."
|
||||
);
|
||||
}
|
||||
|
||||
log::info!(
|
||||
"[slack_backfill] workspace={} connections={} embedder={} extractor={} summariser={}",
|
||||
config.workspace_dir.display(),
|
||||
candidates.len(),
|
||||
component_status(
|
||||
&config.memory_tree.embedding_endpoint,
|
||||
&config.memory_tree.embedding_model,
|
||||
),
|
||||
component_status(
|
||||
&config.memory_tree.llm_extractor_endpoint,
|
||||
&config.memory_tree.llm_extractor_model,
|
||||
),
|
||||
component_status(
|
||||
&config.memory_tree.llm_summariser_endpoint,
|
||||
&config.memory_tree.llm_summariser_model,
|
||||
),
|
||||
);
|
||||
|
||||
let started = Instant::now();
|
||||
let mut total_buckets: usize = 0;
|
||||
let mut connections_ok: usize = 0;
|
||||
|
||||
for conn in &candidates {
|
||||
if cli.reset_state {
|
||||
let key = format!("slack:{}", conn.id);
|
||||
match memory::global::client_if_ready() {
|
||||
Some(mem) => match mem.kv_delete(Some("composio-sync-state"), &key).await {
|
||||
Ok(true) => log::info!(
|
||||
"[slack_backfill] reset SyncState for connection={} (cleared cursors)",
|
||||
conn.id
|
||||
),
|
||||
Ok(false) => log::info!(
|
||||
"[slack_backfill] no SyncState to reset for connection={}",
|
||||
conn.id
|
||||
),
|
||||
Err(e) => log::warn!(
|
||||
"[slack_backfill] reset SyncState failed for connection={}: {e:#}",
|
||||
conn.id
|
||||
),
|
||||
},
|
||||
None => {
|
||||
log::warn!("[slack_backfill] memory client not ready; skipping --reset-state")
|
||||
}
|
||||
}
|
||||
}
|
||||
let ctx = ProviderContext {
|
||||
config: Arc::clone(&config),
|
||||
client: client.clone(),
|
||||
toolkit: conn.toolkit.clone(),
|
||||
connection_id: Some(conn.id.clone()),
|
||||
};
|
||||
match provider.sync(&ctx, SyncReason::Manual).await {
|
||||
Ok(outcome) => {
|
||||
connections_ok += 1;
|
||||
total_buckets += outcome.items_ingested;
|
||||
println!(
|
||||
"connection={} buckets_flushed={} elapsed_ms={} summary={:?}",
|
||||
conn.id,
|
||||
outcome.items_ingested,
|
||||
outcome.elapsed_ms(),
|
||||
outcome.summary,
|
||||
);
|
||||
}
|
||||
Err(err) => {
|
||||
eprintln!("connection={} sync failed: {err:#}", conn.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println!(
|
||||
"slack_backfill done in {:.1}s — connections_ok={}/{} total_buckets_flushed={}",
|
||||
started.elapsed().as_secs_f64(),
|
||||
connections_ok,
|
||||
candidates.len(),
|
||||
total_buckets,
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn component_status(endpoint: &Option<String>, model: &Option<String>) -> String {
|
||||
match (endpoint.as_deref(), model.as_deref()) {
|
||||
(Some(e), Some(m)) if !e.trim().is_empty() && !m.trim().is_empty() => {
|
||||
format!("on/{}", m.trim())
|
||||
}
|
||||
_ => "off".to_string(),
|
||||
}
|
||||
}
|
||||
@@ -125,6 +125,8 @@ fn build_registered_controllers() -> Vec<RegisteredController> {
|
||||
controllers.extend(crate::openhuman::memory::all_memory_tree_registered_controllers());
|
||||
// Memory tree retrieval layer (#710 — LLM-callable read tools over the tree)
|
||||
controllers.extend(crate::openhuman::memory::all_retrieval_registered_controllers());
|
||||
// Slack → memory-tree ingestion engine (backfill + poll + 6hr bucket flush)
|
||||
controllers.extend(crate::openhuman::memory::all_slack_ingestion_registered_controllers());
|
||||
// Link shortener for long tracking URLs — saves LLM tokens
|
||||
controllers
|
||||
.extend(crate::openhuman::redirect_links::all_redirect_links_registered_controllers());
|
||||
@@ -200,6 +202,7 @@ fn build_declared_controller_schemas() -> Vec<ControllerSchema> {
|
||||
schemas.extend(crate::openhuman::memory::all_memory_controller_schemas());
|
||||
schemas.extend(crate::openhuman::memory::all_memory_tree_controller_schemas());
|
||||
schemas.extend(crate::openhuman::memory::all_retrieval_controller_schemas());
|
||||
schemas.extend(crate::openhuman::memory::all_slack_ingestion_controller_schemas());
|
||||
schemas.extend(crate::openhuman::redirect_links::all_redirect_links_controller_schemas());
|
||||
schemas.extend(crate::openhuman::referral::all_referral_controller_schemas());
|
||||
schemas.extend(crate::openhuman::billing::all_billing_controller_schemas());
|
||||
|
||||
@@ -186,6 +186,16 @@ const CAPABILITIES: &[Capability] = &[
|
||||
status: CapabilityStatus::Beta,
|
||||
privacy: None,
|
||||
},
|
||||
Capability {
|
||||
id: "intelligence.slack_memory_ingest",
|
||||
name: "Slack Memory Ingestion",
|
||||
domain: "intelligence",
|
||||
category: CapabilityCategory::Intelligence,
|
||||
description: "Backfill the last 6 days of Slack history into the memory tree and keep it up to date by flushing each closed 6-hour UTC bucket. Driven by an authenticated Slack connection (OAuth via Composio).",
|
||||
how_to: "Settings > Messaging Channels > Slack",
|
||||
status: CapabilityStatus::Beta,
|
||||
privacy: LOCAL_RAW,
|
||||
},
|
||||
Capability {
|
||||
id: "intelligence.notifications_dismiss",
|
||||
name: "Dismiss Notifications",
|
||||
|
||||
@@ -270,6 +270,10 @@ pub async fn start_channels(config: Config) -> Result<()> {
|
||||
sl.channel_id.clone(),
|
||||
sl.allowed_users.clone(),
|
||||
)));
|
||||
// Memory-tree ingestion is handled by the Composio-backed
|
||||
// `SlackProvider`, which runs inside `composio::periodic` and
|
||||
// fires per-connection on its own 15-minute cadence. No spawn
|
||||
// required here.
|
||||
}
|
||||
|
||||
if let Some(ref mm) = config.channels_config.mattermost {
|
||||
|
||||
@@ -45,6 +45,7 @@ pub mod gmail;
|
||||
pub mod notion;
|
||||
pub mod profile;
|
||||
pub mod registry;
|
||||
pub mod slack;
|
||||
pub mod sync_state;
|
||||
|
||||
/// Static toolkit → curated catalog map.
|
||||
|
||||
@@ -80,6 +80,7 @@ pub fn all_providers() -> Vec<ProviderArc> {
|
||||
pub fn init_default_providers() {
|
||||
register_provider(Arc::new(super::gmail::GmailProvider::new()));
|
||||
register_provider(Arc::new(super::notion::NotionProvider::new()));
|
||||
register_provider(Arc::new(super::slack::SlackProvider::new()));
|
||||
tracing::info!(
|
||||
count = all_providers().len(),
|
||||
"[composio:registry] default providers initialised"
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
//! Composio-backed Slack provider.
|
||||
//!
|
||||
//! The provider is wired into the periodic-sync scheduler (see
|
||||
//! [`super::registry::init_default_providers`]) and fires
|
||||
//! `SLACK_LIST_CONVERSATIONS` + `SLACK_FETCH_CONVERSATION_HISTORY`
|
||||
//! against the user's Composio-authorized Slack connection. Messages
|
||||
//! are grouped into 6-hour UTC buckets by
|
||||
//! [`crate::openhuman::memory::slack_ingestion::bucketer`] and ingested
|
||||
//! into the memory tree via
|
||||
//! [`crate::openhuman::memory::slack_ingestion::ops::ingest_bucket`].
|
||||
|
||||
mod provider;
|
||||
mod sync;
|
||||
mod users;
|
||||
|
||||
pub use provider::{run_backfill_via_search, SlackProvider, BACKFILL_DAYS};
|
||||
@@ -0,0 +1,920 @@
|
||||
//! Composio-backed Slack provider.
|
||||
//!
|
||||
//! Drives Slack history ingestion **without** a user-managed bot token
|
||||
//! — authorization lives in the user's Composio Slack connection, and
|
||||
//! the actual API calls fan out through [`ComposioClient::execute_tool`]
|
||||
//! against Composio's action catalog (`SLACK_LIST_CONVERSATIONS`,
|
||||
//! `SLACK_FETCH_CONVERSATION_HISTORY`, `SLACK_FETCH_TEAM_INFO`, …).
|
||||
//!
|
||||
//! ## Per-sync lifecycle
|
||||
//!
|
||||
//! 1. Load [`SyncState`] for `(slack, connection_id)`. `state.cursor` is
|
||||
//! a JSON-encoded [`sync::ChannelCursors`] map — Slack needs a cursor
|
||||
//! per channel, not one global watermark (Gmail's single `cursor`
|
||||
//! string wouldn't cut it). Parse failures degrade to an empty map,
|
||||
//! i.e. full backfill, which is safe because chunk IDs are
|
||||
//! deterministic.
|
||||
//! 2. Enumerate every channel the bot can read via
|
||||
//! [`ACTION_LIST_CONVERSATIONS`] with pagination.
|
||||
//! 3. For each channel, pull messages since the per-channel cursor (or
|
||||
//! `now - BACKFILL_DAYS` if no cursor yet) via
|
||||
//! [`ACTION_FETCH_HISTORY`], paginated.
|
||||
//! 4. Hand every collected message to
|
||||
//! [`slack_ingestion::bucketer::split_closed`] — produces closed
|
||||
//! 6-hour UTC buckets + a "still open" remainder (discarded; the
|
||||
//! next sync will re-fetch that window, which is cheap because we
|
||||
//! advance the cursor **only** after a bucket flushes).
|
||||
//! 5. Ingest each closed bucket via
|
||||
//! [`slack_ingestion::ops::ingest_bucket`] — canonicalise → chunk →
|
||||
//! score (with the config's Ollama entity extractor, if set) →
|
||||
//! persist → seal cascade (with the config's Ollama summariser).
|
||||
//! 6. Advance per-channel cursor to the latest flushed bucket's end
|
||||
//! timestamp; save [`SyncState`].
|
||||
//!
|
||||
//! ## Idempotency
|
||||
//!
|
||||
//! - `source_id = "slack:<channel>:<bucket_start_epoch>"` is stable
|
||||
//! across runs, so chunk IDs are deterministic and re-ingest is an
|
||||
//! UPSERT — no duplicates.
|
||||
//! - The cursor advances **only** after a bucket's `ingest_bucket`
|
||||
//! call returns `Ok`. A crash mid-fetch means the next run re-walks
|
||||
//! that range; a crash mid-ingest re-fetches the range too. Both are
|
||||
//! safe by the chunk-id property above.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde_json::{json, Value};
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
|
||||
use super::sync;
|
||||
use super::users::SlackUsers;
|
||||
use crate::openhuman::composio::client::ComposioClient;
|
||||
use crate::openhuman::composio::providers::sync_state::SyncState;
|
||||
use crate::openhuman::composio::providers::{
|
||||
pick_str, ComposioProvider, CuratedTool, ProviderContext, ProviderUserProfile, SyncOutcome,
|
||||
SyncReason,
|
||||
};
|
||||
use crate::openhuman::composio::types::ComposioExecuteResponse;
|
||||
use crate::openhuman::memory::slack_ingestion::bucketer::{
|
||||
bucket_end_for, split_closed, GRACE_PERIOD,
|
||||
};
|
||||
use crate::openhuman::memory::slack_ingestion::ops::ingest_bucket;
|
||||
use crate::openhuman::memory::slack_ingestion::types::{SlackChannel, SlackMessage};
|
||||
|
||||
/// Composio action slug for channel listing.
|
||||
const ACTION_LIST_CONVERSATIONS: &str = "SLACK_LIST_CONVERSATIONS";
|
||||
/// Composio action slug for message history.
|
||||
const ACTION_FETCH_HISTORY: &str = "SLACK_FETCH_CONVERSATION_HISTORY";
|
||||
/// Composio action slug for team/workspace profile fetch.
|
||||
const ACTION_FETCH_TEAM_INFO: &str = "SLACK_FETCH_TEAM_INFO";
|
||||
|
||||
/// Default backfill window (days) applied when a channel has no
|
||||
/// cursor yet. Override via `OPENHUMAN_SLACK_BACKFILL_DAYS` env var
|
||||
/// when a different window is needed (e.g. 30 days for a fresh
|
||||
/// workspace, or 1 day for fast smoke tests).
|
||||
pub const BACKFILL_DAYS: i64 = 6;
|
||||
|
||||
/// Resolve the active backfill window in days. Reads
|
||||
/// `OPENHUMAN_SLACK_BACKFILL_DAYS` env var if set and parseable as a
|
||||
/// positive integer; falls back to [`BACKFILL_DAYS`] otherwise.
|
||||
fn backfill_days() -> i64 {
|
||||
match std::env::var("OPENHUMAN_SLACK_BACKFILL_DAYS") {
|
||||
Ok(s) => match s.trim().parse::<i64>() {
|
||||
Ok(n) if n >= 1 => n,
|
||||
_ => {
|
||||
log::warn!(
|
||||
"[composio:slack] OPENHUMAN_SLACK_BACKFILL_DAYS={s:?} not a positive integer; \
|
||||
falling back to default {BACKFILL_DAYS}"
|
||||
);
|
||||
BACKFILL_DAYS
|
||||
}
|
||||
},
|
||||
Err(_) => BACKFILL_DAYS,
|
||||
}
|
||||
}
|
||||
|
||||
/// Max channels listed per `SLACK_LIST_CONVERSATIONS` page. Slack caps
|
||||
/// this at 1000; 200 is a safe default.
|
||||
const LIST_PAGE_SIZE: u32 = 200;
|
||||
|
||||
/// Max messages per `SLACK_FETCH_CONVERSATION_HISTORY` page. With
|
||||
/// `INTER_CALL_PACING` clamping us to 3 req/min, the marginal cost of
|
||||
/// asking for 1000 vs 200 per call is just larger response payloads —
|
||||
/// and we want to drain the 30-day window in as few calls as possible
|
||||
/// to minimise total quota burn.
|
||||
const HISTORY_PAGE_SIZE: u32 = 1000;
|
||||
|
||||
/// Stop paginating any single channel's history after this many pages
|
||||
/// so one dormant backfill can't consume the whole daily budget. With
|
||||
/// `HISTORY_PAGE_SIZE=200`, this yields ≤ 4000 messages per channel per
|
||||
/// sync — the next tick picks up the rest incrementally.
|
||||
const MAX_HISTORY_PAGES_PER_CHANNEL: u32 = 20;
|
||||
|
||||
/// Stop paginating channel listings after this many pages.
|
||||
const MAX_LIST_PAGES: u32 = 10;
|
||||
|
||||
/// Sync cadence — matches Gmail (15 minutes). Bucket flush granularity
|
||||
/// is 6 hours anyway, so tighter cadences just burn API calls.
|
||||
const SYNC_INTERVAL_SECS: u64 = 15 * 60;
|
||||
|
||||
/// Initial backoff for rate-limit retries. Slack tier-2 endpoints
|
||||
/// (`conversations.history`, `users.list`) advertise ~50 req/min but
|
||||
/// Composio's quota appears stricter; 2s gives the bucket time to
|
||||
/// refill on a `ratelimited` response.
|
||||
const RATELIMIT_INITIAL_BACKOFF: Duration = Duration::from_secs(2);
|
||||
|
||||
/// Cap on per-retry backoff. Without a ceiling, exponential growth
|
||||
/// would push individual retries into the multi-minute range and stall
|
||||
/// the whole sync.
|
||||
const RATELIMIT_MAX_BACKOFF: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Total retries for a single rate-limited call before giving up. With
|
||||
/// 2s → 4 → 8 → 16 → 30 → 30 backoff this gives ≤ 90s of grace per
|
||||
/// call; longer than the worst Composio rate-window we've observed.
|
||||
const RATELIMIT_MAX_ATTEMPTS: u32 = 6;
|
||||
|
||||
/// Fixed inter-call sleep applied after every successful execute_tool.
|
||||
/// At 20s per call we stay at 3 req/min — well below Slack's tier-2
|
||||
/// limit of 50/min and conservative enough to ride out Composio's
|
||||
/// stricter staging-tier quota replenishment. Trade-off: a full
|
||||
/// 30-day backfill across 11 channels takes 20-30 min wall-clock.
|
||||
const INTER_CALL_PACING: Duration = Duration::from_secs(20);
|
||||
|
||||
/// Resolve the JSON dump directory from `OPENHUMAN_SLACK_DUMP_DIR`.
|
||||
/// When unset, dumping is disabled. When set, every successful Composio
|
||||
/// response is mirrored to `<dir>/<scope>/<kind>-<idx>.json` so the
|
||||
/// raw payload can be replayed into `ingest_chat` later without
|
||||
/// re-burning quota.
|
||||
fn dump_dir() -> Option<PathBuf> {
|
||||
std::env::var_os("OPENHUMAN_SLACK_DUMP_DIR").map(PathBuf::from)
|
||||
}
|
||||
|
||||
/// Write a Composio response payload to disk under the dump dir. Best
|
||||
/// effort — failures are logged at warn level and never fail the sync.
|
||||
pub(super) fn dump_response(scope: &str, kind: &str, idx: u32, data: &Value) {
|
||||
let Some(base) = dump_dir() else {
|
||||
return;
|
||||
};
|
||||
let path = base.join(scope).join(format!("{kind}-{idx:04}.json"));
|
||||
if let Some(parent) = path.parent() {
|
||||
if let Err(e) = std::fs::create_dir_all(parent) {
|
||||
tracing::warn!(
|
||||
error = %e,
|
||||
path = %parent.display(),
|
||||
"[composio:slack] dump_response: create_dir_all failed (skipping dump)"
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
match serde_json::to_string_pretty(data) {
|
||||
Ok(json) => {
|
||||
if let Err(e) = std::fs::write(&path, json) {
|
||||
tracing::warn!(
|
||||
error = %e,
|
||||
path = %path.display(),
|
||||
"[composio:slack] dump_response: write failed"
|
||||
);
|
||||
} else {
|
||||
tracing::debug!(path = %path.display(), "[composio:slack] dumped response");
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "[composio:slack] dump_response: serialize failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Wrap [`ComposioClient::execute_tool`] with rate-limit-aware retry +
|
||||
/// inter-call pacing.
|
||||
///
|
||||
/// Returns `(response, attempts_made)` on first success so callers can
|
||||
/// charge the daily quota meter for **every** attempt that hit Composio,
|
||||
/// not just the one that succeeded — under throttling each retry is a
|
||||
/// real billable call. The retry only fires for `ok=false,
|
||||
/// error=ratelimited` responses; other failures pass through unchanged.
|
||||
pub(super) async fn execute_with_retry(
|
||||
client: &ComposioClient,
|
||||
slug: &str,
|
||||
args: serde_json::Value,
|
||||
description: &str,
|
||||
) -> Result<(ComposioExecuteResponse, u32), String> {
|
||||
let mut delay = RATELIMIT_INITIAL_BACKOFF;
|
||||
for attempt in 1..=RATELIMIT_MAX_ATTEMPTS {
|
||||
let resp = client
|
||||
.execute_tool(slug, Some(args.clone()))
|
||||
.await
|
||||
.map_err(|e| format!("{description}: {e:#}"))?;
|
||||
if resp.successful {
|
||||
tokio::time::sleep(INTER_CALL_PACING).await;
|
||||
return Ok((resp, attempt));
|
||||
}
|
||||
let err_str = resp.error.as_deref().unwrap_or("provider failure");
|
||||
let is_ratelimit = err_str.contains("ratelimited")
|
||||
|| err_str.contains("rate_limit")
|
||||
|| err_str.contains("rate limit");
|
||||
if is_ratelimit && attempt < RATELIMIT_MAX_ATTEMPTS {
|
||||
tracing::warn!(
|
||||
slug,
|
||||
attempt,
|
||||
max_attempts = RATELIMIT_MAX_ATTEMPTS,
|
||||
sleep_ms = delay.as_millis() as u64,
|
||||
"[composio:slack] rate-limited; backing off and retrying"
|
||||
);
|
||||
tokio::time::sleep(delay).await;
|
||||
delay = (delay * 2).min(RATELIMIT_MAX_BACKOFF);
|
||||
continue;
|
||||
}
|
||||
return Err(format!("{description}: {err_str}"));
|
||||
}
|
||||
Err(format!(
|
||||
"{description}: rate-limited after {RATELIMIT_MAX_ATTEMPTS} retries"
|
||||
))
|
||||
}
|
||||
|
||||
pub struct SlackProvider;
|
||||
|
||||
impl SlackProvider {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for SlackProvider {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ComposioProvider for SlackProvider {
|
||||
fn toolkit_slug(&self) -> &'static str {
|
||||
"slack"
|
||||
}
|
||||
|
||||
fn curated_tools(&self) -> Option<&'static [CuratedTool]> {
|
||||
Some(crate::openhuman::composio::providers::catalogs::SLACK_CURATED)
|
||||
}
|
||||
|
||||
fn sync_interval_secs(&self) -> Option<u64> {
|
||||
Some(SYNC_INTERVAL_SECS)
|
||||
}
|
||||
|
||||
async fn fetch_user_profile(
|
||||
&self,
|
||||
ctx: &ProviderContext,
|
||||
) -> Result<ProviderUserProfile, String> {
|
||||
tracing::debug!(
|
||||
connection_id = ?ctx.connection_id,
|
||||
"[composio:slack] fetch_user_profile via {ACTION_FETCH_TEAM_INFO}"
|
||||
);
|
||||
|
||||
let resp = ctx
|
||||
.client
|
||||
.execute_tool(ACTION_FETCH_TEAM_INFO, Some(json!({})))
|
||||
.await
|
||||
.map_err(|e| format!("[composio:slack] {ACTION_FETCH_TEAM_INFO} failed: {e:#}"))?;
|
||||
|
||||
if !resp.successful {
|
||||
let err = resp
|
||||
.error
|
||||
.clone()
|
||||
.unwrap_or_else(|| "provider reported failure".to_string());
|
||||
return Err(format!("[composio:slack] {ACTION_FETCH_TEAM_INFO}: {err}"));
|
||||
}
|
||||
|
||||
let data = &resp.data;
|
||||
let display_name = pick_str(data, &["data.team.name", "data.name", "team.name", "name"]);
|
||||
let profile_url = pick_str(data, &["data.team.url", "data.url", "team.url", "url"]);
|
||||
let email_domain = pick_str(
|
||||
data,
|
||||
&[
|
||||
"data.team.email_domain",
|
||||
"data.email_domain",
|
||||
"team.email_domain",
|
||||
"email_domain",
|
||||
],
|
||||
);
|
||||
let avatar_url = pick_str(
|
||||
data,
|
||||
&[
|
||||
"data.team.icon.image_132",
|
||||
"data.team.icon.image_68",
|
||||
"team.icon.image_132",
|
||||
],
|
||||
);
|
||||
|
||||
let profile = ProviderUserProfile {
|
||||
toolkit: "slack".to_string(),
|
||||
connection_id: ctx.connection_id.clone(),
|
||||
display_name,
|
||||
email: None, // Slack team_info is workspace-scoped, not user-scoped
|
||||
username: None,
|
||||
avatar_url,
|
||||
profile_url,
|
||||
extras: json!({ "email_domain": email_domain, "raw": data }),
|
||||
};
|
||||
|
||||
tracing::info!(
|
||||
connection_id = ?profile.connection_id,
|
||||
display_name = ?profile.display_name,
|
||||
"[composio:slack] fetched team info"
|
||||
);
|
||||
Ok(profile)
|
||||
}
|
||||
|
||||
async fn sync(&self, ctx: &ProviderContext, reason: SyncReason) -> Result<SyncOutcome, String> {
|
||||
let started_at_ms = sync::now_ms();
|
||||
let connection_id = ctx
|
||||
.connection_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| "default".to_string());
|
||||
|
||||
tracing::info!(
|
||||
connection_id = %connection_id,
|
||||
reason = reason.as_str(),
|
||||
"[composio:slack] sync starting"
|
||||
);
|
||||
|
||||
let Some(memory) = ctx.memory_client() else {
|
||||
return Err("[composio:slack] memory client not ready".to_string());
|
||||
};
|
||||
let mut state = SyncState::load(&memory, "slack", &connection_id).await?;
|
||||
|
||||
if state.budget_exhausted() {
|
||||
tracing::info!(
|
||||
connection_id = %connection_id,
|
||||
"[composio:slack] daily request budget exhausted, skipping sync"
|
||||
);
|
||||
return Ok(SyncOutcome {
|
||||
toolkit: "slack".to_string(),
|
||||
connection_id: Some(connection_id),
|
||||
reason: reason.as_str().to_string(),
|
||||
items_ingested: 0,
|
||||
started_at_ms,
|
||||
finished_at_ms: sync::now_ms(),
|
||||
summary: "slack sync skipped: daily budget exhausted".to_string(),
|
||||
details: json!({ "budget_exhausted": true }),
|
||||
});
|
||||
}
|
||||
|
||||
let mut cursors = sync::decode_cursors(state.cursor.as_deref());
|
||||
let now = chrono::Utc::now();
|
||||
|
||||
// Pull the workspace user directory once per sync so author
|
||||
// ids and `<@…>` mentions in message text canonicalise to
|
||||
// human-readable names. Soft-fails to an empty cache; raw ids
|
||||
// simply pass through in that case.
|
||||
let (users, user_call_count) = SlackUsers::fetch(&ctx.client).await;
|
||||
state.record_requests(user_call_count);
|
||||
tracing::info!(
|
||||
connection_id = %connection_id,
|
||||
user_count = users.len(),
|
||||
"[composio:slack] users cached for this sync"
|
||||
);
|
||||
|
||||
// 1. Enumerate channels ────────────────────────────────────────
|
||||
let channels = list_all_channels(ctx, &mut state)
|
||||
.await
|
||||
.map_err(|e| format!("[composio:slack] list_channels: {e:#}"))?;
|
||||
|
||||
tracing::info!(
|
||||
connection_id = %connection_id,
|
||||
channel_count = channels.len(),
|
||||
"[composio:slack] channels discovered"
|
||||
);
|
||||
|
||||
// Save budget state early so a panic mid-fetch doesn't leak the
|
||||
// request counter. Cheap.
|
||||
let _ = state.save(&memory).await;
|
||||
|
||||
let mut total_flushed_buckets: usize = 0;
|
||||
let mut channels_processed: usize = 0;
|
||||
let mut channels_errored: usize = 0;
|
||||
|
||||
// 2. Per-channel: fetch → bucket → ingest → advance cursor ────
|
||||
for channel in &channels {
|
||||
if state.budget_exhausted() {
|
||||
tracing::warn!(
|
||||
connection_id = %connection_id,
|
||||
channel = %channel.id,
|
||||
"[composio:slack] budget exhausted mid-sync, remaining channels deferred"
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
match process_channel(ctx, &mut state, channel, &mut cursors, now, &users).await {
|
||||
Ok(n) => {
|
||||
total_flushed_buckets += n;
|
||||
channels_processed += 1;
|
||||
}
|
||||
Err(err) => {
|
||||
channels_errored += 1;
|
||||
tracing::warn!(
|
||||
connection_id = %connection_id,
|
||||
channel = %channel.id,
|
||||
error = %err,
|
||||
"[composio:slack] channel sync failed (continuing with next channel)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Save after every channel — a crash between channels
|
||||
// shouldn't lose already-advanced cursors.
|
||||
state.advance_cursor(sync::encode_cursors(&cursors));
|
||||
if let Err(err) = state.save(&memory).await {
|
||||
tracing::warn!(
|
||||
error = %err,
|
||||
"[composio:slack] state save failed after channel (non-fatal)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let finished_at_ms = sync::now_ms();
|
||||
let summary = format!(
|
||||
"slack sync: channels_processed={channels_processed} \
|
||||
channels_errored={channels_errored} \
|
||||
buckets_flushed={total_flushed_buckets}"
|
||||
);
|
||||
tracing::info!(
|
||||
connection_id = %connection_id,
|
||||
elapsed_ms = finished_at_ms.saturating_sub(started_at_ms),
|
||||
"{summary}"
|
||||
);
|
||||
|
||||
Ok(SyncOutcome {
|
||||
toolkit: "slack".to_string(),
|
||||
connection_id: Some(connection_id),
|
||||
reason: reason.as_str().to_string(),
|
||||
items_ingested: total_flushed_buckets,
|
||||
started_at_ms,
|
||||
finished_at_ms,
|
||||
summary,
|
||||
details: json!({
|
||||
"channels_processed": channels_processed,
|
||||
"channels_errored": channels_errored,
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
async fn on_trigger(
|
||||
&self,
|
||||
ctx: &ProviderContext,
|
||||
trigger: &str,
|
||||
_payload: &Value,
|
||||
) -> Result<(), String> {
|
||||
// Slack trigger names use SLACK_RECEIVE_MESSAGE / similar —
|
||||
// match loosely so future slug changes don't silently drop this.
|
||||
if trigger.to_ascii_uppercase().contains("MESSAGE") {
|
||||
if let Err(e) = self.sync(ctx, SyncReason::Manual).await {
|
||||
tracing::warn!(
|
||||
error = %e,
|
||||
"[composio:slack] trigger-driven sync failed (non-fatal)"
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Paginate through `SLACK_LIST_CONVERSATIONS` and flatten into a
|
||||
/// single `Vec<SlackChannel>`. Aborts early if the daily budget runs out
|
||||
/// partway through — callers handle the partial list gracefully.
|
||||
async fn list_all_channels(
|
||||
ctx: &ProviderContext,
|
||||
state: &mut SyncState,
|
||||
) -> Result<Vec<SlackChannel>, String> {
|
||||
let mut out: Vec<SlackChannel> = Vec::new();
|
||||
let mut cursor: Option<String> = None;
|
||||
|
||||
for page_num in 0..MAX_LIST_PAGES {
|
||||
if state.budget_exhausted() {
|
||||
tracing::warn!(
|
||||
page = page_num,
|
||||
"[composio:slack] budget exhausted during channel listing"
|
||||
);
|
||||
break;
|
||||
}
|
||||
let mut args = json!({
|
||||
"types": "public_channel,private_channel",
|
||||
"exclude_archived": true,
|
||||
"limit": LIST_PAGE_SIZE,
|
||||
});
|
||||
if let Some(ref c) = cursor {
|
||||
args["cursor"] = json!(c);
|
||||
}
|
||||
|
||||
let (resp, attempts) = execute_with_retry(
|
||||
&ctx.client,
|
||||
ACTION_LIST_CONVERSATIONS,
|
||||
args,
|
||||
&format!("{ACTION_LIST_CONVERSATIONS} page {page_num}"),
|
||||
)
|
||||
.await?;
|
||||
state.record_requests(attempts);
|
||||
dump_response("_meta", "channels", page_num, &resp.data);
|
||||
|
||||
out.extend(sync::extract_channels(&resp.data));
|
||||
cursor = sync::extract_next_cursor(&resp.data);
|
||||
if cursor.is_none() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Pull one channel's history since its cursor, bucket it, and ingest
|
||||
/// every closed bucket. Returns the number of buckets actually flushed.
|
||||
async fn process_channel(
|
||||
ctx: &ProviderContext,
|
||||
state: &mut SyncState,
|
||||
channel: &SlackChannel,
|
||||
cursors: &mut sync::ChannelCursors,
|
||||
now: chrono::DateTime<chrono::Utc>,
|
||||
users: &SlackUsers,
|
||||
) -> Result<usize, String> {
|
||||
// Derive `oldest` — per-channel cursor if we've synced before,
|
||||
// else `now - backfill_days()` for a fresh channel.
|
||||
let oldest_secs = cursors
|
||||
.get(&channel.id)
|
||||
.copied()
|
||||
.unwrap_or_else(|| (now - chrono::Duration::days(backfill_days())).timestamp());
|
||||
|
||||
let mut all_messages: Vec<SlackMessage> = Vec::new();
|
||||
let mut cursor: Option<String> = None;
|
||||
|
||||
for page_num in 0..MAX_HISTORY_PAGES_PER_CHANNEL {
|
||||
if state.budget_exhausted() {
|
||||
tracing::warn!(
|
||||
channel = %channel.id,
|
||||
page = page_num,
|
||||
"[composio:slack] budget exhausted during history fetch"
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
let mut args = json!({
|
||||
"channel": channel.id,
|
||||
"oldest": format!("{oldest_secs}.000000"),
|
||||
"inclusive": false,
|
||||
"limit": HISTORY_PAGE_SIZE,
|
||||
});
|
||||
if let Some(ref c) = cursor {
|
||||
args["cursor"] = json!(c);
|
||||
}
|
||||
|
||||
let (resp, attempts) = execute_with_retry(
|
||||
&ctx.client,
|
||||
ACTION_FETCH_HISTORY,
|
||||
args,
|
||||
&format!(
|
||||
"{ACTION_FETCH_HISTORY} channel={} page {page_num}",
|
||||
channel.id
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
state.record_requests(attempts);
|
||||
dump_response(&channel.id, "history", page_num, &resp.data);
|
||||
|
||||
let msgs = sync::extract_messages(&resp.data, &channel.id, users);
|
||||
tracing::debug!(
|
||||
channel = %channel.id,
|
||||
page = page_num,
|
||||
fetched = msgs.len(),
|
||||
"[composio:slack] history page"
|
||||
);
|
||||
if msgs.is_empty() {
|
||||
break;
|
||||
}
|
||||
all_messages.extend(msgs);
|
||||
cursor = sync::extract_next_cursor(&resp.data);
|
||||
if cursor.is_none() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if all_messages.is_empty() {
|
||||
tracing::debug!(
|
||||
channel = %channel.id,
|
||||
"[composio:slack] no new messages"
|
||||
);
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
// Bucket — closed buckets ingest now; still-open ones get discarded
|
||||
// and will be re-fetched on the next tick (cursor only advances on
|
||||
// successful flush).
|
||||
let (closed, remaining) = split_closed(all_messages, now, GRACE_PERIOD);
|
||||
tracing::debug!(
|
||||
channel = %channel.id,
|
||||
closed_buckets = closed.len(),
|
||||
remaining_msgs = remaining.len(),
|
||||
"[composio:slack] bucket split"
|
||||
);
|
||||
|
||||
let mut flushed = 0usize;
|
||||
let mut latest_end: Option<chrono::DateTime<chrono::Utc>> = None;
|
||||
let connection_id = ctx.connection_id.as_deref().unwrap_or("default");
|
||||
|
||||
for bucket in closed {
|
||||
match ingest_bucket(&ctx.config, channel, &bucket, "", connection_id).await {
|
||||
Ok(res) => {
|
||||
flushed += 1;
|
||||
latest_end = Some(bucket.end);
|
||||
tracing::info!(
|
||||
channel = %channel.id,
|
||||
bucket_start = %bucket.start.to_rfc3339(),
|
||||
chunks_written = res.chunks_written,
|
||||
"[composio:slack] ingested bucket"
|
||||
);
|
||||
}
|
||||
Err(err) => {
|
||||
tracing::warn!(
|
||||
channel = %channel.id,
|
||||
bucket_start = %bucket.start.to_rfc3339(),
|
||||
error = %err,
|
||||
"[composio:slack] ingest failed (cursor not advanced for this bucket)"
|
||||
);
|
||||
// Stop processing more buckets in this channel — the
|
||||
// next run will re-fetch from the current cursor.
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(end) = latest_end {
|
||||
cursors.insert(channel.id.clone(), end.timestamp());
|
||||
}
|
||||
// Ensure bucket_end_for stays referenced (future-proofing for
|
||||
// cascade-aware flushes) without dead-code.
|
||||
let _ = bucket_end_for;
|
||||
|
||||
Ok(flushed)
|
||||
}
|
||||
|
||||
// ── Search-based backfill (one-shot) ────────────────────────────────
|
||||
|
||||
/// Composio action slug for workspace-wide message search.
|
||||
const ACTION_SEARCH_MESSAGES: &str = "SLACK_SEARCH_MESSAGES";
|
||||
|
||||
/// Max matches per `SLACK_SEARCH_MESSAGES` page (Slack's documented cap).
|
||||
const SEARCH_PAGE_SIZE: u32 = 100;
|
||||
|
||||
/// Hard cap on pages walked per backfill run. With 100 matches/page
|
||||
/// that's 5000 messages — plenty for typical 30-day windows on small
|
||||
/// workspaces. Larger backfills can run multiple times against
|
||||
/// successive sub-windows.
|
||||
const MAX_SEARCH_PAGES: u32 = 50;
|
||||
|
||||
/// Run a one-shot historical backfill via `SLACK_SEARCH_MESSAGES` —
|
||||
/// workspace-wide paginated search instead of per-channel
|
||||
/// `conversations.history`. Better quota efficiency: each successful
|
||||
/// call returns matches across many channels at once, so partial
|
||||
/// progress translates to real coverage instead of one channel's
|
||||
/// worth.
|
||||
///
|
||||
/// Designed for the `slack-backfill` bin specifically — the periodic
|
||||
/// `SlackProvider::sync()` keeps the per-channel incremental path so
|
||||
/// it stays cheap on each tick.
|
||||
///
|
||||
/// Lifecycle:
|
||||
/// 1. Cache the channel directory (one `SLACK_LIST_CONVERSATIONS` call,
|
||||
/// paginated) so canonicalisation can label channels by name + know
|
||||
/// private-vs-public.
|
||||
/// 2. Cache the user directory (one `SLACK_LIST_ALL_USERS` paginated
|
||||
/// walk via [`SlackUsers::fetch`]).
|
||||
/// 3. Paginate `SLACK_SEARCH_MESSAGES` with
|
||||
/// `query = "after:<YYYY-MM-DD>"` until exhausted or page cap.
|
||||
/// 4. Group every message by `(channel_id, 6hr_bucket_start)`.
|
||||
/// 5. For each closed bucket, hand off to `ingest_bucket` (same
|
||||
/// canonicalise → chunk → score → seal-cascade path the periodic
|
||||
/// sync uses).
|
||||
pub async fn run_backfill_via_search(
|
||||
ctx: &ProviderContext,
|
||||
backfill_days: i64,
|
||||
) -> Result<SyncOutcome, String> {
|
||||
let started_at_ms = sync::now_ms();
|
||||
let connection_id = ctx
|
||||
.connection_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| "default".to_string());
|
||||
|
||||
tracing::info!(
|
||||
connection_id = %connection_id,
|
||||
backfill_days,
|
||||
"[composio:slack] search-based backfill starting"
|
||||
);
|
||||
|
||||
let memory = ctx
|
||||
.memory_client()
|
||||
.ok_or_else(|| "[composio:slack] memory client not ready".to_string())?;
|
||||
let mut state = SyncState::load(&memory, "slack", &connection_id).await?;
|
||||
|
||||
if state.budget_exhausted() {
|
||||
return Ok(SyncOutcome {
|
||||
toolkit: "slack".to_string(),
|
||||
connection_id: Some(connection_id),
|
||||
reason: SyncReason::Manual.as_str().to_string(),
|
||||
items_ingested: 0,
|
||||
started_at_ms,
|
||||
finished_at_ms: sync::now_ms(),
|
||||
summary: "slack search-backfill skipped: daily budget exhausted".to_string(),
|
||||
details: json!({ "budget_exhausted": true }),
|
||||
});
|
||||
}
|
||||
|
||||
// 1. Channel directory — needed for canonicalisation (name +
|
||||
// is_private flag come off `SlackChannel`).
|
||||
let channels = list_all_channels(ctx, &mut state)
|
||||
.await
|
||||
.map_err(|e| format!("[composio:slack] list_channels: {e:#}"))?;
|
||||
let channel_map: HashMap<String, SlackChannel> =
|
||||
channels.into_iter().map(|c| (c.id.clone(), c)).collect();
|
||||
|
||||
// 2. User directory — for ID → display-name resolution + mention
|
||||
// rewrites.
|
||||
let (users, user_call_count) = SlackUsers::fetch(&ctx.client).await;
|
||||
state.record_requests(user_call_count);
|
||||
tracing::info!(
|
||||
connection_id = %connection_id,
|
||||
user_count = users.len(),
|
||||
channel_count = channel_map.len(),
|
||||
"[composio:slack] caches ready"
|
||||
);
|
||||
let _ = state.save(&memory).await;
|
||||
|
||||
// 3. Paginated workspace-wide search.
|
||||
let now = chrono::Utc::now();
|
||||
let after = (now - chrono::Duration::days(backfill_days))
|
||||
.format("%Y-%m-%d")
|
||||
.to_string();
|
||||
let query = format!("after:{after}");
|
||||
let mut all_messages: Vec<SlackMessage> = Vec::new();
|
||||
let mut page: u32 = 1;
|
||||
let mut total_pages: u32 = 1;
|
||||
|
||||
loop {
|
||||
if state.budget_exhausted() {
|
||||
tracing::warn!(
|
||||
page,
|
||||
"[composio:slack] budget exhausted mid-search, halting"
|
||||
);
|
||||
break;
|
||||
}
|
||||
let args = json!({
|
||||
"query": query,
|
||||
"count": SEARCH_PAGE_SIZE,
|
||||
"sort": "timestamp",
|
||||
"sort_dir": "asc",
|
||||
"page": page,
|
||||
});
|
||||
let (resp, attempts) = execute_with_retry(
|
||||
&ctx.client,
|
||||
ACTION_SEARCH_MESSAGES,
|
||||
args,
|
||||
&format!("{ACTION_SEARCH_MESSAGES} page {page}"),
|
||||
)
|
||||
.await?;
|
||||
state.record_requests(attempts);
|
||||
dump_response("_meta", "search", page, &resp.data);
|
||||
|
||||
let msgs = sync::extract_search_messages(&resp.data, &users);
|
||||
if page == 1 {
|
||||
total_pages = sync::extract_search_total_pages(&resp.data).min(MAX_SEARCH_PAGES);
|
||||
tracing::info!(
|
||||
connection_id = %connection_id,
|
||||
total_pages,
|
||||
first_page_msgs = msgs.len(),
|
||||
"[composio:slack] search pagination plan"
|
||||
);
|
||||
}
|
||||
let fetched = msgs.len();
|
||||
all_messages.extend(msgs);
|
||||
if fetched == 0 || page >= total_pages {
|
||||
break;
|
||||
}
|
||||
page += 1;
|
||||
}
|
||||
let _ = state.save(&memory).await;
|
||||
|
||||
// 4. Group messages by (channel, 6hr-bucket-start).
|
||||
let mut by_bucket: BTreeMap<(String, chrono::DateTime<chrono::Utc>), Vec<SlackMessage>> =
|
||||
BTreeMap::new();
|
||||
for msg in all_messages {
|
||||
let bucket_start =
|
||||
crate::openhuman::memory::slack_ingestion::bucketer::bucket_start_for(msg.timestamp);
|
||||
by_bucket
|
||||
.entry((msg.channel_id.clone(), bucket_start))
|
||||
.or_default()
|
||||
.push(msg);
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
connection_id = %connection_id,
|
||||
total_buckets = by_bucket.len(),
|
||||
"[composio:slack] grouped messages into buckets"
|
||||
);
|
||||
|
||||
// 5. Ingest each closed bucket.
|
||||
let mut buckets_flushed = 0usize;
|
||||
let mut buckets_skipped_open = 0usize;
|
||||
let mut buckets_skipped_unknown_channel = 0usize;
|
||||
let mut buckets_failed = 0usize;
|
||||
|
||||
for ((channel_id, bucket_start), mut msgs) in by_bucket {
|
||||
let Some(channel) = channel_map.get(&channel_id) else {
|
||||
buckets_skipped_unknown_channel += 1;
|
||||
continue;
|
||||
};
|
||||
msgs.sort_by_key(|m| m.timestamp);
|
||||
let bucket = crate::openhuman::memory::slack_ingestion::types::Bucket {
|
||||
start: bucket_start,
|
||||
end: bucket_end_for(bucket_start),
|
||||
messages: msgs,
|
||||
};
|
||||
// Skip still-open buckets — same closed-bucket invariant the
|
||||
// periodic sync enforces.
|
||||
if bucket.end + crate::openhuman::memory::slack_ingestion::bucketer::GRACE_PERIOD > now {
|
||||
buckets_skipped_open += 1;
|
||||
continue;
|
||||
}
|
||||
match ingest_bucket(&ctx.config, channel, &bucket, "", &connection_id).await {
|
||||
Ok(res) => {
|
||||
buckets_flushed += 1;
|
||||
tracing::info!(
|
||||
channel = %channel.id,
|
||||
bucket_start = %bucket.start.to_rfc3339(),
|
||||
messages = bucket.messages.len(),
|
||||
chunks_written = res.chunks_written,
|
||||
"[composio:slack] ingested bucket"
|
||||
);
|
||||
}
|
||||
Err(err) => {
|
||||
buckets_failed += 1;
|
||||
tracing::warn!(
|
||||
channel = %channel.id,
|
||||
bucket_start = %bucket.start.to_rfc3339(),
|
||||
error = %err,
|
||||
"[composio:slack] ingest_bucket failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let finished_at_ms = sync::now_ms();
|
||||
let summary = format!(
|
||||
"slack search-backfill: pages={page} buckets_flushed={buckets_flushed} \
|
||||
buckets_open={buckets_skipped_open} \
|
||||
unknown_channel={buckets_skipped_unknown_channel} \
|
||||
failed={buckets_failed}"
|
||||
);
|
||||
tracing::info!(
|
||||
connection_id = %connection_id,
|
||||
elapsed_ms = finished_at_ms.saturating_sub(started_at_ms),
|
||||
"{summary}"
|
||||
);
|
||||
|
||||
Ok(SyncOutcome {
|
||||
toolkit: "slack".to_string(),
|
||||
connection_id: Some(connection_id),
|
||||
reason: SyncReason::Manual.as_str().to_string(),
|
||||
items_ingested: buckets_flushed,
|
||||
started_at_ms,
|
||||
finished_at_ms,
|
||||
summary,
|
||||
details: json!({
|
||||
"pages_walked": page,
|
||||
"buckets_flushed": buckets_flushed,
|
||||
"buckets_open": buckets_skipped_open,
|
||||
"buckets_unknown_channel": buckets_skipped_unknown_channel,
|
||||
"buckets_failed": buckets_failed,
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn toolkit_slug_is_stable() {
|
||||
assert_eq!(SlackProvider::new().toolkit_slug(), "slack");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_interval_matches_constant() {
|
||||
assert_eq!(
|
||||
SlackProvider::new().sync_interval_secs(),
|
||||
Some(SYNC_INTERVAL_SECS)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn curated_tools_returns_slack_catalog() {
|
||||
let tools = SlackProvider::new().curated_tools().unwrap();
|
||||
assert!(tools
|
||||
.iter()
|
||||
.any(|t| t.slug == "SLACK_FETCH_CONVERSATION_HISTORY"));
|
||||
assert!(tools.iter().any(|t| t.slug == "SLACK_LIST_CONVERSATIONS"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,397 @@
|
||||
//! Helpers for the Composio-backed Slack provider.
|
||||
//!
|
||||
//! Split out from `provider.rs` so the response-shape parsing code can
|
||||
//! evolve independently of the sync orchestration — Composio (and Slack
|
||||
//! beneath it) periodically widens response envelopes, and keeping the
|
||||
//! JSON-pointer walks in one file makes adding new paths cheap.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use chrono::{DateTime, TimeZone, Utc};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::users::SlackUsers;
|
||||
use crate::openhuman::memory::slack_ingestion::types::{SlackChannel, SlackMessage};
|
||||
|
||||
/// Walk the Composio response envelope and pull out the channel array
|
||||
/// from a `SLACK_LIST_CONVERSATIONS` call. Composio often wraps the raw
|
||||
/// upstream shape one or two levels deeper, so we try multiple pointers.
|
||||
pub(crate) fn extract_channels(data: &Value) -> Vec<SlackChannel> {
|
||||
let candidates = [
|
||||
data.pointer("/data/channels"),
|
||||
data.pointer("/channels"),
|
||||
data.pointer("/data/data/channels"),
|
||||
data.pointer("/data/conversations"),
|
||||
data.pointer("/conversations"),
|
||||
];
|
||||
let arr = candidates
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.find_map(|v| v.as_array())
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
|
||||
arr.into_iter().filter_map(parse_channel).collect()
|
||||
}
|
||||
|
||||
fn parse_channel(raw: Value) -> Option<SlackChannel> {
|
||||
let id = raw
|
||||
.get("id")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
if id.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let name = raw
|
||||
.get("name")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or(&id)
|
||||
.to_string();
|
||||
let is_private = raw
|
||||
.get("is_private")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
Some(SlackChannel {
|
||||
id,
|
||||
name,
|
||||
is_private,
|
||||
})
|
||||
}
|
||||
|
||||
/// Walk the Composio response envelope and pull out the `messages` array
|
||||
/// from a `SLACK_FETCH_CONVERSATION_HISTORY` call.
|
||||
///
|
||||
/// `users` resolves Slack user ids both as the message author and inline
|
||||
/// `<@…>` mentions in the text. Pass [`SlackUsers::empty`] to skip
|
||||
/// resolution — raw ids will pass through unchanged.
|
||||
pub(crate) fn extract_messages(
|
||||
data: &Value,
|
||||
channel_id: &str,
|
||||
users: &SlackUsers,
|
||||
) -> Vec<SlackMessage> {
|
||||
let candidates = [
|
||||
data.pointer("/data/messages"),
|
||||
data.pointer("/messages"),
|
||||
data.pointer("/data/data/messages"),
|
||||
];
|
||||
let arr = candidates
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.find_map(|v| v.as_array())
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
|
||||
arr.into_iter()
|
||||
.filter_map(|raw| parse_message(channel_id, raw, users))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn parse_message(channel_id: &str, raw: Value, users: &SlackUsers) -> Option<SlackMessage> {
|
||||
let ts_raw = raw.get("ts").and_then(|t| t.as_str())?.to_string();
|
||||
let timestamp = parse_ts(&ts_raw)?;
|
||||
let raw_text = raw
|
||||
.get("text")
|
||||
.and_then(|t| t.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
if raw_text.trim().is_empty() {
|
||||
return None;
|
||||
}
|
||||
// Replace `<@Uxxx>` mentions with `@<resolved>` so the canonical
|
||||
// markdown reads naturally.
|
||||
let text = users.replace_mentions(&raw_text);
|
||||
let author_id = raw
|
||||
.get("user")
|
||||
.or_else(|| raw.get("bot_id"))
|
||||
.and_then(|u| u.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let author = users.resolve(&author_id);
|
||||
let thread_ts = raw
|
||||
.get("thread_ts")
|
||||
.and_then(|t| t.as_str())
|
||||
.map(String::from);
|
||||
Some(SlackMessage {
|
||||
channel_id: channel_id.to_string(),
|
||||
author,
|
||||
text,
|
||||
timestamp,
|
||||
ts_raw,
|
||||
thread_ts,
|
||||
})
|
||||
}
|
||||
|
||||
/// Slack's `ts` is a decimal string `"<unix_seconds>.<micro>"`. The
|
||||
/// integer part is what we care about for bucketing.
|
||||
fn parse_ts(ts_raw: &str) -> Option<DateTime<Utc>> {
|
||||
let seconds_str = ts_raw.split('.').next()?;
|
||||
let secs: i64 = seconds_str.parse().ok()?;
|
||||
Utc.timestamp_opt(secs, 0).single()
|
||||
}
|
||||
|
||||
/// Walk a `SLACK_SEARCH_MESSAGES` response envelope and pull out every
|
||||
/// matching message. Unlike `extract_messages` (which is per-channel),
|
||||
/// search results carry a `channel.id` field on each match — so the
|
||||
/// returned `SlackMessage`s span every channel that matched the query.
|
||||
pub(crate) fn extract_search_messages(data: &Value, users: &SlackUsers) -> Vec<SlackMessage> {
|
||||
let candidates = [
|
||||
data.pointer("/data/messages/matches"),
|
||||
data.pointer("/messages/matches"),
|
||||
data.pointer("/data/data/messages/matches"),
|
||||
];
|
||||
let arr = candidates
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.find_map(|v| v.as_array())
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
|
||||
arr.into_iter()
|
||||
.filter_map(|raw| parse_search_match(raw, users))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Total page count from the search response. Slack's `search.messages`
|
||||
/// uses page-number pagination (1-indexed) under
|
||||
/// `messages.paging.pages`.
|
||||
pub(crate) fn extract_search_total_pages(data: &Value) -> u32 {
|
||||
let candidates = [
|
||||
data.pointer("/data/messages/paging/pages"),
|
||||
data.pointer("/messages/paging/pages"),
|
||||
];
|
||||
candidates
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.find_map(|v| v.as_u64())
|
||||
.unwrap_or(1) as u32
|
||||
}
|
||||
|
||||
fn parse_search_match(raw: Value, users: &SlackUsers) -> Option<SlackMessage> {
|
||||
let ts_raw = raw.get("ts").and_then(|t| t.as_str())?.to_string();
|
||||
let timestamp = parse_ts(&ts_raw)?;
|
||||
let raw_text = raw
|
||||
.get("text")
|
||||
.and_then(|t| t.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
if raw_text.trim().is_empty() {
|
||||
return None;
|
||||
}
|
||||
let text = users.replace_mentions(&raw_text);
|
||||
let author_id = raw
|
||||
.get("user")
|
||||
.or_else(|| raw.get("bot_id"))
|
||||
.and_then(|u| u.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let author = users.resolve(&author_id);
|
||||
let channel_id = raw
|
||||
.pointer("/channel/id")
|
||||
.and_then(|c| c.as_str())?
|
||||
.to_string();
|
||||
let thread_ts = raw
|
||||
.get("thread_ts")
|
||||
.and_then(|t| t.as_str())
|
||||
.map(String::from);
|
||||
Some(SlackMessage {
|
||||
channel_id,
|
||||
author,
|
||||
text,
|
||||
timestamp,
|
||||
ts_raw,
|
||||
thread_ts,
|
||||
})
|
||||
}
|
||||
|
||||
/// Extract a pagination `next_cursor` from a `SLACK_LIST_CONVERSATIONS`
|
||||
/// or `SLACK_FETCH_CONVERSATION_HISTORY` response.
|
||||
pub(crate) fn extract_next_cursor(data: &Value) -> Option<String> {
|
||||
let candidates = [
|
||||
data.pointer("/data/response_metadata/next_cursor"),
|
||||
data.pointer("/response_metadata/next_cursor"),
|
||||
data.pointer("/data/next_cursor"),
|
||||
data.pointer("/next_cursor"),
|
||||
];
|
||||
for cand in candidates.into_iter().flatten() {
|
||||
if let Some(s) = cand.as_str() {
|
||||
let trimmed = s.trim();
|
||||
if !trimmed.is_empty() {
|
||||
return Some(trimmed.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Per-channel cursor map encoded into `SyncState.cursor`. We use
|
||||
/// `BTreeMap` so serialization is deterministic (makes log diffs
|
||||
/// readable and tests stable).
|
||||
///
|
||||
/// Value is unix-seconds of the **end of the latest flushed bucket** for
|
||||
/// that channel. Fetches for that channel use `oldest = value` so we
|
||||
/// skip already-ingested ranges.
|
||||
pub type ChannelCursors = BTreeMap<String, i64>;
|
||||
|
||||
/// Deserialize the per-channel cursor map out of `SyncState.cursor`.
|
||||
/// Returns an empty map on any parse failure — a "broken" cursor should
|
||||
/// degrade to "start from the backfill window" rather than bail out.
|
||||
pub(crate) fn decode_cursors(raw: Option<&str>) -> ChannelCursors {
|
||||
let Some(raw) = raw else {
|
||||
return ChannelCursors::new();
|
||||
};
|
||||
match serde_json::from_str::<ChannelCursors>(raw) {
|
||||
Ok(map) => map,
|
||||
Err(err) => {
|
||||
tracing::warn!(
|
||||
error = %err,
|
||||
"[composio:slack] cursor parse failed, resetting per-channel cursors"
|
||||
);
|
||||
ChannelCursors::new()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn encode_cursors(map: &ChannelCursors) -> String {
|
||||
serde_json::to_string(map).unwrap_or_else(|_| "{}".to_string())
|
||||
}
|
||||
|
||||
pub(crate) fn now_ms() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as u64)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn extract_channels_from_data_channels() {
|
||||
let data = json!({
|
||||
"data": {
|
||||
"channels": [
|
||||
{"id": "C1", "name": "eng", "is_private": false},
|
||||
{"id": "G1", "name": "ops", "is_private": true},
|
||||
{"id": "", "name": "empty-id-drops"},
|
||||
]
|
||||
}
|
||||
});
|
||||
let out = extract_channels(&data);
|
||||
assert_eq!(out.len(), 2);
|
||||
assert_eq!(out[0].id, "C1");
|
||||
assert!(!out[0].is_private);
|
||||
assert_eq!(out[1].id, "G1");
|
||||
assert!(out[1].is_private);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_channels_honors_nested_envelope() {
|
||||
let data = json!({
|
||||
"data": { "data": { "channels": [{"id": "C1", "name": "eng"}] } }
|
||||
});
|
||||
let out = extract_channels(&data);
|
||||
assert_eq!(out.len(), 1);
|
||||
assert_eq!(out[0].id, "C1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_messages_parses_fields() {
|
||||
let data = json!({
|
||||
"data": {
|
||||
"messages": [
|
||||
{"ts": "1714003200.000100", "user": "U1", "text": "hi"},
|
||||
{"ts": "1714003300.000200", "user": "U2", "text": "world"},
|
||||
{"ts": "1714003400.000300", "user": "U3", "text": " "} // dropped (blank)
|
||||
]
|
||||
}
|
||||
});
|
||||
let users = SlackUsers::empty();
|
||||
let out = extract_messages(&data, "C1", &users);
|
||||
assert_eq!(out.len(), 2);
|
||||
assert_eq!(out[0].channel_id, "C1");
|
||||
// Empty user-cache passes raw id through unchanged.
|
||||
assert_eq!(out[0].author, "U1");
|
||||
assert_eq!(out[0].text, "hi");
|
||||
assert_eq!(out[0].timestamp.timestamp(), 1_714_003_200);
|
||||
assert_eq!(out[0].ts_raw, "1714003200.000100");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_messages_resolves_authors_and_mentions() {
|
||||
let data = json!({
|
||||
"data": {
|
||||
"messages": [
|
||||
{"ts": "1714003200.0", "user": "U1", "text": "ping <@U2> about the migration"}
|
||||
]
|
||||
}
|
||||
});
|
||||
let mut m = std::collections::HashMap::new();
|
||||
m.insert("U1".into(), "alice".into());
|
||||
m.insert("U2".into(), "bob".into());
|
||||
let users = SlackUsers::from_map(m);
|
||||
let out = extract_messages(&data, "C1", &users);
|
||||
assert_eq!(out.len(), 1);
|
||||
assert_eq!(out[0].author, "alice");
|
||||
assert_eq!(out[0].text, "ping @bob about the migration");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_messages_skips_missing_ts() {
|
||||
let data = json!({"messages": [{"user": "U1", "text": "orphan"}]});
|
||||
let users = SlackUsers::empty();
|
||||
let out = extract_messages(&data, "C1", &users);
|
||||
assert!(out.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_next_cursor_finds_response_metadata_path() {
|
||||
let data = json!({
|
||||
"data": {
|
||||
"response_metadata": { "next_cursor": "dXNlcjpVMDY..." }
|
||||
}
|
||||
});
|
||||
assert_eq!(
|
||||
extract_next_cursor(&data),
|
||||
Some("dXNlcjpVMDY...".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_next_cursor_none_when_blank() {
|
||||
let data = json!({"data": {"response_metadata": {"next_cursor": " "}}});
|
||||
assert!(extract_next_cursor(&data).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encode_decode_roundtrip() {
|
||||
let mut map = ChannelCursors::new();
|
||||
map.insert("C1".into(), 1_714_003_200);
|
||||
map.insert("C2".into(), 1_714_010_000);
|
||||
let encoded = encode_cursors(&map);
|
||||
let decoded = decode_cursors(Some(&encoded));
|
||||
assert_eq!(decoded, map);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_empty_cursor_returns_empty_map() {
|
||||
assert!(decode_cursors(None).is_empty());
|
||||
assert!(decode_cursors(Some("")).is_empty());
|
||||
assert!(decode_cursors(Some("not json")).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_ts_accepts_slack_decimal_format() {
|
||||
let dt = parse_ts("1714003200.000100").unwrap();
|
||||
assert_eq!(dt.timestamp(), 1_714_003_200);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_ts_rejects_garbage() {
|
||||
assert!(parse_ts("").is_none());
|
||||
assert!(parse_ts("not.a.number").is_none());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,364 @@
|
||||
//! Slack user-id → display-name resolver.
|
||||
//!
|
||||
//! Slack's `conversations.history` payload references users by their
|
||||
//! workspace-stable id (e.g. `U01Q1TBL20P`) in two places:
|
||||
//!
|
||||
//! 1. The `user` field on each message (the author).
|
||||
//! 2. Inline `<@U01Q1TBL20P>` mention syntax inside `text`.
|
||||
//!
|
||||
//! Neither is human-readable. To make canonical chat transcripts useful
|
||||
//! for retrieval (and for humans reading the seal-cascade summaries),
|
||||
//! we fetch the workspace's user directory once per sync run, build an
|
||||
//! id → display-name map, and apply it both to the author field and to
|
||||
//! every `<@…>` mention in message bodies.
|
||||
//!
|
||||
//! ## Cache scope
|
||||
//!
|
||||
//! Per-sync only. Each `SlackProvider::sync()` invocation calls
|
||||
//! [`SlackUsers::fetch`] once before walking channels. The map lives
|
||||
//! in a local variable for the duration of the sync, then drops.
|
||||
//! Slack's user list rarely changes within a 15-minute sync window,
|
||||
//! and re-fetching per sync keeps stale-cache risk near zero without
|
||||
//! adding persistence machinery.
|
||||
//!
|
||||
//! ## Soft-fallback contract
|
||||
//!
|
||||
//! Following the pattern of [`crate::openhuman::composio::providers::slack::sync::extract_messages`]
|
||||
//! and the [`super::provider::SlackProvider::sync`] error handling, a
|
||||
//! failure to fetch users is **not fatal**. The returned [`SlackUsers`]
|
||||
//! is empty, and `resolve()` / `replace_mentions()` pass through raw
|
||||
//! ids unchanged — same behaviour as before this module existed.
|
||||
|
||||
use regex::Regex;
|
||||
use serde_json::{json, Value};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
use crate::openhuman::composio::client::ComposioClient;
|
||||
|
||||
/// Composio action slug for the bulk user listing.
|
||||
const ACTION_LIST_USERS: &str = "SLACK_LIST_ALL_USERS";
|
||||
|
||||
/// Page size — Slack caps at 1000; 200 keeps each page small.
|
||||
const PAGE_SIZE: u32 = 200;
|
||||
|
||||
/// Maximum pages to walk per sync. With `PAGE_SIZE = 200` this covers
|
||||
/// workspaces up to 4000 users without complaint. Beyond that the tail
|
||||
/// is truncated and unresolved ids will pass through verbatim.
|
||||
const MAX_PAGES: u32 = 20;
|
||||
|
||||
/// Slack mention syntax: `<@U01Q1TBL20P>`. Captures the bare id so we
|
||||
/// can drop the wrapper when substituting in a resolved name.
|
||||
fn mention_re() -> &'static Regex {
|
||||
static RE: OnceLock<Regex> = OnceLock::new();
|
||||
RE.get_or_init(|| Regex::new(r"<@(U[A-Z0-9]+)>").expect("static mention regex compiles"))
|
||||
}
|
||||
|
||||
/// Map of Slack user id → human-readable display name.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct SlackUsers {
|
||||
map: HashMap<String, String>,
|
||||
}
|
||||
|
||||
impl SlackUsers {
|
||||
/// Empty map — `resolve()` passes through raw ids verbatim.
|
||||
pub fn empty() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Number of users in the cache.
|
||||
pub fn len(&self) -> usize {
|
||||
self.map.len()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.map.is_empty()
|
||||
}
|
||||
|
||||
/// Resolve a Slack user id to a display name. Returns the input id
|
||||
/// unchanged when no mapping exists — matches the
|
||||
/// resolve-or-passthrough contract of the parent provider.
|
||||
pub fn resolve(&self, user_id: &str) -> String {
|
||||
self.map
|
||||
.get(user_id)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| user_id.to_string())
|
||||
}
|
||||
|
||||
/// Replace every `<@Uxxx>` mention in `text` with `@<display name>`.
|
||||
/// Unknown ids stay as `@Uxxx` (the wrapper is removed but the id
|
||||
/// is preserved so retrieval can still surface them).
|
||||
pub fn replace_mentions(&self, text: &str) -> String {
|
||||
mention_re()
|
||||
.replace_all(text, |caps: ®ex::Captures| {
|
||||
let id = &caps[1];
|
||||
let resolved = self.map.get(id).map(String::as_str).unwrap_or(id);
|
||||
format!("@{resolved}")
|
||||
})
|
||||
.into_owned()
|
||||
}
|
||||
|
||||
/// Pull the workspace user directory via Composio. Soft-fails to
|
||||
/// [`SlackUsers::empty`] on transport, HTTP, JSON, or
|
||||
/// provider-failure errors so the sync can continue with raw ids.
|
||||
///
|
||||
/// Returns `(users, total_attempts)` where `total_attempts` sums every
|
||||
/// real Composio call this fetch made across pages and rate-limit
|
||||
/// retries, so the caller can charge the daily quota meter
|
||||
/// accurately. Pages walked silently are tracked too — without this,
|
||||
/// large workspaces under-report their request usage.
|
||||
pub async fn fetch(client: &ComposioClient) -> (Self, u32) {
|
||||
let mut map: HashMap<String, String> = HashMap::new();
|
||||
let mut cursor: Option<String> = None;
|
||||
let mut total_attempts: u32 = 0;
|
||||
|
||||
for page_num in 0..MAX_PAGES {
|
||||
let mut args = json!({ "limit": PAGE_SIZE });
|
||||
if let Some(ref c) = cursor {
|
||||
args["cursor"] = json!(c);
|
||||
}
|
||||
|
||||
// Going through `execute_with_retry` so a transient
|
||||
// `ratelimited` page doesn't drop us into a half-built
|
||||
// directory while the rest of the provider uses backoff.
|
||||
// Soft-fall to whatever was collected so far on any failure.
|
||||
let (resp, attempts) = match super::provider::execute_with_retry(
|
||||
client,
|
||||
ACTION_LIST_USERS,
|
||||
args,
|
||||
&format!("{ACTION_LIST_USERS} page {page_num}"),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(t) => t,
|
||||
Err(err) => {
|
||||
// We don't know exactly how many attempts the helper
|
||||
// burned before bailing, but at least one ran — count
|
||||
// it so the budget meter doesn't silently undercount.
|
||||
total_attempts = total_attempts.saturating_add(1);
|
||||
log::warn!(
|
||||
"[composio:slack:users] {ACTION_LIST_USERS} page {page_num} failed: {err} — \
|
||||
degrading to raw ids for the rest of this sync"
|
||||
);
|
||||
return (Self { map }, total_attempts);
|
||||
}
|
||||
};
|
||||
total_attempts = total_attempts.saturating_add(attempts);
|
||||
|
||||
super::provider::dump_response("_meta", "users", page_num, &resp.data);
|
||||
absorb_page(&resp.data, &mut map);
|
||||
|
||||
cursor = extract_next_cursor(&resp.data);
|
||||
if cursor.is_none() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
log::info!(
|
||||
"[composio:slack:users] resolved {} workspace users in {total_attempts} call(s)",
|
||||
map.len()
|
||||
);
|
||||
(Self { map }, total_attempts)
|
||||
}
|
||||
|
||||
/// Construct from a pre-built map. Test-only — production callers
|
||||
/// should use [`Self::fetch`] or [`Self::empty`].
|
||||
#[cfg(test)]
|
||||
pub fn from_map(map: HashMap<String, String>) -> Self {
|
||||
Self { map }
|
||||
}
|
||||
}
|
||||
|
||||
/// Walk a Composio response envelope and absorb every user object's
|
||||
/// `id` + best-available display name into `map`.
|
||||
fn absorb_page(data: &Value, map: &mut HashMap<String, String>) {
|
||||
let candidates = [
|
||||
data.pointer("/data/members"),
|
||||
data.pointer("/members"),
|
||||
data.pointer("/data/users"),
|
||||
data.pointer("/users"),
|
||||
data.pointer("/data/data/members"),
|
||||
];
|
||||
let arr = candidates
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.find_map(|v| v.as_array())
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
|
||||
for raw in arr {
|
||||
let id = raw
|
||||
.get("id")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.trim()
|
||||
.to_string();
|
||||
if id.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if let Some(name) = pick_display_name(&raw) {
|
||||
map.insert(id, name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Slack returns several name fields per user. Prefer the most
|
||||
/// human-readable, fall back through real_name → name → display_name.
|
||||
fn pick_display_name(raw: &Value) -> Option<String> {
|
||||
let candidates = [
|
||||
raw.pointer("/profile/display_name"),
|
||||
raw.pointer("/profile/real_name"),
|
||||
raw.get("real_name"),
|
||||
raw.get("name"),
|
||||
raw.pointer("/profile/display_name_normalized"),
|
||||
raw.pointer("/profile/real_name_normalized"),
|
||||
];
|
||||
for cand in candidates.into_iter().flatten() {
|
||||
if let Some(s) = cand.as_str() {
|
||||
let trimmed = s.trim();
|
||||
if !trimmed.is_empty() {
|
||||
return Some(trimmed.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn extract_next_cursor(data: &Value) -> Option<String> {
|
||||
let candidates = [
|
||||
data.pointer("/data/response_metadata/next_cursor"),
|
||||
data.pointer("/response_metadata/next_cursor"),
|
||||
data.pointer("/data/next_cursor"),
|
||||
data.pointer("/next_cursor"),
|
||||
];
|
||||
for cand in candidates.into_iter().flatten() {
|
||||
if let Some(s) = cand.as_str() {
|
||||
let trimmed = s.trim();
|
||||
if !trimmed.is_empty() {
|
||||
return Some(trimmed.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn sample_users() -> SlackUsers {
|
||||
let mut m = HashMap::new();
|
||||
m.insert("U001".to_string(), "alice".to_string());
|
||||
m.insert("U002".to_string(), "bob".to_string());
|
||||
SlackUsers::from_map(m)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_known_id_returns_name() {
|
||||
let u = sample_users();
|
||||
assert_eq!(u.resolve("U001"), "alice");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_unknown_id_passes_through() {
|
||||
let u = sample_users();
|
||||
assert_eq!(u.resolve("U999"), "U999");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_passes_through_every_id() {
|
||||
let u = SlackUsers::empty();
|
||||
assert_eq!(u.resolve("U001"), "U001");
|
||||
assert_eq!(u.replace_mentions("hi <@U001>"), "hi @U001");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replace_mentions_substitutes_known_ids() {
|
||||
let u = sample_users();
|
||||
let out = u.replace_mentions("Hi <@U001>, please ping <@U002>.");
|
||||
assert_eq!(out, "Hi @alice, please ping @bob.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replace_mentions_strips_wrapper_for_unknown_id() {
|
||||
let u = sample_users();
|
||||
// Unknown id keeps the raw id but loses the `<@...>` wrapper.
|
||||
let out = u.replace_mentions("ping <@U999>");
|
||||
assert_eq!(out, "ping @U999");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replace_mentions_leaves_non_mention_text_alone() {
|
||||
let u = sample_users();
|
||||
let out = u.replace_mentions("no mentions here, just <text>");
|
||||
assert_eq!(out, "no mentions here, just <text>");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replace_mentions_handles_multiple_in_one_line() {
|
||||
let u = sample_users();
|
||||
let out = u.replace_mentions("<@U001> said hi to <@U001> and <@U002>");
|
||||
assert_eq!(out, "@alice said hi to @alice and @bob");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absorb_page_reads_data_members_path() {
|
||||
let data = json!({
|
||||
"data": {
|
||||
"members": [
|
||||
{
|
||||
"id": "U001",
|
||||
"profile": { "display_name": "alice", "real_name": "Alice Smith" }
|
||||
},
|
||||
{
|
||||
"id": "U002",
|
||||
"profile": { "display_name": "" , "real_name": "Bob Jones" }
|
||||
},
|
||||
{
|
||||
"id": "",
|
||||
"profile": { "display_name": "skipped" }
|
||||
}
|
||||
]
|
||||
}
|
||||
});
|
||||
let mut m = HashMap::new();
|
||||
absorb_page(&data, &mut m);
|
||||
assert_eq!(m.get("U001").unwrap(), "alice");
|
||||
// Falls back to real_name when display_name is blank.
|
||||
assert_eq!(m.get("U002").unwrap(), "Bob Jones");
|
||||
// Empty id row is dropped.
|
||||
assert!(!m.contains_key(""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pick_display_name_prefers_display_name_over_real_name() {
|
||||
let raw = json!({
|
||||
"profile": { "display_name": "alice", "real_name": "Alice Smith" }
|
||||
});
|
||||
assert_eq!(pick_display_name(&raw).as_deref(), Some("alice"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pick_display_name_falls_back_to_name() {
|
||||
let raw = json!({ "name": "alice", "profile": {} });
|
||||
assert_eq!(pick_display_name(&raw).as_deref(), Some("alice"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pick_display_name_returns_none_when_all_blank() {
|
||||
let raw = json!({ "profile": { "display_name": " " }, "name": "" });
|
||||
assert!(pick_display_name(&raw).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_next_cursor_finds_response_metadata() {
|
||||
let data = json!({"data": {"response_metadata": {"next_cursor": "abc123"}}});
|
||||
assert_eq!(extract_next_cursor(&data).as_deref(), Some("abc123"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_next_cursor_none_when_blank() {
|
||||
let data = json!({"response_metadata": {"next_cursor": " "}});
|
||||
assert!(extract_next_cursor(&data).is_none());
|
||||
}
|
||||
}
|
||||
@@ -926,6 +926,60 @@ impl Config {
|
||||
}
|
||||
}
|
||||
|
||||
// LLM entity extractor overrides — set endpoint + model to route
|
||||
// ingest scoring through Ollama NER (Phase 2 follow-up). Empty
|
||||
// string explicitly clears (opts out).
|
||||
if let Ok(endpoint) = std::env::var("OPENHUMAN_MEMORY_EXTRACT_ENDPOINT") {
|
||||
let trimmed = endpoint.trim();
|
||||
self.memory_tree.llm_extractor_endpoint = if trimmed.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(trimmed.to_string())
|
||||
};
|
||||
}
|
||||
if let Ok(model) = std::env::var("OPENHUMAN_MEMORY_EXTRACT_MODEL") {
|
||||
let trimmed = model.trim();
|
||||
self.memory_tree.llm_extractor_model = if trimmed.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(trimmed.to_string())
|
||||
};
|
||||
}
|
||||
if let Ok(val) = std::env::var("OPENHUMAN_MEMORY_EXTRACT_TIMEOUT_MS") {
|
||||
if let Ok(ms) = val.trim().parse::<u64>() {
|
||||
if ms > 0 {
|
||||
self.memory_tree.llm_extractor_timeout_ms = Some(ms);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// LLM summariser overrides — set endpoint + model to route
|
||||
// bucket-seal summaries through Ollama instead of InertSummariser
|
||||
// (Phase 3a real-summariser hook).
|
||||
if let Ok(endpoint) = std::env::var("OPENHUMAN_MEMORY_SUMMARISE_ENDPOINT") {
|
||||
let trimmed = endpoint.trim();
|
||||
self.memory_tree.llm_summariser_endpoint = if trimmed.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(trimmed.to_string())
|
||||
};
|
||||
}
|
||||
if let Ok(model) = std::env::var("OPENHUMAN_MEMORY_SUMMARISE_MODEL") {
|
||||
let trimmed = model.trim();
|
||||
self.memory_tree.llm_summariser_model = if trimmed.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(trimmed.to_string())
|
||||
};
|
||||
}
|
||||
if let Ok(val) = std::env::var("OPENHUMAN_MEMORY_SUMMARISE_TIMEOUT_MS") {
|
||||
if let Ok(ms) = val.trim().parse::<u64>() {
|
||||
if ms > 0 {
|
||||
self.memory_tree.llm_summariser_timeout_ms = Some(ms);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-update overrides
|
||||
if let Some(flag) = env.get("OPENHUMAN_AUTO_UPDATE_ENABLED") {
|
||||
let normalized = flag.trim().to_ascii_lowercase();
|
||||
|
||||
@@ -90,6 +90,12 @@ impl Default for MemoryConfig {
|
||||
/// - `OPENHUMAN_MEMORY_EMBED_ENDPOINT`
|
||||
/// - `OPENHUMAN_MEMORY_EMBED_MODEL`
|
||||
/// - `OPENHUMAN_MEMORY_EMBED_TIMEOUT_MS`
|
||||
/// - `OPENHUMAN_MEMORY_EXTRACT_ENDPOINT`
|
||||
/// - `OPENHUMAN_MEMORY_EXTRACT_MODEL`
|
||||
/// - `OPENHUMAN_MEMORY_EXTRACT_TIMEOUT_MS`
|
||||
/// - `OPENHUMAN_MEMORY_SUMMARISE_ENDPOINT`
|
||||
/// - `OPENHUMAN_MEMORY_SUMMARISE_MODEL`
|
||||
/// - `OPENHUMAN_MEMORY_SUMMARISE_TIMEOUT_MS`
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct MemoryTreeConfig {
|
||||
/// Ollama endpoint for the embedder (e.g. `http://localhost:11434`).
|
||||
@@ -113,6 +119,41 @@ pub struct MemoryTreeConfig {
|
||||
/// rerank falls back to scope + recency ordering only.
|
||||
#[serde(default = "default_memory_tree_embedding_strict")]
|
||||
pub embedding_strict: bool,
|
||||
|
||||
/// Ollama endpoint for the LLM entity extractor
|
||||
/// (`memory::tree::score::extract::llm::LlmEntityExtractor`). When
|
||||
/// unset, ingest uses the regex-only extractor — no LLM call. Soft
|
||||
/// failures in the LLM path fall back to regex-only for that chunk.
|
||||
#[serde(default = "default_memory_tree_llm_endpoint")]
|
||||
pub llm_extractor_endpoint: Option<String>,
|
||||
|
||||
/// Model name for the entity extractor (e.g. `qwen2.5:0.5b`).
|
||||
#[serde(default = "default_memory_tree_llm_endpoint")]
|
||||
pub llm_extractor_model: Option<String>,
|
||||
|
||||
/// Per-request timeout for the LLM extractor, in milliseconds.
|
||||
#[serde(default = "default_memory_tree_llm_extractor_timeout_ms")]
|
||||
pub llm_extractor_timeout_ms: Option<u64>,
|
||||
|
||||
/// Ollama endpoint for the summariser
|
||||
/// (`memory::tree::source_tree::summariser::llm::LlmSummariser`).
|
||||
/// When unset, bucket-seal cascades use `InertSummariser` — sealed
|
||||
/// nodes contain concatenated+truncated child text instead of a
|
||||
/// real LLM summary. Soft failures fall back to inert per seal.
|
||||
#[serde(default = "default_memory_tree_llm_endpoint")]
|
||||
pub llm_summariser_endpoint: Option<String>,
|
||||
|
||||
/// Model name for the summariser. Larger models produce better
|
||||
/// summaries at higher latency; `llama3.1:8b` is a reasonable
|
||||
/// default for production.
|
||||
#[serde(default = "default_memory_tree_llm_endpoint")]
|
||||
pub llm_summariser_model: Option<String>,
|
||||
|
||||
/// Per-request timeout for the summariser, in milliseconds. Default
|
||||
/// is higher than the extractor because summarisation uses more
|
||||
/// tokens and therefore takes longer to generate.
|
||||
#[serde(default = "default_memory_tree_llm_summariser_timeout_ms")]
|
||||
pub llm_summariser_timeout_ms: Option<u64>,
|
||||
}
|
||||
|
||||
/// Returns `None` so that existing installs that never opted into Phase 4
|
||||
@@ -139,6 +180,25 @@ fn default_memory_tree_embedding_strict() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Shared `None` default for the LLM-path fields (extractor + summariser
|
||||
/// endpoints + models). Keeping the same function for all of them makes
|
||||
/// the intent explicit: nothing here auto-enables Ollama.
|
||||
fn default_memory_tree_llm_endpoint() -> Option<String> {
|
||||
None
|
||||
}
|
||||
|
||||
fn default_memory_tree_llm_extractor_timeout_ms() -> Option<u64> {
|
||||
Some(15_000)
|
||||
}
|
||||
|
||||
fn default_memory_tree_llm_summariser_timeout_ms() -> Option<u64> {
|
||||
// 120s — large enough for small/medium local models to finish a
|
||||
// seal-budget summary on a cold-loaded weight cache. Tighter
|
||||
// values cause the LlmSummariser to time out and silently fall
|
||||
// back to InertSummariser (no LLM signal in the resulting node).
|
||||
Some(120_000)
|
||||
}
|
||||
|
||||
impl Default for MemoryTreeConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
@@ -146,6 +206,12 @@ impl Default for MemoryTreeConfig {
|
||||
embedding_model: default_memory_tree_embedding_model(),
|
||||
embedding_timeout_ms: default_memory_tree_embedding_timeout_ms(),
|
||||
embedding_strict: default_memory_tree_embedding_strict(),
|
||||
llm_extractor_endpoint: default_memory_tree_llm_endpoint(),
|
||||
llm_extractor_model: default_memory_tree_llm_endpoint(),
|
||||
llm_extractor_timeout_ms: default_memory_tree_llm_extractor_timeout_ms(),
|
||||
llm_summariser_endpoint: default_memory_tree_llm_endpoint(),
|
||||
llm_summariser_model: default_memory_tree_llm_endpoint(),
|
||||
llm_summariser_timeout_ms: default_memory_tree_llm_summariser_timeout_ms(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ pub mod ingestion_queue;
|
||||
pub mod ops;
|
||||
pub mod rpc_models;
|
||||
pub mod schemas;
|
||||
pub mod slack_ingestion;
|
||||
pub mod store;
|
||||
pub mod traits;
|
||||
pub mod tree;
|
||||
@@ -30,6 +31,9 @@ pub use schemas::{
|
||||
all_controller_schemas as all_memory_controller_schemas,
|
||||
all_registered_controllers as all_memory_registered_controllers,
|
||||
};
|
||||
pub use slack_ingestion::{
|
||||
all_slack_ingestion_controller_schemas, all_slack_ingestion_registered_controllers,
|
||||
};
|
||||
pub use store::{
|
||||
create_memory, create_memory_for_migration, create_memory_with_storage,
|
||||
create_memory_with_storage_and_routes, effective_memory_backend_name, MemoryClient,
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
//! 6-hour UTC-aligned bucketing for Slack messages.
|
||||
//!
|
||||
//! Buckets have fixed wall-clock starts at `00:00`, `06:00`, `12:00`, and
|
||||
//! `18:00` UTC each day. A message's bucket is determined solely by its
|
||||
//! timestamp — the engine never decides bucketing based on order of
|
||||
//! arrival or polling cadence, so the same message always lands in the
|
||||
//! same bucket regardless of when it's seen.
|
||||
//!
|
||||
//! ## Why closed-bucket-only?
|
||||
//!
|
||||
//! Each ingested chunk becomes a *leaf* in the source tree via
|
||||
//! `append_leaf` (see `memory::tree::source_tree::bucket_seal`). Leaves
|
||||
//! participate in a token-budget seal cascade — once appended, they
|
||||
//! cannot be cleanly retracted from already-sealed summary nodes
|
||||
//! upstream. So we only ingest a bucket once, *after* it has closed
|
||||
//! (window end + grace period is past).
|
||||
//!
|
||||
//! The `source_id` convention (`"slack:<channel>:<bucket_start_epoch>"`)
|
||||
//! plus deterministic chunk IDs give us idempotent re-ingest for late
|
||||
//! edits in a future iteration without breaking that invariant.
|
||||
|
||||
use chrono::{DateTime, Datelike, Duration, TimeZone, Timelike, Utc};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use super::types::{Bucket, SlackMessage};
|
||||
|
||||
/// Width of each bucket. Changing this changes the ingest granularity
|
||||
/// *and* invalidates existing `source_id`s, so treat it as a schema
|
||||
/// constant — not a runtime tunable.
|
||||
pub const BUCKET_HOURS: u32 = 6;
|
||||
|
||||
/// Safety window after a bucket's nominal end before we consider it
|
||||
/// closed. Protects against clock skew, slow API polls, or messages
|
||||
/// that arrive via `conversations.history` a few minutes after they
|
||||
/// were sent.
|
||||
pub const GRACE_PERIOD: Duration = Duration::minutes(15);
|
||||
|
||||
/// Round a timestamp down to its 6-hour UTC bucket start.
|
||||
///
|
||||
/// For `2026-04-25T13:42:07Z` this returns `2026-04-25T12:00:00Z`.
|
||||
pub fn bucket_start_for(ts: DateTime<Utc>) -> DateTime<Utc> {
|
||||
let hour = ts.hour();
|
||||
let bucket_hour = (hour / BUCKET_HOURS) * BUCKET_HOURS;
|
||||
Utc.with_ymd_and_hms(ts.year(), ts.month(), ts.day(), bucket_hour, 0, 0)
|
||||
.single()
|
||||
.expect("00/06/12/18 are always valid wall-clock hours in UTC")
|
||||
}
|
||||
|
||||
/// Exclusive bucket end — `start + BUCKET_HOURS`.
|
||||
pub fn bucket_end_for(start: DateTime<Utc>) -> DateTime<Utc> {
|
||||
start + Duration::hours(BUCKET_HOURS as i64)
|
||||
}
|
||||
|
||||
/// Stable identifier for a channel's bucket, used as the memory-tree
|
||||
/// `source_id`. Uses Unix seconds so the id is compact, comparable, and
|
||||
/// independent of the caller's locale.
|
||||
pub fn source_id_for(channel_id: &str, bucket_start: DateTime<Utc>) -> String {
|
||||
format!("slack:{channel_id}:{}", bucket_start.timestamp())
|
||||
}
|
||||
|
||||
/// Partition an in-memory buffer into closed buckets (ready to ingest)
|
||||
/// and the messages that remain in still-open buckets.
|
||||
///
|
||||
/// A bucket is *closed* when `bucket_end + GRACE_PERIOD <= now`. Messages
|
||||
/// in closed buckets are grouped into [`Bucket`] values ordered by start.
|
||||
/// Messages in open buckets flow back into `remaining` and will be
|
||||
/// re-evaluated on a later tick when their window closes.
|
||||
///
|
||||
/// Empty buckets are never produced — if no messages fall into a range
|
||||
/// there is no [`Bucket`] for it.
|
||||
pub fn split_closed(
|
||||
buffer: Vec<SlackMessage>,
|
||||
now: DateTime<Utc>,
|
||||
grace: Duration,
|
||||
) -> (Vec<Bucket>, Vec<SlackMessage>) {
|
||||
let mut by_bucket: BTreeMap<DateTime<Utc>, Vec<SlackMessage>> = BTreeMap::new();
|
||||
for msg in buffer {
|
||||
let start = bucket_start_for(msg.timestamp);
|
||||
by_bucket.entry(start).or_default().push(msg);
|
||||
}
|
||||
|
||||
let mut closed = Vec::new();
|
||||
let mut remaining = Vec::new();
|
||||
for (start, mut messages) in by_bucket {
|
||||
let end = bucket_end_for(start);
|
||||
if end + grace <= now {
|
||||
// Preserve chronological order within the bucket.
|
||||
messages.sort_by_key(|m| m.timestamp);
|
||||
closed.push(Bucket {
|
||||
start,
|
||||
end,
|
||||
messages,
|
||||
});
|
||||
} else {
|
||||
remaining.extend(messages);
|
||||
}
|
||||
}
|
||||
(closed, remaining)
|
||||
}
|
||||
|
||||
/// Enumerate every 6-hour bucket start in `[from, until)` — used by the
|
||||
/// backfill path to walk the last N days in order.
|
||||
pub fn buckets_between(from: DateTime<Utc>, until: DateTime<Utc>) -> Vec<DateTime<Utc>> {
|
||||
if until <= from {
|
||||
return Vec::new();
|
||||
}
|
||||
let mut starts = Vec::new();
|
||||
let mut cursor = bucket_start_for(from);
|
||||
while cursor < until {
|
||||
starts.push(cursor);
|
||||
cursor = bucket_end_for(cursor);
|
||||
}
|
||||
starts
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn ts(year: i32, month: u32, day: u32, hour: u32, min: u32) -> DateTime<Utc> {
|
||||
Utc.with_ymd_and_hms(year, month, day, hour, min, 0)
|
||||
.single()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn msg(channel: &str, timestamp: DateTime<Utc>, body: &str) -> SlackMessage {
|
||||
SlackMessage {
|
||||
channel_id: channel.into(),
|
||||
author: "U1".into(),
|
||||
text: body.into(),
|
||||
timestamp,
|
||||
ts_raw: format!("{}.000000", timestamp.timestamp()),
|
||||
thread_ts: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_start_rounds_down_to_six_hour_boundary() {
|
||||
assert_eq!(
|
||||
bucket_start_for(ts(2026, 4, 25, 0, 0)),
|
||||
ts(2026, 4, 25, 0, 0)
|
||||
);
|
||||
assert_eq!(
|
||||
bucket_start_for(ts(2026, 4, 25, 5, 59)),
|
||||
ts(2026, 4, 25, 0, 0)
|
||||
);
|
||||
assert_eq!(
|
||||
bucket_start_for(ts(2026, 4, 25, 6, 0)),
|
||||
ts(2026, 4, 25, 6, 0)
|
||||
);
|
||||
assert_eq!(
|
||||
bucket_start_for(ts(2026, 4, 25, 13, 42)),
|
||||
ts(2026, 4, 25, 12, 0)
|
||||
);
|
||||
assert_eq!(
|
||||
bucket_start_for(ts(2026, 4, 25, 23, 59)),
|
||||
ts(2026, 4, 25, 18, 0)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_end_is_start_plus_six_hours() {
|
||||
let start = ts(2026, 4, 25, 6, 0);
|
||||
assert_eq!(bucket_end_for(start), ts(2026, 4, 25, 12, 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_id_is_deterministic_and_channel_scoped() {
|
||||
let start = ts(2026, 4, 25, 12, 0);
|
||||
assert_eq!(
|
||||
source_id_for("C0123456", start),
|
||||
format!("slack:C0123456:{}", start.timestamp())
|
||||
);
|
||||
assert_ne!(
|
||||
source_id_for("C0123456", start),
|
||||
source_id_for("C9999999", start),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_closed_keeps_open_bucket_in_remaining() {
|
||||
let now = ts(2026, 4, 25, 7, 0); // bucket [06:00, 12:00) still open
|
||||
let buf = vec![msg("C1", ts(2026, 4, 25, 6, 30), "hi")];
|
||||
let (closed, remaining) = split_closed(buf, now, GRACE_PERIOD);
|
||||
assert!(closed.is_empty());
|
||||
assert_eq!(remaining.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_closed_emits_fully_closed_bucket() {
|
||||
// Bucket [00:00, 06:00) closes at 06:15 with grace; now is 07:00.
|
||||
let now = ts(2026, 4, 25, 7, 0);
|
||||
let buf = vec![
|
||||
msg("C1", ts(2026, 4, 25, 1, 0), "a"),
|
||||
msg("C1", ts(2026, 4, 25, 3, 30), "b"),
|
||||
];
|
||||
let (closed, remaining) = split_closed(buf, now, GRACE_PERIOD);
|
||||
assert_eq!(closed.len(), 1);
|
||||
assert_eq!(remaining.len(), 0);
|
||||
assert_eq!(closed[0].start, ts(2026, 4, 25, 0, 0));
|
||||
assert_eq!(closed[0].end, ts(2026, 4, 25, 6, 0));
|
||||
assert_eq!(closed[0].messages.len(), 2);
|
||||
// Sorted chronologically inside the bucket.
|
||||
assert!(closed[0].messages[0].timestamp < closed[0].messages[1].timestamp);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_closed_respects_grace_boundary() {
|
||||
// Bucket ends at 06:00. With 15-min grace, it closes at 06:15.
|
||||
let just_before = ts(2026, 4, 25, 6, 14);
|
||||
let just_after = ts(2026, 4, 25, 6, 15);
|
||||
let buf = || vec![msg("C1", ts(2026, 4, 25, 3, 0), "x")];
|
||||
|
||||
let (closed_before, _) = split_closed(buf(), just_before, GRACE_PERIOD);
|
||||
assert!(closed_before.is_empty(), "still within grace window");
|
||||
|
||||
let (closed_after, remaining) = split_closed(buf(), just_after, GRACE_PERIOD);
|
||||
assert_eq!(closed_after.len(), 1, "grace expired exactly at 06:15");
|
||||
assert!(remaining.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_closed_handles_mixed_buckets() {
|
||||
let now = ts(2026, 4, 25, 13, 0);
|
||||
let buf = vec![
|
||||
msg("C1", ts(2026, 4, 25, 1, 0), "closed bucket 1"),
|
||||
msg("C1", ts(2026, 4, 25, 7, 0), "closed bucket 2"),
|
||||
msg("C1", ts(2026, 4, 25, 12, 5), "open bucket"),
|
||||
];
|
||||
let (closed, remaining) = split_closed(buf, now, GRACE_PERIOD);
|
||||
assert_eq!(closed.len(), 2);
|
||||
assert_eq!(remaining.len(), 1);
|
||||
assert_eq!(closed[0].start, ts(2026, 4, 25, 0, 0));
|
||||
assert_eq!(closed[1].start, ts(2026, 4, 25, 6, 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_closed_empty_buffer_is_noop() {
|
||||
let (closed, remaining) = split_closed(vec![], ts(2026, 4, 25, 12, 0), GRACE_PERIOD);
|
||||
assert!(closed.is_empty());
|
||||
assert!(remaining.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn buckets_between_walks_six_hour_starts() {
|
||||
let from = ts(2026, 4, 25, 2, 0); // snaps to 00:00
|
||||
let until = ts(2026, 4, 26, 0, 0);
|
||||
let starts = buckets_between(from, until);
|
||||
assert_eq!(starts.len(), 4);
|
||||
assert_eq!(starts[0], ts(2026, 4, 25, 0, 0));
|
||||
assert_eq!(starts[1], ts(2026, 4, 25, 6, 0));
|
||||
assert_eq!(starts[2], ts(2026, 4, 25, 12, 0));
|
||||
assert_eq!(starts[3], ts(2026, 4, 25, 18, 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn buckets_between_empty_when_range_reversed() {
|
||||
let starts = buckets_between(ts(2026, 4, 26, 0, 0), ts(2026, 4, 25, 0, 0));
|
||||
assert!(starts.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
//! Slack → memory-tree ingestion support.
|
||||
//!
|
||||
//! This module is the "Slack-specific plumbing the memory tree needs":
|
||||
//!
|
||||
//! - [`bucketer`] — 6-hour UTC-aligned window grouping + grace-period
|
||||
//! closed-bucket extraction (`split_closed`) + stable `source_id`
|
||||
//! generation.
|
||||
//! - [`ops`] — bucket → [`memory::tree::canonicalize::chat::ChatBatch`]
|
||||
//! conversion + wrapper around
|
||||
//! [`memory::tree::ingest::ingest_chat`] so the caller just says
|
||||
//! "ingest this bucket for this channel".
|
||||
//! - [`types`] — `SlackMessage`, `SlackChannel`, `Bucket`. Used by
|
||||
//! both the Composio-backed [`crate::openhuman::composio::providers::slack`]
|
||||
//! provider and the RPC/observability surfaces below.
|
||||
//! - [`rpc`] / [`schemas`] — JSON-RPC surface for manually triggering a
|
||||
//! Slack sync (`openhuman.slack_memory_sync_trigger`) + inspecting
|
||||
//! per-connection state (`openhuman.slack_memory_sync_status`).
|
||||
//!
|
||||
//! Auth + scheduling live elsewhere:
|
||||
//!
|
||||
//! - OAuth is delegated to Composio (the user's Slack connection in
|
||||
//! Composio's hosted flow is what authorises the API calls).
|
||||
//! - Periodic scheduling comes from `composio::periodic` — every 15
|
||||
//! minutes the scheduler fires `SlackProvider::sync()` for every
|
||||
//! active Slack connection.
|
||||
//!
|
||||
//! What this module does NOT contain (removed when we pivoted from
|
||||
//! direct-bot-token to Composio):
|
||||
//!
|
||||
//! - A Slack Web API HTTP client. Calls go through `ctx.client.execute_tool()`.
|
||||
//! - A custom per-channel cursor SQLite table. State is persisted via
|
||||
//! `composio::providers::sync_state::SyncState` in the memory KV store,
|
||||
//! keyed by `(toolkit="slack", connection_id)`.
|
||||
//! - A long-running engine task. `composio::periodic::run_one_tick` is
|
||||
//! the scheduler.
|
||||
|
||||
pub mod bucketer;
|
||||
pub mod ops;
|
||||
pub mod rpc;
|
||||
pub mod schemas;
|
||||
pub mod types;
|
||||
|
||||
pub use schemas::{
|
||||
all_controller_schemas as all_slack_ingestion_controller_schemas,
|
||||
all_registered_controllers as all_slack_ingestion_registered_controllers,
|
||||
};
|
||||
@@ -0,0 +1,234 @@
|
||||
//! Bucket → memory-tree ingest plumbing.
|
||||
//!
|
||||
//! This module owns the one-way conversion from our internal
|
||||
//! [`Bucket`] type to [`ChatBatch`] + the call into
|
||||
//! [`memory::tree::ingest::ingest_chat`]. It is intentionally free of
|
||||
//! HTTP, timers, or state — those belong to [`super::engine`] — so the
|
||||
//! canonicalisation path is easy to unit test.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
use super::types::{Bucket, SlackChannel};
|
||||
use crate::openhuman::config::Config;
|
||||
use crate::openhuman::memory::tree::canonicalize::chat::{ChatBatch, ChatMessage};
|
||||
use crate::openhuman::memory::tree::ingest::{ingest_chat, IngestResult};
|
||||
|
||||
/// Platform identifier embedded in the canonical chat transcript header.
|
||||
/// Matches the entry in `memory::tree::retrieval::source::PLATFORM_KINDS`.
|
||||
pub const SLACK_PLATFORM: &str = "slack";
|
||||
|
||||
/// Tags attached to every Slack-ingested chunk. Keep this list stable —
|
||||
/// retrieval callers filter on them.
|
||||
pub const DEFAULT_TAGS: &[&str] = &["slack", "ingested"];
|
||||
|
||||
/// Convert a closed [`Bucket`] into a [`ChatBatch`] ready for
|
||||
/// `memory::tree::ingest::ingest_chat`.
|
||||
///
|
||||
/// Empty buckets return `None` — upstream code should skip them rather
|
||||
/// than ingest an empty payload. Bucket authorship is preserved by
|
||||
/// using the Slack user ID (`author`) directly; the tree will not
|
||||
/// attempt to resolve it to a display name (future enhancement could
|
||||
/// plug in a `users.info` cache).
|
||||
pub fn bucket_to_chat_batch(bucket: &Bucket, channel: &SlackChannel) -> Option<ChatBatch> {
|
||||
if bucket.messages.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let messages = bucket
|
||||
.messages
|
||||
.iter()
|
||||
.map(|m| ChatMessage {
|
||||
author: if m.author.is_empty() {
|
||||
"unknown".to_string()
|
||||
} else {
|
||||
m.author.clone()
|
||||
},
|
||||
timestamp: m.timestamp,
|
||||
text: m.text.clone(),
|
||||
source_ref: Some(slack_archive_url(&channel.id, &m.ts_raw)),
|
||||
})
|
||||
.collect();
|
||||
|
||||
Some(ChatBatch {
|
||||
platform: SLACK_PLATFORM.to_string(),
|
||||
channel_label: channel_label(channel),
|
||||
messages,
|
||||
})
|
||||
}
|
||||
|
||||
/// Render a channel label for the canonical transcript header —
|
||||
/// `"#eng"` for public channels, `"private:ops"` for private ones so
|
||||
/// the retrieval side can distinguish them at a glance.
|
||||
pub fn channel_label(channel: &SlackChannel) -> String {
|
||||
if channel.is_private {
|
||||
format!("private:{}", channel.name)
|
||||
} else {
|
||||
format!("#{}", channel.name)
|
||||
}
|
||||
}
|
||||
|
||||
/// Permalink-shaped pointer to a Slack message. Not a real HTTPS URL —
|
||||
/// we don't know the workspace hostname from the bot token alone — but
|
||||
/// retains enough info for humans to reconstruct one.
|
||||
fn slack_archive_url(channel_id: &str, ts_raw: &str) -> String {
|
||||
// Example: slack://archives/C012345/1714003200.000100
|
||||
format!("slack://archives/{channel_id}/{ts_raw}")
|
||||
}
|
||||
|
||||
/// Ingest one bucket. Returns the tree's [`IngestResult`] so callers
|
||||
/// can surface chunk counts in logs / RPC responses.
|
||||
///
|
||||
/// `connection_id` is the Composio connection (typically one Slack
|
||||
/// workspace per connection). It becomes the `source_id` so all
|
||||
/// channels' all buckets accumulate into ONE source tree per
|
||||
/// workspace — letting the L0 buffer fill across many ingest calls
|
||||
/// and eventually trigger a seal cascade. Chunk-id uniqueness across
|
||||
/// buckets is preserved by `chunk_id` now hashing content.
|
||||
pub async fn ingest_bucket(
|
||||
config: &Config,
|
||||
channel: &SlackChannel,
|
||||
bucket: &Bucket,
|
||||
owner: &str,
|
||||
connection_id: &str,
|
||||
) -> Result<IngestResult> {
|
||||
let source_id = format!("slack:{connection_id}");
|
||||
let batch = match bucket_to_chat_batch(bucket, channel) {
|
||||
Some(b) => b,
|
||||
None => {
|
||||
log::debug!(
|
||||
"[slack_ingest][ops] skip empty bucket channel={} start={}",
|
||||
channel.id,
|
||||
bucket.start.to_rfc3339()
|
||||
);
|
||||
return Ok(IngestResult {
|
||||
source_id: source_id.clone(),
|
||||
chunks_written: 0,
|
||||
chunks_dropped: 0,
|
||||
chunk_ids: Vec::new(),
|
||||
});
|
||||
}
|
||||
};
|
||||
let tags = DEFAULT_TAGS.iter().map(|s| (*s).to_string()).collect();
|
||||
log::info!(
|
||||
"[slack_ingest][ops] ingest bucket channel={} start={} messages={} source_id={}",
|
||||
channel.id,
|
||||
bucket.start.to_rfc3339(),
|
||||
batch.messages.len(),
|
||||
source_id
|
||||
);
|
||||
ingest_chat(config, &source_id, owner, tags, batch)
|
||||
.await
|
||||
.with_context(|| {
|
||||
format!(
|
||||
"[slack_ingest][ops] ingest_chat failed channel={} source_id={}",
|
||||
channel.id, source_id
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Unix seconds for an `end`-of-bucket cursor advance — used by the
|
||||
/// engine to persist `last_synced_ts` after a successful ingest.
|
||||
pub fn cursor_ts_for_flushed_bucket(end: DateTime<Utc>) -> i64 {
|
||||
end.timestamp()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::openhuman::memory::slack_ingestion::types::SlackMessage;
|
||||
use chrono::TimeZone;
|
||||
|
||||
fn sample_channel() -> SlackChannel {
|
||||
SlackChannel {
|
||||
id: "C0123456".into(),
|
||||
name: "eng".into(),
|
||||
is_private: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_message(ts_secs: i64, author: &str, text: &str) -> SlackMessage {
|
||||
SlackMessage {
|
||||
channel_id: "C0123456".into(),
|
||||
author: author.into(),
|
||||
text: text.into(),
|
||||
timestamp: Utc.timestamp_opt(ts_secs, 0).single().unwrap(),
|
||||
ts_raw: format!("{ts_secs}.000000"),
|
||||
thread_ts: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_to_chat_batch_builds_expected_payload() {
|
||||
let channel = sample_channel();
|
||||
let start = Utc.with_ymd_and_hms(2026, 4, 25, 6, 0, 0).unwrap();
|
||||
let end = start + chrono::Duration::hours(6);
|
||||
let bucket = Bucket {
|
||||
start,
|
||||
end,
|
||||
messages: vec![
|
||||
sample_message(start.timestamp() + 60, "U1", "hello"),
|
||||
sample_message(start.timestamp() + 120, "U2", "world"),
|
||||
],
|
||||
};
|
||||
let batch = bucket_to_chat_batch(&bucket, &channel).unwrap();
|
||||
assert_eq!(batch.platform, "slack");
|
||||
assert_eq!(batch.channel_label, "#eng");
|
||||
assert_eq!(batch.messages.len(), 2);
|
||||
assert_eq!(batch.messages[0].author, "U1");
|
||||
assert_eq!(batch.messages[0].text, "hello");
|
||||
assert!(batch.messages[0]
|
||||
.source_ref
|
||||
.as_deref()
|
||||
.unwrap()
|
||||
.starts_with("slack://archives/C0123456/"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_to_chat_batch_none_for_empty_messages() {
|
||||
let channel = sample_channel();
|
||||
let start = Utc.with_ymd_and_hms(2026, 4, 25, 6, 0, 0).unwrap();
|
||||
let bucket = Bucket {
|
||||
start,
|
||||
end: start + chrono::Duration::hours(6),
|
||||
messages: vec![],
|
||||
};
|
||||
assert!(bucket_to_chat_batch(&bucket, &channel).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn channel_label_distinguishes_private() {
|
||||
let pub_ch = sample_channel();
|
||||
let priv_ch = SlackChannel {
|
||||
id: "G1".into(),
|
||||
name: "ops".into(),
|
||||
is_private: true,
|
||||
};
|
||||
assert_eq!(channel_label(&pub_ch), "#eng");
|
||||
assert_eq!(channel_label(&priv_ch), "private:ops");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_author_defaults_to_unknown() {
|
||||
let channel = sample_channel();
|
||||
let start = Utc.with_ymd_and_hms(2026, 4, 25, 6, 0, 0).unwrap();
|
||||
let bucket = Bucket {
|
||||
start,
|
||||
end: start + chrono::Duration::hours(6),
|
||||
messages: vec![sample_message(start.timestamp() + 30, "", "anon")],
|
||||
};
|
||||
let batch = bucket_to_chat_batch(&bucket, &channel).unwrap();
|
||||
assert_eq!(batch.messages[0].author, "unknown");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn slack_archive_url_format_is_stable() {
|
||||
let url = slack_archive_url("C1", "1714003200.000100");
|
||||
assert_eq!(url, "slack://archives/C1/1714003200.000100");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cursor_ts_uses_bucket_end() {
|
||||
let end = Utc.with_ymd_and_hms(2026, 4, 25, 12, 0, 0).unwrap();
|
||||
assert_eq!(cursor_ts_for_flushed_bucket(end), end.timestamp());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
//! JSON-RPC handler functions for the Composio-backed Slack provider.
|
||||
//!
|
||||
//! Public JSON-RPC surface:
|
||||
//! - `openhuman.slack_memory_sync_trigger` — run `SlackProvider::sync()`
|
||||
//! once for each active Slack connection (or just one, if
|
||||
//! `connection_id` is supplied).
|
||||
//! - `openhuman.slack_memory_sync_status` — list the per-connection
|
||||
//! sync cursors + last-synced timestamps.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::openhuman::composio::client::build_composio_client;
|
||||
use crate::openhuman::composio::providers::registry::get_provider;
|
||||
use crate::openhuman::composio::providers::sync_state::SyncState;
|
||||
use crate::openhuman::composio::providers::{ProviderContext, SyncOutcome, SyncReason};
|
||||
use crate::openhuman::config::Config;
|
||||
use crate::openhuman::memory::global::client_if_ready;
|
||||
use crate::rpc::RpcOutcome;
|
||||
|
||||
/// Optional connection-id override for the trigger. When absent, all
|
||||
/// active Slack connections are synced (serially, one-by-one).
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct SyncTriggerRequest {
|
||||
#[serde(default)]
|
||||
pub connection_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct SyncTriggerResponse {
|
||||
pub outcomes: Vec<SyncOutcome>,
|
||||
pub connections_considered: usize,
|
||||
pub connections_synced: usize,
|
||||
}
|
||||
|
||||
/// Run `SlackProvider::sync()` once for every active Slack connection
|
||||
/// (or exactly one, if `connection_id` is provided). Fails if the
|
||||
/// user is not signed in (no Composio JWT available).
|
||||
pub async fn sync_trigger_rpc(
|
||||
config: &Config,
|
||||
req: SyncTriggerRequest,
|
||||
) -> Result<RpcOutcome<SyncTriggerResponse>, String> {
|
||||
let provider = get_provider("slack")
|
||||
.ok_or_else(|| "[slack_ingest] SlackProvider not registered".to_string())?;
|
||||
|
||||
let client = build_composio_client(config).ok_or_else(|| {
|
||||
"[slack_ingest] Composio client unavailable (user not signed in?)".to_string()
|
||||
})?;
|
||||
|
||||
// Discover connections via the backend; filter for slack ones.
|
||||
let connections = client
|
||||
.list_connections()
|
||||
.await
|
||||
.map_err(|e| format!("[slack_ingest] list_connections failed: {e:#}"))?;
|
||||
|
||||
let mut candidates: Vec<_> = connections
|
||||
.connections
|
||||
.into_iter()
|
||||
.filter(|c| {
|
||||
c.toolkit.eq_ignore_ascii_case("slack")
|
||||
&& matches!(c.status.as_str(), "ACTIVE" | "CONNECTED")
|
||||
})
|
||||
.collect();
|
||||
|
||||
if let Some(ref wanted) = req.connection_id {
|
||||
candidates.retain(|c| &c.id == wanted);
|
||||
if candidates.is_empty() {
|
||||
return Err(format!(
|
||||
"[slack_ingest] no active Slack connection with id={wanted}"
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let considered = candidates.len();
|
||||
let config_arc = Arc::new(config.clone());
|
||||
let mut outcomes: Vec<SyncOutcome> = Vec::with_capacity(considered);
|
||||
|
||||
for conn in candidates {
|
||||
let ctx = ProviderContext {
|
||||
config: Arc::clone(&config_arc),
|
||||
client: client.clone(),
|
||||
toolkit: conn.toolkit.clone(),
|
||||
connection_id: Some(conn.id.clone()),
|
||||
};
|
||||
match provider.sync(&ctx, SyncReason::Manual).await {
|
||||
Ok(o) => outcomes.push(o),
|
||||
Err(err) => {
|
||||
log::warn!(
|
||||
"[slack_ingest] connection={} sync failed: {err:#} (continuing)",
|
||||
conn.id
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let synced = outcomes.len();
|
||||
Ok(RpcOutcome::single_log(
|
||||
SyncTriggerResponse {
|
||||
outcomes,
|
||||
connections_considered: considered,
|
||||
connections_synced: synced,
|
||||
},
|
||||
format!("slack_ingest: trigger considered={considered} synced={synced}"),
|
||||
))
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
|
||||
pub struct SyncStatusRequest {}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct SyncStatusResponse {
|
||||
pub connections: Vec<ConnectionStatus>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct ConnectionStatus {
|
||||
pub connection_id: String,
|
||||
/// JSON-encoded per-channel cursors (see
|
||||
/// `composio::providers::slack::sync::ChannelCursors`). Empty map
|
||||
/// when no channels have been flushed yet.
|
||||
pub per_channel_cursors: String,
|
||||
pub synced_ids_count: usize,
|
||||
pub requests_used_today: u32,
|
||||
pub daily_request_limit: u32,
|
||||
}
|
||||
|
||||
/// Report one row per active Slack Composio connection, pulled from
|
||||
/// the Composio sync-state KV store.
|
||||
pub async fn sync_status_rpc(
|
||||
config: &Config,
|
||||
_req: SyncStatusRequest,
|
||||
) -> Result<RpcOutcome<SyncStatusResponse>, String> {
|
||||
let client = build_composio_client(config).ok_or_else(|| {
|
||||
"[slack_ingest] Composio client unavailable (user not signed in?)".to_string()
|
||||
})?;
|
||||
let memory =
|
||||
client_if_ready().ok_or_else(|| "[slack_ingest] memory client not ready".to_string())?;
|
||||
|
||||
let connections = client
|
||||
.list_connections()
|
||||
.await
|
||||
.map_err(|e| format!("[slack_ingest] list_connections failed: {e:#}"))?;
|
||||
|
||||
let mut rows = Vec::new();
|
||||
for conn in connections.connections {
|
||||
if !conn.toolkit.eq_ignore_ascii_case("slack") {
|
||||
continue;
|
||||
}
|
||||
if !matches!(conn.status.as_str(), "ACTIVE" | "CONNECTED") {
|
||||
continue;
|
||||
}
|
||||
let state = match SyncState::load(&memory, "slack", &conn.id).await {
|
||||
Ok(s) => s,
|
||||
Err(err) => {
|
||||
log::warn!(
|
||||
"[slack_ingest] load_state connection={} failed: {err:#}",
|
||||
conn.id
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
rows.push(ConnectionStatus {
|
||||
connection_id: conn.id.clone(),
|
||||
per_channel_cursors: state.cursor.clone().unwrap_or_else(|| "{}".to_string()),
|
||||
synced_ids_count: state.synced_ids.len(),
|
||||
requests_used_today: state.daily_budget.requests_used,
|
||||
daily_request_limit: state.daily_budget.limit,
|
||||
});
|
||||
}
|
||||
|
||||
let count = rows.len();
|
||||
Ok(RpcOutcome::single_log(
|
||||
SyncStatusResponse { connections: rows },
|
||||
format!("slack_ingest: status connections={count}"),
|
||||
))
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
//! Controller schemas + JSON-RPC handler dispatch for the Slack
|
||||
//! ingestion path.
|
||||
//!
|
||||
//! Registered JSON-RPC methods (namespace `slack_memory`):
|
||||
//! - `openhuman.slack_memory_sync_trigger` — run the Composio-backed
|
||||
//! `SlackProvider::sync()` once per active Slack connection.
|
||||
//! - `openhuman.slack_memory_sync_status` — list per-connection
|
||||
//! cursor + dedup + budget state.
|
||||
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::core::all::{ControllerFuture, RegisteredController};
|
||||
use crate::core::{ControllerSchema, FieldSchema, TypeSchema};
|
||||
use crate::openhuman::config::rpc as config_rpc;
|
||||
use crate::openhuman::memory::slack_ingestion::rpc as slack_rpc;
|
||||
use crate::rpc::RpcOutcome;
|
||||
|
||||
const NAMESPACE: &str = "slack_memory";
|
||||
|
||||
pub fn all_controller_schemas() -> Vec<ControllerSchema> {
|
||||
vec![schemas("sync_trigger"), schemas("sync_status")]
|
||||
}
|
||||
|
||||
pub fn all_registered_controllers() -> Vec<RegisteredController> {
|
||||
vec![
|
||||
RegisteredController {
|
||||
schema: schemas("sync_trigger"),
|
||||
handler: handle_sync_trigger,
|
||||
},
|
||||
RegisteredController {
|
||||
schema: schemas("sync_status"),
|
||||
handler: handle_sync_status,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
pub fn schemas(function: &str) -> ControllerSchema {
|
||||
match function {
|
||||
"sync_trigger" => ControllerSchema {
|
||||
namespace: NAMESPACE,
|
||||
function: "sync_trigger",
|
||||
description: "Run the Composio-backed Slack provider sync once per active \
|
||||
Slack connection. When `connection_id` is provided, only that one \
|
||||
connection is synced.",
|
||||
inputs: vec![FieldSchema {
|
||||
name: "connection_id",
|
||||
ty: TypeSchema::String,
|
||||
comment: "Optional — restrict the trigger to one Composio connection id.",
|
||||
required: false,
|
||||
}],
|
||||
outputs: vec![
|
||||
FieldSchema {
|
||||
name: "outcomes",
|
||||
ty: TypeSchema::Array(Box::new(TypeSchema::Ref("SyncOutcome"))),
|
||||
comment: "Per-connection SyncOutcome records returned by SlackProvider::sync.",
|
||||
required: true,
|
||||
},
|
||||
FieldSchema {
|
||||
name: "connections_considered",
|
||||
ty: TypeSchema::I64,
|
||||
comment: "Number of active Slack connections evaluated in this call.",
|
||||
required: true,
|
||||
},
|
||||
FieldSchema {
|
||||
name: "connections_synced",
|
||||
ty: TypeSchema::I64,
|
||||
comment: "Number of connections whose sync completed without error.",
|
||||
required: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
"sync_status" => ControllerSchema {
|
||||
namespace: NAMESPACE,
|
||||
function: "sync_status",
|
||||
description: "List per-connection Slack ingestion state (cursors, synced-id \
|
||||
count, daily budget).",
|
||||
inputs: vec![],
|
||||
outputs: vec![FieldSchema {
|
||||
name: "connections",
|
||||
ty: TypeSchema::Array(Box::new(TypeSchema::Ref("ConnectionStatus"))),
|
||||
comment: "One row per active Slack Composio connection.",
|
||||
required: true,
|
||||
}],
|
||||
},
|
||||
_ => ControllerSchema {
|
||||
namespace: NAMESPACE,
|
||||
function: "unknown",
|
||||
description: "Unknown slack_memory controller function.",
|
||||
inputs: vec![FieldSchema {
|
||||
name: "function",
|
||||
ty: TypeSchema::String,
|
||||
comment: "Unknown function requested for schema lookup.",
|
||||
required: true,
|
||||
}],
|
||||
outputs: vec![FieldSchema {
|
||||
name: "error",
|
||||
ty: TypeSchema::String,
|
||||
comment: "Lookup error details.",
|
||||
required: true,
|
||||
}],
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_sync_trigger(params: Map<String, Value>) -> ControllerFuture {
|
||||
Box::pin(async move {
|
||||
let config = config_rpc::load_config_with_timeout().await?;
|
||||
let req = parse_value::<slack_rpc::SyncTriggerRequest>(Value::Object(params))?;
|
||||
to_json(slack_rpc::sync_trigger_rpc(&config, req).await?)
|
||||
})
|
||||
}
|
||||
|
||||
fn handle_sync_status(params: Map<String, Value>) -> ControllerFuture {
|
||||
Box::pin(async move {
|
||||
let config = config_rpc::load_config_with_timeout().await?;
|
||||
let req = parse_value::<slack_rpc::SyncStatusRequest>(Value::Object(params))?;
|
||||
to_json(slack_rpc::sync_status_rpc(&config, req).await?)
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_value<T: DeserializeOwned>(v: Value) -> Result<T, String> {
|
||||
serde_json::from_value(v).map_err(|e| format!("invalid params: {e}"))
|
||||
}
|
||||
|
||||
fn to_json<T: serde::Serialize>(outcome: RpcOutcome<T>) -> Result<Value, String> {
|
||||
outcome.into_cli_compatible_json()
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
//! Shared types for the Slack ingestion path.
|
||||
//!
|
||||
//! These are intentionally independent of the Slack Web API payload
|
||||
//! shape. Parsing of Composio-wrapped JSON into these structs happens in
|
||||
//! [`crate::openhuman::composio::providers::slack::sync`]; everything
|
||||
//! downstream only deals with the canonical types below.
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// A single message fetched from Slack's `conversations.history`.
|
||||
///
|
||||
/// The Slack API represents `ts` as a decimal string like `"1714003200.123456"`
|
||||
/// where the integer part is Unix seconds and the fractional part is a
|
||||
/// per-workspace message sequence. We retain the original string in `ts_raw`
|
||||
/// so it can round-trip back to the API (e.g. as the `oldest` cursor on the
|
||||
/// next poll, and as the permalink suffix for provenance).
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SlackMessage {
|
||||
/// Channel ID this message belongs to (e.g. `"C0123456"`).
|
||||
pub channel_id: String,
|
||||
/// Slack user ID of the author (e.g. `"U01234"`). Empty if the message
|
||||
/// was a bot/system event we still want to retain (rare).
|
||||
pub author: String,
|
||||
/// Message body (plain text; may contain Slack-flavoured markdown).
|
||||
pub text: String,
|
||||
/// Canonical bucketing timestamp derived from `ts_raw` — Unix millis.
|
||||
pub timestamp: DateTime<Utc>,
|
||||
/// Raw Slack `ts` string (used for API cursors + permalinks).
|
||||
pub ts_raw: String,
|
||||
/// Root thread `ts` if this message is a reply; `None` for channel-level
|
||||
/// messages. Retained for future thread-aware ingestion (v2).
|
||||
pub thread_ts: Option<String>,
|
||||
}
|
||||
|
||||
/// A Slack channel visible to the bot, as returned by `conversations.list`.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SlackChannel {
|
||||
/// Channel ID (stable across renames).
|
||||
pub id: String,
|
||||
/// Human-readable name (e.g. `"eng"` → rendered as `"#eng"` in headers).
|
||||
/// May change if admins rename the channel.
|
||||
pub name: String,
|
||||
/// `true` if this is a private channel the bot has been invited to.
|
||||
pub is_private: bool,
|
||||
}
|
||||
|
||||
/// A closed time window of messages ready for ingest.
|
||||
///
|
||||
/// Created by [`super::bucketer::split_closed`] when every 6-hour UTC
|
||||
/// window older than `now - GRACE_PERIOD` is extracted from the buffer.
|
||||
/// `messages` is non-empty by construction — empty buckets are dropped
|
||||
/// before they reach this type.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Bucket {
|
||||
/// Inclusive bucket start (wall-clock UTC aligned to 00/06/12/18).
|
||||
pub start: DateTime<Utc>,
|
||||
/// Exclusive bucket end (start + 6 hours).
|
||||
pub end: DateTime<Utc>,
|
||||
/// Messages that fall in `[start, end)`, in arrival order.
|
||||
pub messages: Vec<SlackMessage>,
|
||||
}
|
||||
|
||||
// Per-channel cursors are stored by the Composio-backed SlackProvider
|
||||
// via `composio::providers::sync_state::SyncState` (a JSON-encoded
|
||||
// `BTreeMap<channel_id, epoch_secs>` in the `cursor` field). The
|
||||
// standalone `SyncCursor` struct that previously lived here has been
|
||||
// removed along with the SQLite cursor table — retention happens in
|
||||
// the memory KV store now, keyed by (toolkit="slack", connection_id).
|
||||
@@ -66,8 +66,9 @@ pub fn chunk_markdown(input: &ChunkerInput, opts: &ChunkerOptions) -> Vec<Chunk>
|
||||
.map(|(idx, content)| {
|
||||
let seq = idx as u32;
|
||||
let token_count = approx_token_count(&content);
|
||||
let id = super::types::chunk_id(input.source_kind, &input.source_id, seq, &content);
|
||||
Chunk {
|
||||
id: super::types::chunk_id(input.source_kind, &input.source_id, seq),
|
||||
id,
|
||||
content,
|
||||
metadata: input.metadata.clone(),
|
||||
token_count,
|
||||
|
||||
@@ -25,7 +25,7 @@ async fn seed_source_tree_with_sealed_l1(cfg: &Config, scope: &str, ts: DateTime
|
||||
let summariser = InertSummariser::new();
|
||||
|
||||
let c1 = Chunk {
|
||||
id: chunk_id(SourceKind::Chat, scope, 0),
|
||||
id: chunk_id(SourceKind::Chat, scope, 0, "test-content"),
|
||||
content: format!("chunk 1 in {scope}"),
|
||||
metadata: Metadata {
|
||||
source_kind: SourceKind::Chat,
|
||||
@@ -41,7 +41,7 @@ async fn seed_source_tree_with_sealed_l1(cfg: &Config, scope: &str, ts: DateTime
|
||||
created_at: ts,
|
||||
};
|
||||
let c2 = Chunk {
|
||||
id: chunk_id(SourceKind::Chat, scope, 1),
|
||||
id: chunk_id(SourceKind::Chat, scope, 1, "test-content"),
|
||||
content: format!("chunk 2 in {scope}"),
|
||||
metadata: Metadata {
|
||||
source_kind: SourceKind::Chat,
|
||||
|
||||
@@ -202,7 +202,7 @@ mod tests {
|
||||
let tree = get_or_create_source_tree(cfg, scope).unwrap();
|
||||
let summariser = InertSummariser::new();
|
||||
let c1 = Chunk {
|
||||
id: chunk_id(SourceKind::Chat, scope, 0),
|
||||
id: chunk_id(SourceKind::Chat, scope, 0, "test-content"),
|
||||
content: format!("c1-{scope}"),
|
||||
metadata: Metadata {
|
||||
source_kind: SourceKind::Chat,
|
||||
@@ -218,7 +218,7 @@ mod tests {
|
||||
created_at: ts,
|
||||
};
|
||||
let c2 = Chunk {
|
||||
id: chunk_id(SourceKind::Chat, scope, 1),
|
||||
id: chunk_id(SourceKind::Chat, scope, 1, "test-content"),
|
||||
content: format!("c2-{scope}"),
|
||||
metadata: Metadata {
|
||||
source_kind: SourceKind::Chat,
|
||||
|
||||
@@ -22,7 +22,7 @@ use crate::openhuman::memory::tree::chunker::{chunk_markdown, ChunkerInput, Chun
|
||||
use crate::openhuman::memory::tree::score::embed::{build_embedder_from_config, pack_checked};
|
||||
use crate::openhuman::memory::tree::score::{self, ScoreResult, ScoringConfig};
|
||||
use crate::openhuman::memory::tree::source_tree::{
|
||||
append_leaf, get_or_create_source_tree, InertSummariser, LeafRef,
|
||||
append_leaf, get_or_create_source_tree, LeafRef,
|
||||
};
|
||||
use crate::openhuman::memory::tree::store;
|
||||
use crate::openhuman::memory::tree::topic_tree::route_leaf_to_topic_trees;
|
||||
@@ -134,8 +134,11 @@ async fn persist(
|
||||
return Ok(IngestResult::empty(source_id));
|
||||
}
|
||||
|
||||
// 2. Score (async; uses configured extractor)
|
||||
let scoring_cfg = ScoringConfig::default_regex_only();
|
||||
// 2. Score (async; extractor chain driven by config.memory_tree —
|
||||
// regex-only unless llm_extractor_endpoint + model are set, in
|
||||
// which case LlmEntityExtractor runs as second-pass on borderline
|
||||
// chunks with soft-fallback to regex on LLM failure).
|
||||
let scoring_cfg = ScoringConfig::from_config(config);
|
||||
let scores = score::score_chunks(&chunks, &scoring_cfg).await?;
|
||||
|
||||
// Fail fast on scorer length mismatch — silently truncating via zip would
|
||||
@@ -252,8 +255,10 @@ async fn persist(
|
||||
|
||||
/// Push every kept chunk into its source tree. Scoped to Phase 3a — all
|
||||
/// chunks from one ingest batch share the same `source_id`, so they share
|
||||
/// one tree lookup. The inert summariser is the default; future wiring
|
||||
/// can swap in an LLM summariser here.
|
||||
/// one tree lookup. The summariser comes from `build_summariser(config)`:
|
||||
/// Ollama-backed [`LlmSummariser`] when `memory_tree.llm_summariser_*` is
|
||||
/// set, else deterministic [`InertSummariser`]. Both share the same trait,
|
||||
/// so the seal cascade doesn't notice the difference.
|
||||
async fn append_leaves_to_tree(
|
||||
config: &Config,
|
||||
source_id: &str,
|
||||
@@ -264,7 +269,7 @@ async fn append_leaves_to_tree(
|
||||
return Ok(());
|
||||
}
|
||||
let tree = get_or_create_source_tree(config, source_id)?;
|
||||
let summariser = InertSummariser::new();
|
||||
let summariser = crate::openhuman::memory::tree::source_tree::build_summariser(config);
|
||||
|
||||
// Build a chunk_id → (score, entities, topics) map for quick lookup.
|
||||
use std::collections::HashMap;
|
||||
@@ -294,12 +299,14 @@ async fn append_leaves_to_tree(
|
||||
topics,
|
||||
score: score_value,
|
||||
};
|
||||
append_leaf(config, &tree, &leaf, &summariser).await?;
|
||||
append_leaf(config, &tree, &leaf, summariser.as_ref()).await?;
|
||||
|
||||
// Phase 3c (#709): route the leaf to every matching topic tree
|
||||
// and tick the curator for each entity. Non-fatal on error —
|
||||
// the source-tree append has already succeeded above.
|
||||
if let Err(e) = route_leaf_to_topic_trees(config, &leaf, &entities, &summariser).await {
|
||||
if let Err(e) =
|
||||
route_leaf_to_topic_trees(config, &leaf, &entities, summariser.as_ref()).await
|
||||
{
|
||||
log::warn!(
|
||||
"[memory_tree::ingest] topic_tree routing failed chunk_id={} err={:#}",
|
||||
chunk.id,
|
||||
|
||||
@@ -243,7 +243,7 @@ mod tests {
|
||||
let mut leaf_ids: Vec<String> = Vec::new();
|
||||
for seq in 0..2u32 {
|
||||
let c = Chunk {
|
||||
id: chunk_id(SourceKind::Chat, "slack:#eng", seq),
|
||||
id: chunk_id(SourceKind::Chat, "slack:#eng", seq, "test-content"),
|
||||
content: format!("content-{seq}"),
|
||||
metadata: Metadata {
|
||||
source_kind: SourceKind::Chat,
|
||||
|
||||
@@ -99,7 +99,7 @@ mod tests {
|
||||
fn sample_chunk(source: &str, seq: u32) -> Chunk {
|
||||
let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap();
|
||||
Chunk {
|
||||
id: chunk_id(SourceKind::Chat, source, seq),
|
||||
id: chunk_id(SourceKind::Chat, source, seq, "test-content"),
|
||||
content: format!("content-{source}-{seq}"),
|
||||
metadata: Metadata {
|
||||
source_kind: SourceKind::Chat,
|
||||
|
||||
@@ -120,7 +120,7 @@ mod tests {
|
||||
let summariser = InertSummariser::new();
|
||||
for seq in 0..2u32 {
|
||||
let c = Chunk {
|
||||
id: chunk_id(SourceKind::Chat, scope, seq),
|
||||
id: chunk_id(SourceKind::Chat, scope, seq, "test-content"),
|
||||
content: format!("daily-{scope}-{seq}"),
|
||||
metadata: Metadata {
|
||||
source_kind: SourceKind::Chat,
|
||||
|
||||
@@ -202,7 +202,7 @@ async fn seal_populates_summary_embedding() {
|
||||
let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap();
|
||||
|
||||
let mk_chunk = |seq: u32, tokens: u32| Chunk {
|
||||
id: chunk_id(SourceKind::Chat, "slack:#seal-test", seq),
|
||||
id: chunk_id(SourceKind::Chat, "slack:#seal-test", seq, "test-content"),
|
||||
content: format!("substantive chunk content {seq}"),
|
||||
metadata: Metadata {
|
||||
source_kind: SourceKind::Chat,
|
||||
|
||||
@@ -302,7 +302,7 @@ mod tests {
|
||||
fn sample_chunk(source: &str, seq: u32) -> Chunk {
|
||||
let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap();
|
||||
Chunk {
|
||||
id: chunk_id(SourceKind::Chat, source, seq),
|
||||
id: chunk_id(SourceKind::Chat, source, seq, "test-content"),
|
||||
content: format!("content-{source}-{seq}"),
|
||||
metadata: Metadata {
|
||||
source_kind: SourceKind::Chat,
|
||||
|
||||
@@ -317,7 +317,7 @@ mod tests {
|
||||
let summariser = InertSummariser::new();
|
||||
for seq in 0..2u32 {
|
||||
let c = Chunk {
|
||||
id: chunk_id(SourceKind::Chat, scope, seq),
|
||||
id: chunk_id(SourceKind::Chat, scope, seq, "test-content"),
|
||||
content: format!("payload-{scope}-{seq}"),
|
||||
metadata: Metadata {
|
||||
source_kind: SourceKind::Chat,
|
||||
|
||||
@@ -104,6 +104,64 @@ impl ScoringConfig {
|
||||
definite_drop_threshold: DEFAULT_DEFINITE_DROP,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a [`ScoringConfig`] from the workspace [`Config`]. When
|
||||
/// `memory_tree.llm_extractor_endpoint` and `llm_extractor_model`
|
||||
/// are both set, wires [`extract::LlmEntityExtractor`] as the
|
||||
/// second-pass extractor. Otherwise falls back to
|
||||
/// [`Self::default_regex_only`]. Construction errors in the LLM
|
||||
/// extractor (rare — only client-builder failures) also fall back
|
||||
/// to regex-only with a warn log; scoring never blocks on LLM
|
||||
/// availability.
|
||||
pub fn from_config(config: &crate::openhuman::config::Config) -> Self {
|
||||
let endpoint = config
|
||||
.memory_tree
|
||||
.llm_extractor_endpoint
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty());
|
||||
let model = config
|
||||
.memory_tree
|
||||
.llm_extractor_model
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty());
|
||||
|
||||
let (Some(endpoint), Some(model)) = (endpoint, model) else {
|
||||
log::debug!("[memory_tree::score] llm_extractor not configured — using regex-only");
|
||||
return Self::default_regex_only();
|
||||
};
|
||||
|
||||
let timeout_ms = config
|
||||
.memory_tree
|
||||
.llm_extractor_timeout_ms
|
||||
.unwrap_or(15_000);
|
||||
|
||||
let cfg = extract::LlmExtractorConfig {
|
||||
endpoint: endpoint.to_string(),
|
||||
model: model.to_string(),
|
||||
timeout: std::time::Duration::from_millis(timeout_ms),
|
||||
..extract::LlmExtractorConfig::default()
|
||||
};
|
||||
match extract::LlmEntityExtractor::new(cfg) {
|
||||
Ok(llm) => {
|
||||
log::info!(
|
||||
"[memory_tree::score] using LlmEntityExtractor endpoint={} model={} timeout_ms={}",
|
||||
endpoint,
|
||||
model,
|
||||
timeout_ms
|
||||
);
|
||||
Self::with_llm_extractor(Arc::new(llm))
|
||||
}
|
||||
Err(err) => {
|
||||
log::warn!(
|
||||
"[memory_tree::score] LlmEntityExtractor construction failed: {err:#} — \
|
||||
falling back to regex-only"
|
||||
);
|
||||
Self::default_regex_only()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the score for one chunk.
|
||||
|
||||
@@ -5,7 +5,7 @@ use chrono::Utc;
|
||||
fn test_chunk(content: &str) -> Chunk {
|
||||
let meta = Metadata::point_in_time(SourceKind::Email, "t1", "alice", Utc::now());
|
||||
Chunk {
|
||||
id: chunk_id(SourceKind::Email, "t1", 0),
|
||||
id: chunk_id(SourceKind::Email, "t1", 0, "test-content"),
|
||||
content: content.to_string(),
|
||||
token_count: crate::openhuman::memory::tree::types::approx_token_count(content),
|
||||
metadata: meta,
|
||||
|
||||
@@ -1,11 +1,20 @@
|
||||
//! Append + cascade-seal for summary trees (#709).
|
||||
//!
|
||||
//! `append_leaf` pushes a persisted chunk into the L0 buffer of a tree. If
|
||||
//! the buffer's running `token_sum` crosses `TOKEN_BUDGET`, the buffer
|
||||
//! seals into a level-1 summary, its items move into the summary's
|
||||
//! `append_leaf` pushes a persisted chunk into the L0 buffer of a tree.
|
||||
//! Seal gates differ by level:
|
||||
//!
|
||||
//! - **L0 (leaves → L1)**: seal when `token_sum >= TOKEN_BUDGET`. Bounds
|
||||
//! the summariser's raw input.
|
||||
//! - **L≥1 (summaries → next level)**: seal when `item_ids.len() >=
|
||||
//! SUMMARY_FANOUT`. Per-summary token size depends on summariser
|
||||
//! quality, so a token-based gate collapses to a 1:1:1 chain when the
|
||||
//! summariser is weak. Counting siblings keeps the tree's fan-in
|
||||
//! stable regardless.
|
||||
//!
|
||||
//! When a buffer seals, its items move into the new summary's
|
||||
//! `child_ids`, the buffer clears, and the new summary id is queued at
|
||||
//! level 2. The cascade continues upward until a buffer stays under the
|
||||
//! token budget.
|
||||
//! the next level. The cascade continues upward until a buffer fails its
|
||||
//! gate.
|
||||
//!
|
||||
//! Concurrency: Phase 3a assumes a single-process SQLite workspace. All
|
||||
//! writes in one seal step run in a single transaction; the async
|
||||
@@ -29,7 +38,7 @@ use crate::openhuman::memory::tree::source_tree::summariser::{
|
||||
Summariser, SummaryContext, SummaryInput,
|
||||
};
|
||||
use crate::openhuman::memory::tree::source_tree::types::{
|
||||
Buffer, SummaryNode, Tree, TreeKind, TOKEN_BUDGET,
|
||||
Buffer, SummaryNode, Tree, TreeKind, SUMMARY_FANOUT, TOKEN_BUDGET,
|
||||
};
|
||||
use crate::openhuman::memory::tree::store::with_connection;
|
||||
|
||||
@@ -168,8 +177,23 @@ pub async fn cascade_all_from(
|
||||
Ok(sealed_ids)
|
||||
}
|
||||
|
||||
/// Level-aware seal gate.
|
||||
///
|
||||
/// L0 buffers (raw leaves) gate on `token_sum` so the summariser's input
|
||||
/// stays bounded. L≥1 buffers gate on sibling count so the tree's
|
||||
/// fan-in is independent of per-summary token size — without this,
|
||||
/// summarisers that emit at the full token budget (e.g. the inert
|
||||
/// fallback) collapse the cascade into a 1:1:1 chain instead of a real
|
||||
/// tree.
|
||||
fn should_seal(buf: &Buffer) -> bool {
|
||||
!buf.is_empty() && buf.token_sum >= TOKEN_BUDGET as i64
|
||||
if buf.is_empty() {
|
||||
return false;
|
||||
}
|
||||
if buf.level == 0 {
|
||||
buf.token_sum >= TOKEN_BUDGET as i64
|
||||
} else {
|
||||
(buf.item_ids.len() as u32) >= SUMMARY_FANOUT
|
||||
}
|
||||
}
|
||||
|
||||
/// Seal `buf` at `level` into one summary at `level + 1`. Returns the new
|
||||
@@ -227,8 +251,26 @@ async fn seal_one_level(
|
||||
// the buffer stays intact, and a retry re-embeds from scratch. The
|
||||
// tx below would otherwise commit a summary with no embedding,
|
||||
// polluting retrieval's semantic rerank.
|
||||
//
|
||||
// Embedder context-window guard: `nomic-embed-text-v1.5` accepts
|
||||
// up to 8192 tokens of input. Summary content is bounded by
|
||||
// `ctx.token_budget = TOKEN_BUDGET = 10_000` so a worst-case
|
||||
// summary overshoots the embedder. We truncate the input passed
|
||||
// to `embed()` to fit (the persisted summary content stays full;
|
||||
// only the embedding's "view" of it is clamped). 6000 tokens
|
||||
// leaves headroom for tokenizer differences across embedders.
|
||||
let embedder = build_embedder_from_config(config).context("build embedder during seal")?;
|
||||
let embedding = embedder.embed(&output.content).await.with_context(|| {
|
||||
// Conservative cap. Slack-style chat content (URLs, mentions,
|
||||
// emoji) tokenizes 2-4× higher than the 4-chars/token heuristic.
|
||||
// 1000 approx-tokens (~4000 chars) is comfortably under 8192
|
||||
// even at 4× tokenizer ratio.
|
||||
let embed_input = truncate_for_embed(&output.content, 1_000);
|
||||
log::info!(
|
||||
"[source_tree::bucket_seal] embed input: original_chars={} truncated_chars={}",
|
||||
output.content.len(),
|
||||
embed_input.len()
|
||||
);
|
||||
let embedding = embedder.embed(&embed_input).await.with_context(|| {
|
||||
format!(
|
||||
"embed summary during seal tree_id={} level={}",
|
||||
tree.id, level
|
||||
@@ -365,6 +407,21 @@ async fn seal_one_level(
|
||||
Ok(summary_id)
|
||||
}
|
||||
|
||||
/// Clamp `text` to roughly `max_tokens` tokens before passing to the
|
||||
/// embedder. Uses the same ~4 chars/token heuristic as
|
||||
/// `approx_token_count`. Embedders have hard input-size limits (e.g.
|
||||
/// `nomic-embed-text-v1.5` = 8192 tokens) and an overshoot returns
|
||||
/// HTTP 500 from Ollama rather than auto-truncating, which would
|
||||
/// abort the seal transaction.
|
||||
fn truncate_for_embed(text: &str, max_tokens: u32) -> String {
|
||||
let approx = crate::openhuman::memory::tree::types::approx_token_count(text);
|
||||
if approx <= max_tokens {
|
||||
return text.to_string();
|
||||
}
|
||||
let char_ceiling = (max_tokens as usize).saturating_mul(4);
|
||||
text.chars().take(char_ceiling).collect()
|
||||
}
|
||||
|
||||
fn refresh_last_sealed_tx(
|
||||
tx: &Transaction<'_>,
|
||||
tree_id: &str,
|
||||
|
||||
@@ -58,7 +58,7 @@ async fn crossing_budget_triggers_seal() {
|
||||
// Persist two chunks that the hydrator can load during seal.
|
||||
let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap();
|
||||
let mk_chunk = |seq: u32, tokens: u32| Chunk {
|
||||
id: chunk_id(SourceKind::Chat, "slack:#eng", seq),
|
||||
id: chunk_id(SourceKind::Chat, "slack:#eng", seq, "test-content"),
|
||||
content: format!("substantive chunk content {seq}"),
|
||||
metadata: Metadata {
|
||||
source_kind: SourceKind::Chat,
|
||||
@@ -136,3 +136,148 @@ async fn crossing_budget_triggers_seal() {
|
||||
.unwrap();
|
||||
assert_eq!(parent.as_deref(), Some(summary_id.as_str()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fanout_at_l1_triggers_l2_seal() {
|
||||
use crate::openhuman::memory::tree::source_tree::types::SUMMARY_FANOUT;
|
||||
use crate::openhuman::memory::tree::store::upsert_chunks;
|
||||
use crate::openhuman::memory::tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef};
|
||||
use chrono::TimeZone;
|
||||
|
||||
let (_tmp, cfg) = test_config();
|
||||
let tree = get_or_create_source_tree(&cfg, "slack:#eng").unwrap();
|
||||
let summariser = InertSummariser::new();
|
||||
|
||||
let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap();
|
||||
let mk_chunk = |seq: u32| {
|
||||
let content = format!("substantive chunk content {seq}");
|
||||
Chunk {
|
||||
id: chunk_id(SourceKind::Chat, "slack:#eng", seq, &content),
|
||||
content,
|
||||
metadata: Metadata {
|
||||
source_kind: SourceKind::Chat,
|
||||
source_id: "slack:#eng".into(),
|
||||
owner: "alice".into(),
|
||||
timestamp: ts,
|
||||
time_range: (ts, ts),
|
||||
tags: vec![],
|
||||
source_ref: Some(SourceRef::new("slack://x")),
|
||||
},
|
||||
// Each leaf alone busts TOKEN_BUDGET so the L0→L1 seal
|
||||
// fires on every append. After SUMMARY_FANOUT seals, the
|
||||
// L1 buffer's count-based gate trips and cascades to L2.
|
||||
token_count: 10_000,
|
||||
seq_in_source: seq,
|
||||
created_at: ts,
|
||||
}
|
||||
};
|
||||
|
||||
let fanout = SUMMARY_FANOUT;
|
||||
let mut all_sealed: Vec<String> = Vec::new();
|
||||
for seq in 0..fanout {
|
||||
let chunk = mk_chunk(seq);
|
||||
upsert_chunks(&cfg, &[chunk.clone()]).unwrap();
|
||||
let leaf = LeafRef {
|
||||
chunk_id: chunk.id.clone(),
|
||||
token_count: chunk.token_count,
|
||||
timestamp: ts,
|
||||
content: chunk.content.clone(),
|
||||
entities: vec![],
|
||||
topics: vec![],
|
||||
score: 0.5,
|
||||
};
|
||||
let sealed = append_leaf(&cfg, &tree, &leaf, &summariser).await.unwrap();
|
||||
all_sealed.extend(sealed);
|
||||
}
|
||||
|
||||
// First (fanout-1) appends each emit one L1 seal. The final
|
||||
// append emits an L1 seal AND cascades into one L2 seal.
|
||||
assert_eq!(
|
||||
all_sealed.len() as u32,
|
||||
fanout + 1,
|
||||
"expected {} L1 seals + 1 L2 seal, got {}",
|
||||
fanout,
|
||||
all_sealed.len()
|
||||
);
|
||||
|
||||
let t = store::get_tree(&cfg, &tree.id).unwrap().unwrap();
|
||||
assert_eq!(t.max_level, 2, "tree should have climbed to L2");
|
||||
|
||||
let l1 = store::get_buffer(&cfg, &tree.id, 1).unwrap();
|
||||
assert!(
|
||||
l1.is_empty(),
|
||||
"L1 buffer should clear when the fanout seal fires"
|
||||
);
|
||||
|
||||
let l2 = store::get_buffer(&cfg, &tree.id, 2).unwrap();
|
||||
assert_eq!(l2.item_ids.len(), 1, "exactly one L2 summary queued");
|
||||
|
||||
let l2_summary = store::get_summary(&cfg, &l2.item_ids[0]).unwrap().unwrap();
|
||||
assert_eq!(l2_summary.level, 2);
|
||||
assert_eq!(
|
||||
l2_summary.child_ids.len() as u32,
|
||||
fanout,
|
||||
"L2 summary should fold all {fanout} L1 children"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn upper_level_does_not_seal_below_fanout() {
|
||||
use crate::openhuman::memory::tree::source_tree::types::SUMMARY_FANOUT;
|
||||
use crate::openhuman::memory::tree::store::upsert_chunks;
|
||||
use crate::openhuman::memory::tree::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef};
|
||||
use chrono::TimeZone;
|
||||
|
||||
let (_tmp, cfg) = test_config();
|
||||
let tree = get_or_create_source_tree(&cfg, "slack:#eng").unwrap();
|
||||
let summariser = InertSummariser::new();
|
||||
|
||||
let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap();
|
||||
// Emit (fanout - 1) L1 summaries — should leave the L1 buffer
|
||||
// populated but BELOW the count gate, so no L2 seal.
|
||||
let stop_before = SUMMARY_FANOUT.saturating_sub(1);
|
||||
for seq in 0..stop_before {
|
||||
let content = format!("c{seq}");
|
||||
let chunk = Chunk {
|
||||
id: chunk_id(SourceKind::Chat, "slack:#eng", seq, &content),
|
||||
content,
|
||||
metadata: Metadata {
|
||||
source_kind: SourceKind::Chat,
|
||||
source_id: "slack:#eng".into(),
|
||||
owner: "alice".into(),
|
||||
timestamp: ts,
|
||||
time_range: (ts, ts),
|
||||
tags: vec![],
|
||||
source_ref: Some(SourceRef::new("slack://x")),
|
||||
},
|
||||
token_count: 10_000,
|
||||
seq_in_source: seq,
|
||||
created_at: ts,
|
||||
};
|
||||
upsert_chunks(&cfg, &[chunk.clone()]).unwrap();
|
||||
let leaf = LeafRef {
|
||||
chunk_id: chunk.id,
|
||||
token_count: chunk.token_count,
|
||||
timestamp: ts,
|
||||
content: chunk.content,
|
||||
entities: vec![],
|
||||
topics: vec![],
|
||||
score: 0.5,
|
||||
};
|
||||
let _ = append_leaf(&cfg, &tree, &leaf, &summariser).await.unwrap();
|
||||
}
|
||||
|
||||
let t = store::get_tree(&cfg, &tree.id).unwrap().unwrap();
|
||||
assert_eq!(t.max_level, 1, "should plateau at L1 below fanout");
|
||||
|
||||
let l1 = store::get_buffer(&cfg, &tree.id, 1).unwrap();
|
||||
assert_eq!(
|
||||
l1.item_ids.len() as u32,
|
||||
stop_before,
|
||||
"L1 buffer should hold the unsealed siblings"
|
||||
);
|
||||
assert_eq!(
|
||||
store::count_summaries(&cfg, &tree.id).unwrap(),
|
||||
stop_before as u64
|
||||
);
|
||||
}
|
||||
|
||||
@@ -110,7 +110,7 @@ mod tests {
|
||||
// Persist one chunk with an old timestamp (10 days ago).
|
||||
let old_ts = Utc::now() - Duration::days(10);
|
||||
let c = Chunk {
|
||||
id: chunk_id(SourceKind::Chat, "slack:#eng", 0),
|
||||
id: chunk_id(SourceKind::Chat, "slack:#eng", 0, "test-content"),
|
||||
content: "old content that should get sealed".into(),
|
||||
metadata: Metadata {
|
||||
source_kind: SourceKind::Chat,
|
||||
@@ -158,7 +158,7 @@ mod tests {
|
||||
// Persist a leaf stamped now so it's NOT stale.
|
||||
let now = Utc::now();
|
||||
let c = Chunk {
|
||||
id: chunk_id(SourceKind::Chat, "slack:#eng", 0),
|
||||
id: chunk_id(SourceKind::Chat, "slack:#eng", 0, "test-content"),
|
||||
content: "fresh".into(),
|
||||
metadata: Metadata {
|
||||
source_kind: SourceKind::Chat,
|
||||
|
||||
@@ -25,5 +25,5 @@ pub mod types;
|
||||
pub use bucket_seal::{append_leaf, LeafRef};
|
||||
pub use registry::get_or_create_source_tree;
|
||||
pub use store::{get_summary_embedding, set_summary_embedding};
|
||||
pub use summariser::{inert::InertSummariser, Summariser};
|
||||
pub use summariser::{build_summariser, inert::InertSummariser, llm::LlmSummariser, Summariser};
|
||||
pub use types::{Buffer, SummaryNode, Tree, TreeKind, TreeStatus, TOKEN_BUDGET};
|
||||
|
||||
@@ -0,0 +1,607 @@
|
||||
//! LLM-backed summariser — Ollama `/api/chat` peer of
|
||||
//! [`crate::openhuman::memory::tree::score::extract::llm::LlmEntityExtractor`].
|
||||
//!
|
||||
//! ## Responsibility
|
||||
//!
|
||||
//! When the source / topic / global tree's bucket-seal cascade decides to
|
||||
//! fold N contributions (raw leaves at L0→L1, or lower-level summaries at
|
||||
//! L_n→L_{n+1}), this summariser is asked to produce the parent node's
|
||||
//! `content` + derived `entities` + `topics`. The seal machinery itself
|
||||
//! (bucket budgeting, level promotion, `mem_tree_summaries` persistence)
|
||||
//! is unchanged — only the text inside the summary row differs from
|
||||
//! [`super::inert::InertSummariser`].
|
||||
//!
|
||||
//! ## Soft-fallback contract
|
||||
//!
|
||||
//! A summariser that returns `Err` would abort the seal cascade and leave
|
||||
//! the tree in an inconsistent state — a half-sealed buffer with no
|
||||
//! parent row. We therefore promise **never** to return `Err`: every
|
||||
//! failure (transport, HTTP status, JSON shape) falls back to the same
|
||||
//! deterministic concat-and-truncate behaviour as `InertSummariser` and
|
||||
//! logs a warn. Callers distinguish "LLM ran fine" from "we fell back"
|
||||
//! only by observing whether the returned `entities`/`topics` are
|
||||
//! populated — the inert branch emits empty vecs.
|
||||
//!
|
||||
//! ## Prompt shape
|
||||
//!
|
||||
//! The system prompt commits the model to returning JSON with the shape
|
||||
//! `{ summary, entities, topics }`. We use Ollama's `format: "json"` +
|
||||
//! `temperature: 0.0` to maximise determinism — same knobs the entity
|
||||
//! extractor already uses with success.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use reqwest::Client;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::inert::InertSummariser;
|
||||
use super::{Summariser, SummaryContext, SummaryInput, SummaryOutput};
|
||||
use crate::openhuman::memory::tree::types::approx_token_count;
|
||||
|
||||
/// Hard cap on summary OUTPUT tokens, regardless of the seal's
|
||||
/// `ctx.token_budget`. Driven by the embedder's context window —
|
||||
/// `nomic-embed-text-v1.5` accepts up to 8192 tokens and Phase 4
|
||||
/// (`source_tree::bucket_seal`) embeds the summary right after we
|
||||
/// produce it. If our summary overshoots, the embedder returns 500
|
||||
/// and the whole seal transaction rolls back → no summary persists.
|
||||
/// 6000 leaves a safety margin for tokenizer differences and the
|
||||
/// JSON wrapper.
|
||||
const MAX_SUMMARY_OUTPUT_TOKENS: u32 = 6_000;
|
||||
|
||||
/// Context window we ask Ollama for. Must match the value below in
|
||||
/// [`OllamaOptions::num_ctx`] so the per-input clamp computed in
|
||||
/// [`LlmSummariser::summarise`] sizes inputs against the same window
|
||||
/// the model actually sees.
|
||||
const NUM_CTX_TOKENS: u32 = 16_384;
|
||||
|
||||
/// Tokens reserved for the system prompt, JSON wrapper, and tokenizer
|
||||
/// drift between our 4-chars/token heuristic and the model's tokenizer.
|
||||
/// Trades a small loss of input capacity for a guarantee that the
|
||||
/// prompt body + output budget never exceeds `num_ctx`.
|
||||
const OVERHEAD_RESERVE_TOKENS: u32 = 512;
|
||||
|
||||
/// Configuration for [`LlmSummariser`]. Endpoint + model defaults match
|
||||
/// [`crate::openhuman::memory::tree::score::extract::llm::LlmExtractorConfig`]
|
||||
/// so a workspace configured for one LLM path also satisfies the other
|
||||
/// by default.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct LlmSummariserConfig {
|
||||
/// Base URL of the Ollama-compatible endpoint (e.g.
|
||||
/// `http://localhost:11434`). Do NOT include `/api/chat` — the
|
||||
/// summariser appends it.
|
||||
pub endpoint: String,
|
||||
/// Model identifier (e.g. `qwen2.5:0.5b` or `llama3.1:8b`).
|
||||
pub model: String,
|
||||
/// Per-request timeout. Generous default because first-call weight
|
||||
/// loading can be slow.
|
||||
pub timeout: Duration,
|
||||
}
|
||||
|
||||
impl Default for LlmSummariserConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
endpoint: "http://localhost:11434".to_string(),
|
||||
model: "qwen2.5:0.5b".to_string(),
|
||||
// 120s — generous enough for small/medium models (1B-8B
|
||||
// params) summarising the seal cascade's full token
|
||||
// budget on first invocation, when Ollama may also be
|
||||
// loading model weights into VRAM. Large models on CPU
|
||||
// can still time out; bump via config for those.
|
||||
timeout: Duration::from_secs(120),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// LLM-backed summariser. Delegates to [`InertSummariser`] on any
|
||||
/// failure so seal cascades never fail.
|
||||
pub struct LlmSummariser {
|
||||
cfg: LlmSummariserConfig,
|
||||
http: Client,
|
||||
fallback: InertSummariser,
|
||||
}
|
||||
|
||||
impl LlmSummariser {
|
||||
pub fn new(cfg: LlmSummariserConfig) -> Result<Self> {
|
||||
let http = Client::builder()
|
||||
.timeout(cfg.timeout)
|
||||
.build()
|
||||
.map_err(anyhow::Error::from)?;
|
||||
Ok(Self {
|
||||
cfg,
|
||||
http,
|
||||
fallback: InertSummariser::new(),
|
||||
})
|
||||
}
|
||||
|
||||
fn build_request(&self, prompt_body: &str, budget: u32) -> OllamaChatRequest {
|
||||
OllamaChatRequest {
|
||||
model: self.cfg.model.clone(),
|
||||
messages: vec![
|
||||
OllamaMessage {
|
||||
role: "system".to_string(),
|
||||
content: system_prompt(budget),
|
||||
},
|
||||
OllamaMessage {
|
||||
role: "user".to_string(),
|
||||
content: prompt_body.to_string(),
|
||||
},
|
||||
],
|
||||
format: "json".to_string(),
|
||||
stream: false,
|
||||
options: OllamaOptions {
|
||||
temperature: 0.0,
|
||||
// 16k context window. Sized so that the per-input
|
||||
// clamp in `summarise` (NUM_CTX - output_budget -
|
||||
// overhead, divided by `inputs.len()`) keeps the
|
||||
// joined prompt body inside this window even at
|
||||
// upper-level seals where SUMMARY_FANOUT children
|
||||
// each near MAX_SUMMARY_OUTPUT_TOKENS would otherwise
|
||||
// overflow. Keeping `num_ctx` modest also keeps the
|
||||
// kv-cache small enough to fit on consumer GPUs
|
||||
// alongside the model weights.
|
||||
num_ctx: NUM_CTX_TOKENS,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Summariser for LlmSummariser {
|
||||
async fn summarise(
|
||||
&self,
|
||||
inputs: &[SummaryInput],
|
||||
ctx: &SummaryContext<'_>,
|
||||
) -> Result<SummaryOutput> {
|
||||
// Clamp the model-side output budget so the summary fits the
|
||||
// downstream embedder. The seal-cascade hands us
|
||||
// `ctx.token_budget = 10k` by default but `nomic-embed-text`
|
||||
// only accepts ≤ 8k tokens of input. Producing a smaller
|
||||
// summary upfront avoids the embed-fails-after-summary
|
||||
// dead end.
|
||||
let effective_budget = ctx.token_budget.min(MAX_SUMMARY_OUTPUT_TOKENS);
|
||||
|
||||
// Per-input clamp scaled by fanout. Without this, an upper-level
|
||||
// seal feeding `SUMMARY_FANOUT=4` children each near
|
||||
// `MAX_SUMMARY_OUTPUT_TOKENS` would push the prompt body alone
|
||||
// past `num_ctx` and Ollama would silently truncate (or error).
|
||||
// Divide the input budget evenly across contributors.
|
||||
let per_input_cap = if inputs.is_empty() {
|
||||
0
|
||||
} else {
|
||||
NUM_CTX_TOKENS
|
||||
.saturating_sub(effective_budget)
|
||||
.saturating_sub(OVERHEAD_RESERVE_TOKENS)
|
||||
/ inputs.len() as u32
|
||||
};
|
||||
|
||||
// Assemble the user-side prompt. We prefix each contribution with
|
||||
// its id so the model can weigh them and so log diffs are
|
||||
// traceable to source rows if anything looks odd.
|
||||
let body = build_user_prompt(inputs, per_input_cap);
|
||||
if body.trim().is_empty() {
|
||||
log::debug!(
|
||||
"[source_tree::summariser::llm] empty prompt body (no non-blank inputs) \
|
||||
tree_id={} level={} — returning empty summary",
|
||||
ctx.tree_id,
|
||||
ctx.target_level
|
||||
);
|
||||
return Ok(SummaryOutput {
|
||||
content: String::new(),
|
||||
token_count: 0,
|
||||
entities: Vec::new(),
|
||||
topics: Vec::new(),
|
||||
});
|
||||
}
|
||||
|
||||
let url = format!("{}/api/chat", self.cfg.endpoint.trim_end_matches('/'));
|
||||
let req = self.build_request(&body, effective_budget);
|
||||
|
||||
log::debug!(
|
||||
"[source_tree::summariser::llm] POST {url} model={} tree_id={} level={} \
|
||||
inputs={} budget={}",
|
||||
self.cfg.model,
|
||||
ctx.tree_id,
|
||||
ctx.target_level,
|
||||
inputs.len(),
|
||||
ctx.token_budget
|
||||
);
|
||||
|
||||
let resp = match self.http.post(&url).json(&req).send().await {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
log::warn!(
|
||||
"[source_tree::summariser::llm] transport failure to {url}: {e} — \
|
||||
falling back to inert summariser for tree_id={} level={}",
|
||||
ctx.tree_id,
|
||||
ctx.target_level
|
||||
);
|
||||
return self.fallback.summarise(inputs, ctx).await;
|
||||
}
|
||||
};
|
||||
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
log::warn!(
|
||||
"[source_tree::summariser::llm] ollama non-success status {status} \
|
||||
tree_id={} level={}: {} — falling back to inert",
|
||||
ctx.tree_id,
|
||||
ctx.target_level,
|
||||
truncate_for_log(&body, 200)
|
||||
);
|
||||
return self.fallback.summarise(inputs, ctx).await;
|
||||
}
|
||||
|
||||
let envelope: OllamaChatResponse = match resp.json().await {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
log::warn!(
|
||||
"[source_tree::summariser::llm] response not Ollama-shaped JSON: {e} — \
|
||||
falling back to inert for tree_id={} level={}",
|
||||
ctx.tree_id,
|
||||
ctx.target_level
|
||||
);
|
||||
return self.fallback.summarise(inputs, ctx).await;
|
||||
}
|
||||
};
|
||||
|
||||
let parsed: LlmSummaryOutput = match serde_json::from_str(&envelope.message.content) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
log::warn!(
|
||||
"[source_tree::summariser::llm] model returned non-JSON or wrong-shape \
|
||||
body: {e}; content was: {} — falling back to inert",
|
||||
truncate_for_log(&envelope.message.content, 400)
|
||||
);
|
||||
return self.fallback.summarise(inputs, ctx).await;
|
||||
}
|
||||
};
|
||||
|
||||
let (content, token_count) = clamp_to_budget(&parsed.summary, effective_budget);
|
||||
log::debug!(
|
||||
"[source_tree::summariser::llm] sealed tree_id={} level={} inputs={} tokens={} \
|
||||
surface_entities_dropped={} topics={}",
|
||||
ctx.tree_id,
|
||||
ctx.target_level,
|
||||
inputs.len(),
|
||||
token_count,
|
||||
parsed.entities.len(),
|
||||
parsed.topics.len()
|
||||
);
|
||||
|
||||
// Drop LLM-emitted entities. The model returns surface forms
|
||||
// ("Alice", "she"), but `SummaryNode.entities` is indexed via
|
||||
// `index_summary_entity_ids_tx` as canonical ids. Surface forms
|
||||
// would silently corrupt that index — searches by canonical id
|
||||
// would not find these summaries. Canonicalisation is the
|
||||
// entity extractor's job, not the summariser's. Topics stay
|
||||
// because they're free-form labels, not indexed as canonical
|
||||
// ids. The `entities` field stays in the prompt to nudge the
|
||||
// model toward entity-aware summarisation; we just don't
|
||||
// persist its output.
|
||||
Ok(SummaryOutput {
|
||||
content,
|
||||
token_count,
|
||||
entities: Vec::new(),
|
||||
topics: dedupe_sorted(parsed.topics),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the user-message body that precedes the model call. Each
|
||||
/// contribution is prefixed with a short id header and separated by a
|
||||
/// blank line — matches the layout the model is instructed to
|
||||
/// summarise. Each input's content is clamped to
|
||||
/// `per_input_cap_tokens` so the joined body fits inside `num_ctx` even
|
||||
/// at upper-level seals where many large summaries fold together. A
|
||||
/// `0` cap means "don't include any content" (used when there are no
|
||||
/// inputs); pass `u32::MAX` to disable clamping.
|
||||
fn build_user_prompt(inputs: &[SummaryInput], per_input_cap_tokens: u32) -> String {
|
||||
let mut out = String::new();
|
||||
for inp in inputs {
|
||||
let trimmed = inp.content.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let (clamped, _) = clamp_to_budget(trimmed, per_input_cap_tokens);
|
||||
if !out.is_empty() {
|
||||
out.push_str("\n\n");
|
||||
}
|
||||
out.push_str(&format!("[{}]\n{clamped}", inp.id));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// System prompt. Token budget is templated in so the model aims under it.
|
||||
fn system_prompt(budget: u32) -> String {
|
||||
format!(
|
||||
"You are a precise summariser. Summarise the user-provided contributions into a \
|
||||
single cohesive passage that preserves concrete facts, decisions, named entities, \
|
||||
and temporal ordering. Do not invent facts. Stay well under {budget} tokens.\n\
|
||||
\n\
|
||||
Return JSON only — no prose, no markdown, no commentary. Schema:\n\
|
||||
{{\n\
|
||||
\x20 \"summary\": \"<summary body>\",\n\
|
||||
\x20 \"entities\": [\"<named entity surface forms mentioned in the summary>\"],\n\
|
||||
\x20 \"topics\": [\"<short topic labels covered by the summary>\"]\n\
|
||||
}}"
|
||||
)
|
||||
}
|
||||
|
||||
/// Truncate to the caller's token budget using the same ~4 chars/token
|
||||
/// heuristic as [`InertSummariser`].
|
||||
fn clamp_to_budget(text: &str, budget: u32) -> (String, u32) {
|
||||
let initial = approx_token_count(text);
|
||||
if initial <= budget {
|
||||
return (text.to_string(), initial);
|
||||
}
|
||||
let char_ceiling = (budget as usize).saturating_mul(4);
|
||||
let truncated: String = text.chars().take(char_ceiling).collect();
|
||||
let tokens = approx_token_count(&truncated);
|
||||
(truncated, tokens)
|
||||
}
|
||||
|
||||
fn dedupe_sorted(mut items: Vec<String>) -> Vec<String> {
|
||||
for item in items.iter_mut() {
|
||||
*item = item.trim().to_string();
|
||||
}
|
||||
items.retain(|s| !s.is_empty());
|
||||
items.sort();
|
||||
items.dedup();
|
||||
items
|
||||
}
|
||||
|
||||
fn truncate_for_log(s: &str, max_chars: usize) -> String {
|
||||
if s.chars().count() <= max_chars {
|
||||
return s.to_string();
|
||||
}
|
||||
let truncated: String = s.chars().take(max_chars).collect();
|
||||
format!("{truncated}…")
|
||||
}
|
||||
|
||||
// ── Wire types (Ollama API) ──────────────────────────────────────────────
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct OllamaChatRequest {
|
||||
model: String,
|
||||
messages: Vec<OllamaMessage>,
|
||||
format: String,
|
||||
stream: bool,
|
||||
options: OllamaOptions,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct OllamaMessage {
|
||||
role: String,
|
||||
content: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct OllamaOptions {
|
||||
temperature: f32,
|
||||
/// Override Ollama's default 32k context window — large local
|
||||
/// models inflate kv-cache to >15 GiB at 32k which won't fit on
|
||||
/// consumer GPUs. 16k is plenty for one bucket summary (input is
|
||||
/// bounded by the source-tree's ~10k-token bucket budget) while
|
||||
/// keeping kv-cache reasonable.
|
||||
num_ctx: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct OllamaChatResponse {
|
||||
message: OllamaResponseMessage,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct OllamaResponseMessage {
|
||||
content: String,
|
||||
}
|
||||
|
||||
// ── LLM JSON output ──────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct LlmSummaryOutput {
|
||||
#[serde(default)]
|
||||
summary: String,
|
||||
#[serde(default)]
|
||||
entities: Vec<String>,
|
||||
#[serde(default)]
|
||||
topics: Vec<String>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::openhuman::memory::tree::source_tree::types::TreeKind;
|
||||
use chrono::Utc;
|
||||
|
||||
fn sample_input(id: &str, content: &str) -> SummaryInput {
|
||||
let ts = Utc::now();
|
||||
SummaryInput {
|
||||
id: id.to_string(),
|
||||
content: content.to_string(),
|
||||
token_count: approx_token_count(content),
|
||||
entities: Vec::new(),
|
||||
topics: Vec::new(),
|
||||
time_range_start: ts,
|
||||
time_range_end: ts,
|
||||
score: 0.5,
|
||||
}
|
||||
}
|
||||
|
||||
fn test_ctx() -> SummaryContext<'static> {
|
||||
SummaryContext {
|
||||
tree_id: "tree-1",
|
||||
tree_kind: TreeKind::Source,
|
||||
target_level: 1,
|
||||
token_budget: 10_000,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_user_prompt_includes_ids_and_content() {
|
||||
let inputs = vec![
|
||||
sample_input("a", "hello world"),
|
||||
sample_input("b", "second contribution"),
|
||||
];
|
||||
let out = build_user_prompt(&inputs, u32::MAX);
|
||||
assert!(out.contains("[a]"));
|
||||
assert!(out.contains("hello world"));
|
||||
assert!(out.contains("[b]"));
|
||||
assert!(out.contains("second contribution"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_user_prompt_skips_blank_contributions() {
|
||||
let inputs = vec![sample_input("a", " "), sample_input("b", "kept")];
|
||||
let out = build_user_prompt(&inputs, u32::MAX);
|
||||
assert!(!out.contains("[a]"));
|
||||
assert!(out.contains("[b]"));
|
||||
assert!(out.contains("kept"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_user_prompt_clamps_each_input_to_per_input_cap() {
|
||||
// Regression guard for upper-level context overflow: at L2 with
|
||||
// SUMMARY_FANOUT=4 and large child summaries, the joined body
|
||||
// would otherwise blow past NUM_CTX_TOKENS. The clamp keeps
|
||||
// each contribution under per_input_cap_tokens regardless of
|
||||
// how big the original content is.
|
||||
let long = "x".repeat(2_000); // ~500 approx-tokens
|
||||
let inputs = vec![
|
||||
sample_input("a", &long),
|
||||
sample_input("b", &long),
|
||||
sample_input("c", &long),
|
||||
sample_input("d", &long),
|
||||
];
|
||||
let cap_tokens: u32 = 50; // ~200 chars per input
|
||||
let out = build_user_prompt(&inputs, cap_tokens);
|
||||
|
||||
// Each input contributes at most cap_tokens*4 chars of content,
|
||||
// plus a small id header. Total stays well under the unclamped
|
||||
// 4 * 2_000 = 8_000 chars baseline.
|
||||
let unclamped_baseline = 4 * 2_000;
|
||||
assert!(
|
||||
out.len() < unclamped_baseline / 2,
|
||||
"expected clamp to halve the body or better, got {} chars",
|
||||
out.len()
|
||||
);
|
||||
assert!(out.contains("[a]"));
|
||||
assert!(out.contains("[d]"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn system_prompt_templates_budget() {
|
||||
let p = system_prompt(4096);
|
||||
assert!(p.contains("4096"));
|
||||
assert!(p.contains("\"summary\""));
|
||||
assert!(p.contains("\"entities\""));
|
||||
assert!(p.contains("\"topics\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clamp_to_budget_no_op_when_under() {
|
||||
let (out, t) = clamp_to_budget("short", 1000);
|
||||
assert_eq!(out, "short");
|
||||
assert_eq!(t, approx_token_count("short"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clamp_to_budget_truncates_when_over() {
|
||||
let long = "a".repeat(1000);
|
||||
let (out, t) = clamp_to_budget(&long, 5);
|
||||
assert!(out.len() < long.len());
|
||||
assert!(t <= 6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dedupe_sorted_trims_and_dedupes() {
|
||||
let out = dedupe_sorted(vec![
|
||||
"Bob".into(),
|
||||
" Alice ".into(),
|
||||
"Bob".into(),
|
||||
"".into(),
|
||||
" ".into(),
|
||||
"Alice".into(),
|
||||
]);
|
||||
assert_eq!(out, vec!["Alice", "Bob"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncate_for_log_short_input_unchanged() {
|
||||
assert_eq!(truncate_for_log("hi", 10), "hi");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncate_for_log_long_input_appends_ellipsis() {
|
||||
let long = "x".repeat(500);
|
||||
let out = truncate_for_log(&long, 10);
|
||||
assert_eq!(out.chars().count(), 11);
|
||||
assert!(out.ends_with('…'));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_inputs_yield_empty_summary_without_network_call() {
|
||||
// All inputs are blank → prompt body is empty → the summariser
|
||||
// short-circuits and returns an empty output. Importantly, this
|
||||
// path must work even if Ollama is unreachable.
|
||||
let cfg = LlmSummariserConfig {
|
||||
endpoint: "http://127.0.0.1:1".to_string(),
|
||||
timeout: Duration::from_millis(50),
|
||||
..LlmSummariserConfig::default()
|
||||
};
|
||||
let s = LlmSummariser::new(cfg).unwrap();
|
||||
let inputs = vec![sample_input("a", " "), sample_input("b", "")];
|
||||
let out = s.summarise(&inputs, &test_ctx()).await.unwrap();
|
||||
assert!(out.content.is_empty());
|
||||
assert_eq!(out.token_count, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transport_failure_falls_back_to_inert() {
|
||||
// Unreachable endpoint → transport error → must NOT return Err;
|
||||
// must fall through to InertSummariser's concatenate+truncate
|
||||
// behaviour (content present, entities empty).
|
||||
let cfg = LlmSummariserConfig {
|
||||
endpoint: "http://127.0.0.1:1".to_string(),
|
||||
timeout: Duration::from_millis(100),
|
||||
..LlmSummariserConfig::default()
|
||||
};
|
||||
let s = LlmSummariser::new(cfg).unwrap();
|
||||
let inputs = vec![sample_input("a", "alice decided to ship friday")];
|
||||
let out = s.summarise(&inputs, &test_ctx()).await.unwrap();
|
||||
assert!(out.content.contains("alice decided to ship"));
|
||||
// Inert branch emits empty entities/topics — this is how callers
|
||||
// can distinguish fallback from a real LLM success with no entities.
|
||||
assert!(out.entities.is_empty());
|
||||
assert!(out.topics.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_request_uses_configured_model_and_json_format() {
|
||||
let cfg = LlmSummariserConfig {
|
||||
model: "llama3.1:8b".into(),
|
||||
..LlmSummariserConfig::default()
|
||||
};
|
||||
let s = LlmSummariser::new(cfg).unwrap();
|
||||
let req = s.build_request("body", 2048);
|
||||
assert_eq!(req.model, "llama3.1:8b");
|
||||
assert_eq!(req.format, "json");
|
||||
assert!(!req.stream);
|
||||
assert_eq!(req.options.temperature, 0.0);
|
||||
assert_eq!(req.messages[0].role, "system");
|
||||
assert!(req.messages[0].content.contains("2048"));
|
||||
assert_eq!(req.messages[1].role, "user");
|
||||
assert_eq!(req.messages[1].content, "body");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn llm_output_deserialises_with_missing_fields() {
|
||||
let v: LlmSummaryOutput = serde_json::from_str(r#"{"summary":"hi"}"#).unwrap();
|
||||
assert_eq!(v.summary, "hi");
|
||||
assert!(v.entities.is_empty());
|
||||
assert!(v.topics.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -10,9 +10,13 @@ use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::openhuman::config::Config;
|
||||
use crate::openhuman::memory::tree::source_tree::types::TreeKind;
|
||||
|
||||
pub mod inert;
|
||||
pub mod llm;
|
||||
|
||||
/// One contribution being folded — either a raw leaf (chunk) at L0→L1, or
|
||||
/// a lower-level summary at L_n→L_{n+1}.
|
||||
@@ -60,3 +64,66 @@ pub trait Summariser: Send + Sync {
|
||||
ctx: &SummaryContext<'_>,
|
||||
) -> Result<SummaryOutput>;
|
||||
}
|
||||
|
||||
/// Build the summariser implementation driven by the workspace's
|
||||
/// [`Config`]. When `memory_tree.llm_summariser_endpoint` and
|
||||
/// `llm_summariser_model` are both set, return the Ollama-backed
|
||||
/// [`llm::LlmSummariser`] (which itself soft-falls-back to inert on
|
||||
/// transport failure). Otherwise return [`inert::InertSummariser`].
|
||||
///
|
||||
/// Returned as `Arc<dyn Summariser>` so the ingest pipeline can pass it
|
||||
/// by reference to `append_leaf` and `route_leaf_to_topic_trees`
|
||||
/// without threading a generic type parameter through every caller.
|
||||
pub fn build_summariser(config: &Config) -> Arc<dyn Summariser> {
|
||||
let endpoint = config
|
||||
.memory_tree
|
||||
.llm_summariser_endpoint
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty());
|
||||
let model = config
|
||||
.memory_tree
|
||||
.llm_summariser_model
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty());
|
||||
|
||||
let (Some(endpoint), Some(model)) = (endpoint, model) else {
|
||||
log::debug!(
|
||||
"[source_tree::summariser] llm_summariser not configured — using InertSummariser"
|
||||
);
|
||||
return Arc::new(inert::InertSummariser::new());
|
||||
};
|
||||
|
||||
// 120s default — matches `LlmSummariserConfig::default()`. Lets a
|
||||
// small/medium local model finish the seal-budget summary on a
|
||||
// cold-loaded weight cache without spurious timeouts.
|
||||
let timeout_ms = config
|
||||
.memory_tree
|
||||
.llm_summariser_timeout_ms
|
||||
.unwrap_or(120_000);
|
||||
|
||||
let cfg = llm::LlmSummariserConfig {
|
||||
endpoint: endpoint.to_string(),
|
||||
model: model.to_string(),
|
||||
timeout: std::time::Duration::from_millis(timeout_ms),
|
||||
};
|
||||
match llm::LlmSummariser::new(cfg) {
|
||||
Ok(s) => {
|
||||
log::info!(
|
||||
"[source_tree::summariser] using LlmSummariser endpoint={} model={} timeout_ms={}",
|
||||
endpoint,
|
||||
model,
|
||||
timeout_ms
|
||||
);
|
||||
Arc::new(s)
|
||||
}
|
||||
Err(err) => {
|
||||
log::warn!(
|
||||
"[source_tree::summariser] LlmSummariser construction failed: {err:#} — \
|
||||
falling back to InertSummariser"
|
||||
);
|
||||
Arc::new(inert::InertSummariser::new())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -174,8 +174,22 @@ impl Buffer {
|
||||
|
||||
/// Token ceiling for one summariser invocation — aligned with the Phase 1
|
||||
/// chunker ceiling so a single leaf never busts a seal on its own.
|
||||
///
|
||||
/// Gates only the L0 → L1 seal: leaves are fan-in by raw token volume so
|
||||
/// the summariser input stays bounded. Summaries above L0 use
|
||||
/// [`SUMMARY_FANOUT`] instead — see `bucket_seal::should_seal`.
|
||||
pub const TOKEN_BUDGET: u32 = 10_000;
|
||||
|
||||
/// Sibling count that triggers a seal at level ≥ 1 (summaries → next level).
|
||||
///
|
||||
/// Decouples upper-level seals from per-summary token size so the tree's
|
||||
/// fan-in stays stable regardless of summariser quality. With a real
|
||||
/// summariser each L1 might be ~500 tokens; with the inert fallback each
|
||||
/// L1 fills the full [`TOKEN_BUDGET`]. Token-based gating collapses the
|
||||
/// inert case into a 1:1:1 chain — count-based gating gives a real tree
|
||||
/// shape (`SUMMARY_FANOUT` children per parent) in both cases.
|
||||
pub const SUMMARY_FANOUT: u32 = 4;
|
||||
|
||||
/// Default age at which a non-empty buffer is force-sealed even under the
|
||||
/// token budget. Keeps recent activity from stalling waiting for more
|
||||
/// leaves that may never arrive.
|
||||
|
||||
@@ -13,7 +13,7 @@ fn test_config() -> (TempDir, Config) {
|
||||
fn sample_chunk(source_id: &str, seq: u32, ts_ms: i64) -> Chunk {
|
||||
let ts = Utc.timestamp_millis_opt(ts_ms).unwrap();
|
||||
Chunk {
|
||||
id: chunk_id(SourceKind::Chat, source_id, seq),
|
||||
id: chunk_id(SourceKind::Chat, source_id, seq, "test-content"),
|
||||
content: format!("content {source_id} {seq}"),
|
||||
metadata: Metadata {
|
||||
source_kind: SourceKind::Chat,
|
||||
|
||||
@@ -144,7 +144,7 @@ mod tests {
|
||||
fn mk_chunk(source_id: &str, seq: u32, ts_ms: i64, tokens: u32) -> Chunk {
|
||||
let ts = Utc.timestamp_millis_opt(ts_ms).unwrap();
|
||||
Chunk {
|
||||
id: chunk_id(SourceKind::Chat, source_id, seq),
|
||||
id: chunk_id(SourceKind::Chat, source_id, seq, "test-content"),
|
||||
content: format!("substantive chunk mentioning alice {source_id}#{seq}"),
|
||||
metadata: Metadata {
|
||||
source_kind: SourceKind::Chat,
|
||||
|
||||
@@ -189,7 +189,7 @@ mod tests {
|
||||
let ts_ms = 1_700_000_000_000 + (seq as i64) * 1_000;
|
||||
let ts = Utc.timestamp_millis_opt(ts_ms).unwrap();
|
||||
let c = Chunk {
|
||||
id: chunk_id(SourceKind::Chat, source_tree, seq),
|
||||
id: chunk_id(SourceKind::Chat, source_tree, seq, "test-content"),
|
||||
content: format!("mentioning entity in {source_tree}#{seq}"),
|
||||
metadata: Metadata {
|
||||
source_kind: SourceKind::Chat,
|
||||
|
||||
@@ -148,7 +148,7 @@ mod tests {
|
||||
fn persist_chunk(cfg: &Config, source_id: &str, seq: u32, ts_ms: i64, tokens: u32) -> String {
|
||||
let ts = Utc.timestamp_millis_opt(ts_ms).unwrap();
|
||||
let c = Chunk {
|
||||
id: chunk_id(SourceKind::Chat, source_id, seq),
|
||||
id: chunk_id(SourceKind::Chat, source_id, seq, "test-content"),
|
||||
content: format!("chunk content {source_id} {seq}"),
|
||||
metadata: Metadata {
|
||||
source_kind: SourceKind::Chat,
|
||||
|
||||
@@ -236,16 +236,30 @@ pub struct Chunk {
|
||||
|
||||
/// Deterministic chunk id.
|
||||
///
|
||||
/// `sha256(source_kind | "\0" | source_id | "\0" | seq)` hex-encoded, first
|
||||
/// 32 chars (128 bits of collision resistance). Short enough for human
|
||||
/// inspection, long enough for global uniqueness in a single-user workspace.
|
||||
pub fn chunk_id(source_kind: SourceKind, source_id: &str, seq_in_source: u32) -> String {
|
||||
/// `sha256(source_kind | "\0" | source_id | "\0" | seq | "\0" | content)`
|
||||
/// hex-encoded, first 32 chars (128 bits of collision resistance). Short
|
||||
/// enough for human inspection, long enough for global uniqueness in a
|
||||
/// single-user workspace.
|
||||
///
|
||||
/// Content is included so multiple ingest calls that share a `source_id`
|
||||
/// (e.g. successive Slack 6-hour buckets all flowing into one
|
||||
/// per-connection source tree) don't collide on `seq=0,1,2,…`. Re-ingesting
|
||||
/// the same canonical content under the same `(source_id, seq)` still
|
||||
/// produces the same id, so upserts stay idempotent.
|
||||
pub fn chunk_id(
|
||||
source_kind: SourceKind,
|
||||
source_id: &str,
|
||||
seq_in_source: u32,
|
||||
content: &str,
|
||||
) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(source_kind.as_str().as_bytes());
|
||||
hasher.update([0u8]);
|
||||
hasher.update(source_id.as_bytes());
|
||||
hasher.update([0u8]);
|
||||
hasher.update(seq_in_source.to_be_bytes());
|
||||
hasher.update([0u8]);
|
||||
hasher.update(content.as_bytes());
|
||||
let digest = hasher.finalize();
|
||||
let hex = digest.iter().fold(String::with_capacity(64), |mut acc, b| {
|
||||
use std::fmt::Write;
|
||||
@@ -308,30 +322,40 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn chunk_id_is_deterministic() {
|
||||
let a = chunk_id(SourceKind::Chat, "slack:#eng", 0);
|
||||
let b = chunk_id(SourceKind::Chat, "slack:#eng", 0);
|
||||
let a = chunk_id(SourceKind::Chat, "slack:#eng", 0, "hello");
|
||||
let b = chunk_id(SourceKind::Chat, "slack:#eng", 0, "hello");
|
||||
assert_eq!(a, b);
|
||||
assert_eq!(a.len(), 32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunk_id_varies_with_seq() {
|
||||
let a = chunk_id(SourceKind::Chat, "slack:#eng", 0);
|
||||
let b = chunk_id(SourceKind::Chat, "slack:#eng", 1);
|
||||
let a = chunk_id(SourceKind::Chat, "slack:#eng", 0, "hello");
|
||||
let b = chunk_id(SourceKind::Chat, "slack:#eng", 1, "hello");
|
||||
assert_ne!(a, b);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunk_id_varies_with_source_kind() {
|
||||
let a = chunk_id(SourceKind::Chat, "foo", 0);
|
||||
let b = chunk_id(SourceKind::Email, "foo", 0);
|
||||
let a = chunk_id(SourceKind::Chat, "foo", 0, "hello");
|
||||
let b = chunk_id(SourceKind::Email, "foo", 0, "hello");
|
||||
assert_ne!(a, b);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunk_id_varies_with_source_id() {
|
||||
let a = chunk_id(SourceKind::Chat, "x", 0);
|
||||
let b = chunk_id(SourceKind::Chat, "y", 0);
|
||||
let a = chunk_id(SourceKind::Chat, "x", 0, "hello");
|
||||
let b = chunk_id(SourceKind::Chat, "y", 0, "hello");
|
||||
assert_ne!(a, b);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunk_id_varies_with_content() {
|
||||
// Critical for the per-connection source_id design: two ingests
|
||||
// sharing source_id but different content (e.g. different 6-hour
|
||||
// Slack buckets) must produce distinct ids at seq=0,1,2,…
|
||||
let a = chunk_id(SourceKind::Chat, "slack:c1", 0, "bucket A content");
|
||||
let b = chunk_id(SourceKind::Chat, "slack:c1", 0, "bucket B content");
|
||||
assert_ne!(a, b);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user