Merge pull request #112 from sanil-23/issue-69-memory-rpc

Add core memory RPC surface (#69)
This commit is contained in:
Al629176
2026-03-31 17:52:14 +05:30
committed by GitHub
19 changed files with 3983 additions and 282 deletions
Generated
+561 -31
View File
File diff suppressed because it is too large Load Diff
+4
View File
@@ -36,6 +36,8 @@ anyhow = "1.0"
async-trait = "0.1"
chacha20poly1305 = "0.10"
hex = "0.4"
fastembed = { version = "5.13", default-features = false, features = ["hf-hub-native-tls", "ort-load-dynamic"] }
ort = { version = "=2.0.0-rc.11", default-features = false, features = ["std", "ndarray", "load-dynamic"] }
tokio-util = { version = "0.7", features = ["rt"] }
tokio-tungstenite = { version = "0.24", features = ["rustls-tls-webpki-roots"] }
futures = "0.3"
@@ -61,6 +63,8 @@ dialoguer = { version = "0.12", features = ["fuzzy-select"] }
console = "0.16"
glob = "0.3"
regex = "1.10"
ndarray = "0.17.2"
tokenizers = "0.21.0"
hostname = "0.4.2"
rustyline = { version = "15", features = ["with-file-history"] }
rustls = { version = "0.23", features = ["ring"] }
+44
View File
@@ -476,6 +476,50 @@ mod tests {
assert!(err.contains("unknown param 'x'"));
}
#[tokio::test]
async fn invoke_memory_init_missing_required_param_fails() {
let err = invoke_method(default_state(), "memory.init", json!({}))
.await
.expect_err("missing jwt_token should fail");
assert!(err.contains("missing field `jwt_token`") || err.contains("jwt_token"));
}
#[tokio::test]
async fn invoke_memory_list_namespaces_rejects_unknown_param() {
let err = invoke_method(
default_state(),
"memory.list_namespaces",
json!({ "extra": true }),
)
.await
.expect_err("unknown param should fail");
assert!(err.contains("unknown field `extra`") || err.contains("extra"));
}
#[tokio::test]
async fn invoke_memory_query_namespace_missing_namespace_fails() {
let err = invoke_method(
default_state(),
"memory.query_namespace",
json!({ "query": "who owns atlas" }),
)
.await
.expect_err("missing namespace should fail");
assert!(err.contains("missing field `namespace`") || err.contains("namespace"));
}
#[tokio::test]
async fn invoke_memory_recall_memories_rejects_unknown_param() {
let err = invoke_method(
default_state(),
"memory.recall_memories",
json!({ "namespace": "team", "extra": true }),
)
.await
.expect_err("unknown param should fail");
assert!(err.contains("unknown field `extra`") || err.contains("extra"));
}
#[tokio::test]
async fn invoke_migrate_openclaw_rejects_unknown_param() {
let err = invoke_method(
@@ -109,7 +109,7 @@ fn default_true() -> bool {
}
fn default_embedding_provider() -> String {
"none".into()
"fastembed".into()
}
fn default_hygiene_enabled() -> bool {
true
@@ -124,10 +124,10 @@ fn default_conversation_retention_days() -> u32 {
30
}
fn default_embedding_model() -> String {
"text-embedding-3-small".into()
"BGESmallENV15".into()
}
fn default_embedding_dims() -> usize {
1536
384
}
fn default_vector_weight() -> f64 {
0.7
+208
View File
@@ -1,4 +1,13 @@
use async_trait::async_trait;
use parking_lot::Mutex;
use std::env;
use std::path::PathBuf;
use std::str::FromStr;
use std::sync::Arc;
pub const DEFAULT_FASTEMBED_MODEL: &str = "BGESmallENV15";
pub const DEFAULT_FASTEMBED_DIMENSIONS: usize = 384;
const DEFAULT_MANAGED_RELEX_DIR: &str = ".openhuman/models/gliner-relex-large-v0.5-onnx";
/// Trait for embedding providers — convert text to vectors
#[async_trait]
@@ -40,6 +49,183 @@ impl EmbeddingProvider for NoopEmbedding {
}
}
enum FastembedState {
Uninitialized,
Ready(fastembed::TextEmbedding),
Failed(String),
}
pub struct FastembedEmbedding {
model: String,
dims: usize,
state: Arc<Mutex<FastembedState>>,
}
impl FastembedEmbedding {
pub fn new(model: &str, dims: usize) -> Self {
Self {
model: if model.trim().is_empty() {
DEFAULT_FASTEMBED_MODEL.to_string()
} else {
model.trim().to_string()
},
dims: if dims == 0 {
DEFAULT_FASTEMBED_DIMENSIONS
} else {
dims
},
state: Arc::new(Mutex::new(FastembedState::Uninitialized)),
}
}
fn resolve_model(&self) -> fastembed::EmbeddingModel {
fastembed::EmbeddingModel::from_str(&self.model)
.unwrap_or(fastembed::EmbeddingModel::BGESmallENV15)
}
fn init_model(&self) -> anyhow::Result<fastembed::TextEmbedding> {
ensure_fastembed_ort_dylib_path();
fastembed::TextEmbedding::try_new(
fastembed::InitOptions::new(self.resolve_model()).with_show_download_progress(false),
)
.map_err(|e| anyhow::anyhow!("fastembed init failed for {}: {e}", self.model))
}
}
fn ensure_fastembed_ort_dylib_path() {
if env::var_os("ORT_DYLIB_PATH").is_some() {
return;
}
if let Some(lib_path) = env::var_os("ORT_LIB_LOCATION") {
let candidate = PathBuf::from(lib_path);
if candidate.is_file() {
env::set_var("ORT_DYLIB_PATH", candidate);
return;
}
#[cfg(target_os = "windows")]
let runtime_lib = candidate.join("onnxruntime.dll");
#[cfg(target_os = "macos")]
let runtime_lib = candidate.join("libonnxruntime.dylib");
#[cfg(target_os = "linux")]
let runtime_lib = candidate.join("libonnxruntime.so");
if runtime_lib.exists() {
env::set_var("ORT_DYLIB_PATH", runtime_lib);
return;
}
}
if let Ok(path) = env::var("OPENHUMAN_GLINER_RELEX_CACHE_DIR") {
#[cfg(target_os = "windows")]
let bundled = PathBuf::from(path).join("onnxruntime.dll");
#[cfg(target_os = "macos")]
let bundled = PathBuf::from(path).join("libonnxruntime.dylib");
#[cfg(target_os = "linux")]
let bundled = PathBuf::from(path).join("libonnxruntime.so");
if bundled.exists() {
env::set_var("ORT_DYLIB_PATH", bundled);
return;
}
}
if let Some(user_dirs) = directories::UserDirs::new() {
#[cfg(target_os = "windows")]
let bundled = user_dirs
.home_dir()
.join(DEFAULT_MANAGED_RELEX_DIR)
.join("onnxruntime.dll");
#[cfg(target_os = "macos")]
let bundled = user_dirs
.home_dir()
.join(DEFAULT_MANAGED_RELEX_DIR)
.join("libonnxruntime.dylib");
#[cfg(target_os = "linux")]
let bundled = user_dirs
.home_dir()
.join(DEFAULT_MANAGED_RELEX_DIR)
.join("libonnxruntime.so");
if bundled.exists() {
env::set_var("ORT_DYLIB_PATH", bundled);
return;
}
}
#[cfg(target_os = "linux")]
{
for candidate in [
"/usr/lib/x86_64-linux-gnu/libonnxruntime.so",
"/usr/local/lib/libonnxruntime.so",
"/usr/lib/libonnxruntime.so",
] {
let candidate = PathBuf::from(candidate);
if candidate.exists() {
env::set_var("ORT_DYLIB_PATH", candidate);
return;
}
}
}
}
#[async_trait]
impl EmbeddingProvider for FastembedEmbedding {
fn name(&self) -> &str {
"fastembed"
}
fn dimensions(&self) -> usize {
self.dims
}
async fn embed(&self, texts: &[&str]) -> anyhow::Result<Vec<Vec<f32>>> {
if texts.is_empty() {
return Ok(Vec::new());
}
let items = texts
.iter()
.map(|text| (*text).to_string())
.collect::<Vec<_>>();
let state = Arc::clone(&self.state);
let provider = self.model.clone();
let join_result = tokio::task::spawn_blocking(move || {
ensure_fastembed_ort_dylib_path();
let mut guard = state.lock();
if matches!(*guard, FastembedState::Uninitialized) {
match fastembed::TextEmbedding::try_new(
fastembed::InitOptions::new(
fastembed::EmbeddingModel::from_str(&provider)
.unwrap_or(fastembed::EmbeddingModel::BGESmallENV15),
)
.with_show_download_progress(false),
) {
Ok(model) => *guard = FastembedState::Ready(model),
Err(err) => {
let message = format!("fastembed init failed for {provider}: {err}");
*guard = FastembedState::Failed(message.clone());
}
}
}
match &mut *guard {
FastembedState::Ready(model) => model
.embed(items, None)
.map_err(|e| anyhow::anyhow!("fastembed embed failed: {e}")),
FastembedState::Failed(message) => Err(anyhow::anyhow!(message.clone())),
FastembedState::Uninitialized => {
Err(anyhow::anyhow!("fastembed provider did not initialize"))
}
}
})
.await;
join_result.map_err(|e| anyhow::anyhow!("fastembed task join failed: {e}"))?
}
}
// ── OpenAI-compatible embedding provider ─────────────────────
pub struct OpenAiEmbedding {
@@ -163,6 +349,7 @@ pub fn create_embedding_provider(
dims: usize,
) -> Box<dyn EmbeddingProvider> {
match provider {
"fastembed" => Box::new(FastembedEmbedding::new(model, dims)),
"openai" => {
let key = api_key.unwrap_or("");
Box::new(OpenAiEmbedding::new(
@@ -181,6 +368,13 @@ pub fn create_embedding_provider(
}
}
pub fn default_local_embedding_provider() -> Arc<dyn EmbeddingProvider> {
Arc::new(FastembedEmbedding::new(
DEFAULT_FASTEMBED_MODEL,
DEFAULT_FASTEMBED_DIMENSIONS,
))
}
#[cfg(test)]
mod tests {
use super::*;
@@ -212,6 +406,13 @@ mod tests {
assert_eq!(p.dimensions(), 1536);
}
#[test]
fn factory_fastembed() {
let p = create_embedding_provider("fastembed", None, DEFAULT_FASTEMBED_MODEL, 384);
assert_eq!(p.name(), "fastembed");
assert_eq!(p.dimensions(), 384);
}
#[test]
fn factory_custom_url() {
let p = create_embedding_provider("custom:http://localhost:1234", None, "model", 768);
@@ -255,6 +456,13 @@ mod tests {
assert_eq!(p.name(), "none");
}
#[test]
fn default_local_provider_uses_fastembed_defaults() {
let p = default_local_embedding_provider();
assert_eq!(p.name(), "fastembed");
assert_eq!(p.dimensions(), DEFAULT_FASTEMBED_DIMENSIONS);
}
#[test]
fn factory_custom_empty_url() {
// "custom:" with no URL — should still construct without panic
+5 -1
View File
@@ -1,13 +1,17 @@
pub mod chunker;
pub mod embeddings;
pub mod ops;
pub mod rpc_models;
pub mod store;
pub mod traits;
pub use ops as rpc;
pub use ops::*;
pub use rpc_models::*;
pub use store::{
create_memory, create_memory_for_migration, create_memory_with_storage,
create_memory_with_storage_and_routes, effective_memory_backend_name, MemoryClient,
MemoryClientRef, MemoryState, NamespaceDocumentInput, NamespaceQueryResult, UnifiedMemory,
MemoryClientRef, MemoryItemKind, MemoryState, NamespaceDocumentInput, NamespaceMemoryHit,
NamespaceQueryResult, NamespaceRetrievalContext, RetrievalScoreBreakdown, UnifiedMemory,
};
pub use traits::{Memory, MemoryCategory, MemoryEntry};
+764 -19
View File
@@ -1,6 +1,431 @@
use crate::openhuman::memory::{MemoryClient, NamespaceDocumentInput};
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Component, Path, PathBuf};
use std::sync::{Arc, Mutex, OnceLock};
use crate::openhuman::config::Config;
use crate::openhuman::memory::store::GraphRelationRecord;
use crate::openhuman::memory::{
ApiEnvelope, ApiError, ApiMeta, DeleteDocumentRequest, DeleteDocumentResponse, EmptyRequest,
ListDocumentsRequest, ListDocumentsResponse, ListMemoryFilesRequest, ListMemoryFilesResponse,
ListNamespacesResponse, MemoryClient, MemoryClientRef, MemoryDocumentSummary,
MemoryInitRequest, MemoryInitResponse, MemoryItemKind, MemoryRecallItem, MemoryRetrievalChunk,
MemoryRetrievalContext, MemoryRetrievalEntity, MemoryRetrievalRelation, NamespaceDocumentInput,
NamespaceMemoryHit, NamespaceRetrievalContext, PaginationMeta, QueryNamespaceRequest,
QueryNamespaceResponse, ReadMemoryFileRequest, ReadMemoryFileResponse, RecallContextRequest,
RecallContextResponse, RecallMemoriesRequest, RecallMemoriesResponse, WriteMemoryFileRequest,
WriteMemoryFileResponse,
};
use crate::rpc::RpcOutcome;
use chrono::TimeZone;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
static MEMORY_CLIENT_STATE: OnceLock<Mutex<Option<MemoryClientRef>>> = OnceLock::new();
fn memory_client_state() -> &'static Mutex<Option<MemoryClientRef>> {
MEMORY_CLIENT_STATE.get_or_init(|| Mutex::new(None))
}
fn lock_memory_client_state(
) -> Result<std::sync::MutexGuard<'static, Option<MemoryClientRef>>, String> {
memory_client_state()
.lock()
.map_err(|_| "memory client state lock poisoned".to_string())
}
fn memory_request_id() -> String {
uuid::Uuid::new_v4().to_string()
}
fn memory_counts(
entries: impl IntoIterator<Item = (&'static str, usize)>,
) -> BTreeMap<String, usize> {
entries
.into_iter()
.map(|(key, value)| (key.to_string(), value))
.collect()
}
fn envelope<T: Serialize>(
data: T,
counts: Option<BTreeMap<String, usize>>,
pagination: Option<PaginationMeta>,
) -> RpcOutcome<ApiEnvelope<T>> {
RpcOutcome::new(
ApiEnvelope {
data: Some(data),
error: None,
meta: ApiMeta {
request_id: memory_request_id(),
latency_seconds: None,
cached: None,
counts,
pagination,
},
},
vec![],
)
}
fn error_envelope<T: Serialize>(code: &str, message: String) -> RpcOutcome<ApiEnvelope<T>> {
RpcOutcome::new(
ApiEnvelope {
data: None,
error: Some(ApiError {
code: code.to_string(),
message,
details: None,
}),
meta: ApiMeta {
request_id: memory_request_id(),
latency_seconds: None,
cached: None,
counts: None,
pagination: None,
},
},
vec![],
)
}
fn timestamp_to_rfc3339(timestamp: f64) -> Option<String> {
if !timestamp.is_finite() || timestamp < 0.0 {
return None;
}
let secs = timestamp.trunc() as i64;
let nanos = ((timestamp.fract().abs()) * 1_000_000_000.0).round() as u32;
chrono::Utc
.timestamp_opt(secs, nanos.min(999_999_999))
.single()
.map(|value| value.to_rfc3339())
}
fn memory_kind_label(kind: &MemoryItemKind) -> &'static str {
match kind {
MemoryItemKind::Document => "document",
MemoryItemKind::Kv => "kv",
}
}
fn relation_identity(relation: &GraphRelationRecord) -> String {
format!(
"{}|{}|{}|{}",
relation.namespace.as_deref().unwrap_or("global"),
relation.subject.as_str(),
relation.predicate.as_str(),
relation.object.as_str()
)
}
fn relation_metadata(relation: &GraphRelationRecord) -> Value {
json!({
"namespace": relation.namespace.clone(),
"attrs": relation.attrs.clone(),
"order_index": relation.order_index,
"document_ids": relation.document_ids.clone(),
"chunk_ids": relation.chunk_ids.clone(),
"updated_at": timestamp_to_rfc3339(relation.updated_at),
})
}
fn chunk_metadata(hit: &NamespaceMemoryHit) -> Value {
json!({
"kind": memory_kind_label(&hit.kind),
"namespace": hit.namespace.clone(),
"key": hit.key.clone(),
"title": hit.title.clone(),
"category": hit.category.clone(),
"source_type": hit.source_type.clone(),
"score_breakdown": {
"keyword_relevance": hit.score_breakdown.keyword_relevance,
"vector_similarity": hit.score_breakdown.vector_similarity,
"graph_relevance": hit.score_breakdown.graph_relevance,
"freshness": hit.score_breakdown.freshness,
"final_score": hit.score_breakdown.final_score,
}
})
}
fn build_retrieval_context(hits: &[NamespaceMemoryHit]) -> MemoryRetrievalContext {
let mut entities = BTreeSet::new();
let mut relations = BTreeMap::new();
let chunks = hits
.iter()
.map(|hit| {
for relation in &hit.supporting_relations {
if !relation.subject.trim().is_empty() {
entities.insert(relation.subject.clone());
}
if !relation.object.trim().is_empty() {
entities.insert(relation.object.clone());
}
relations
.entry(relation_identity(relation))
.or_insert_with(|| MemoryRetrievalRelation {
subject: relation.subject.clone(),
predicate: relation.predicate.clone(),
object: relation.object.clone(),
score: None,
evidence_count: Some(relation.evidence_count),
metadata: relation_metadata(relation),
});
}
MemoryRetrievalChunk {
chunk_id: hit.chunk_id.clone(),
document_id: hit.document_id.clone(),
content: hit.content.clone(),
score: hit.score,
metadata: chunk_metadata(hit),
created_at: None,
updated_at: timestamp_to_rfc3339(hit.updated_at),
}
})
.collect();
MemoryRetrievalContext {
entities: entities
.into_iter()
.map(|name| MemoryRetrievalEntity {
id: None,
name,
entity_type: None,
score: None,
metadata: json!({}),
})
.collect(),
relations: relations.into_values().collect(),
chunks,
}
}
fn format_llm_context_message(query: Option<&str>, hits: &[NamespaceMemoryHit]) -> Option<String> {
if hits.is_empty() {
return None;
}
let mut parts = Vec::new();
if let Some(query) = query {
parts.push(format!("Query: {query}"));
}
for hit in hits {
let summary = match hit.kind {
MemoryItemKind::Document => {
let title = hit.title.clone().unwrap_or_else(|| hit.key.clone());
format!("{title}: {}", hit.content.trim())
}
MemoryItemKind::Kv => format!("[kv:{}] {}", hit.key, hit.content.trim()),
};
parts.push(summary);
if !hit.supporting_relations.is_empty() {
let relations = hit
.supporting_relations
.iter()
.map(|relation| {
format!(
"{} -[{}]-> {}",
relation.subject.as_str(),
relation.predicate.as_str(),
relation.object.as_str()
)
})
.collect::<Vec<_>>()
.join("; ");
parts.push(format!("Relations: {relations}"));
}
}
Some(parts.join("\n\n"))
}
fn filter_hits_by_document_ids(
hits: Vec<NamespaceMemoryHit>,
document_ids: Option<&[String]>,
) -> Vec<NamespaceMemoryHit> {
let Some(document_ids) = document_ids else {
return hits;
};
let allowed = document_ids.iter().cloned().collect::<BTreeSet<_>>();
hits.into_iter()
.filter(|hit| {
hit.document_id
.as_ref()
.map(|document_id| allowed.contains(document_id))
.unwrap_or(false)
})
.collect()
}
fn maybe_retrieval_context(
include_references: bool,
context: MemoryRetrievalContext,
) -> Option<MemoryRetrievalContext> {
if !include_references {
return None;
}
if context.entities.is_empty() && context.relations.is_empty() && context.chunks.is_empty() {
return None;
}
Some(context)
}
async fn current_workspace_dir() -> Result<PathBuf, String> {
Config::load_or_init()
.await
.map(|config| config.workspace_dir)
.map_err(|e| format!("load config: {e}"))
}
async fn active_memory_client() -> Result<MemoryClientRef, String> {
if let Some(client) = lock_memory_client_state()?.clone() {
return Ok(client);
}
let workspace_dir = current_workspace_dir().await?;
Ok(Arc::new(MemoryClient::from_workspace_dir(workspace_dir)?))
}
fn validate_memory_relative_path(path: &str) -> Result<(), String> {
let candidate = Path::new(path);
if candidate.as_os_str().is_empty() {
return Err("memory path must not be empty".to_string());
}
if candidate.is_absolute() {
return Err("absolute paths are not allowed".to_string());
}
for component in candidate.components() {
match component {
Component::ParentDir | Component::RootDir | Component::Prefix(_) => {
return Err("path traversal is not allowed".to_string());
}
_ => {}
}
}
Ok(())
}
async fn resolve_memory_root() -> Result<PathBuf, String> {
let workspace_dir = current_workspace_dir().await?;
let memory_root = workspace_dir.join("memory");
std::fs::create_dir_all(&memory_root)
.map_err(|e| format!("create memory dir {}: {e}", memory_root.display()))?;
memory_root
.canonicalize()
.map_err(|e| format!("resolve memory dir {}: {e}", memory_root.display()))
}
async fn resolve_existing_memory_path(relative_path: &str) -> Result<PathBuf, String> {
validate_memory_relative_path(relative_path)?;
let memory_root = resolve_memory_root().await?;
let workspace_root = memory_root
.parent()
.ok_or_else(|| "memory root is missing a parent workspace".to_string())?;
let full_path = workspace_root.join(relative_path);
let resolved = full_path
.canonicalize()
.map_err(|e| format!("resolve memory path {}: {e}", full_path.display()))?;
if !resolved.starts_with(&memory_root) {
return Err("memory path escapes the memory directory".to_string());
}
Ok(resolved)
}
async fn resolve_writable_memory_path(relative_path: &str) -> Result<PathBuf, String> {
validate_memory_relative_path(relative_path)?;
let memory_root = resolve_memory_root().await?;
let workspace_root = memory_root
.parent()
.ok_or_else(|| "memory root is missing a parent workspace".to_string())?;
let full_path = workspace_root.join(relative_path);
let parent = full_path
.parent()
.ok_or_else(|| "memory path must include a file name".to_string())?;
std::fs::create_dir_all(parent)
.map_err(|e| format!("create memory path {}: {e}", parent.display()))?;
let resolved_parent = parent
.canonicalize()
.map_err(|e| format!("resolve memory parent {}: {e}", parent.display()))?;
if !resolved_parent.starts_with(&memory_root) {
return Err("memory path escapes the memory directory".to_string());
}
let file_name = full_path
.file_name()
.ok_or_else(|| "memory path must include a file name".to_string())?;
let resolved = resolved_parent.join(file_name);
if let Ok(metadata) = std::fs::symlink_metadata(&resolved) {
if metadata.file_type().is_symlink() {
return Err(format!(
"refusing to write through symlink: {}",
resolved.display()
));
}
}
Ok(resolved)
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
struct RawMemoryDocumentSummary {
document_id: String,
namespace: String,
key: String,
title: String,
source_type: String,
priority: String,
created_at: f64,
updated_at: f64,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
struct RawDeleteDocumentResult {
deleted: bool,
namespace: String,
document_id: String,
}
fn parse_memory_document_summaries(raw: Value) -> Result<Vec<MemoryDocumentSummary>, String> {
let documents = raw
.get("documents")
.and_then(Value::as_array)
.ok_or_else(|| "memory document list missing 'documents' array".to_string())?;
documents
.iter()
.cloned()
.map(|value| {
let raw: RawMemoryDocumentSummary = serde_json::from_value(value)
.map_err(|e| format!("decode memory document: {e}"))?;
Ok(MemoryDocumentSummary {
document_id: raw.document_id,
namespace: raw.namespace,
key: raw.key,
title: raw.title,
source_type: raw.source_type,
priority: raw.priority,
created_at: raw.created_at,
updated_at: raw.updated_at,
})
})
.collect()
}
async fn query_limit_for_request(
client: &MemoryClient,
request: &QueryNamespaceRequest,
) -> Result<u32, String> {
let requested = request.resolved_limit();
if request.document_ids.is_none() {
return Ok(requested);
}
let raw = client.list_documents(Some(&request.namespace)).await?;
let documents = parse_memory_document_summaries(raw)?;
let total_documents = u32::try_from(documents.len()).unwrap_or(u32::MAX);
Ok(requested.max(total_documents))
}
#[derive(Debug, Deserialize)]
pub struct PutDocParams {
@@ -29,12 +454,6 @@ pub struct NamespaceOnlyParams {
pub namespace: String,
}
#[derive(Debug, Deserialize)]
pub struct OptionalNamespaceParams {
#[serde(default)]
pub namespace: Option<String>,
}
#[derive(Debug, Deserialize)]
pub struct DeleteDocParams {
pub namespace: String,
@@ -110,7 +529,7 @@ fn default_category() -> String {
}
pub async fn namespace_list() -> Result<RpcOutcome<Vec<String>>, String> {
let client = MemoryClient::new_local()?;
let client = active_memory_client().await?;
let namespaces = client.list_namespaces().await?;
Ok(RpcOutcome::single_log(
namespaces,
@@ -119,7 +538,7 @@ pub async fn namespace_list() -> Result<RpcOutcome<Vec<String>>, String> {
}
pub async fn doc_put(params: PutDocParams) -> Result<RpcOutcome<PutDocResult>, String> {
let client = MemoryClient::new_local()?;
let client = active_memory_client().await?;
let document_id = client
.put_doc(NamespaceDocumentInput {
namespace: params.namespace,
@@ -144,7 +563,7 @@ pub async fn doc_put(params: PutDocParams) -> Result<RpcOutcome<PutDocResult>, S
pub async fn doc_list(
params: Option<NamespaceOnlyParams>,
) -> Result<RpcOutcome<serde_json::Value>, String> {
let client = MemoryClient::new_local()?;
let client = active_memory_client().await?;
let docs = client
.list_documents(params.as_ref().map(|v| v.namespace.as_str()))
.await?;
@@ -152,7 +571,7 @@ pub async fn doc_list(
}
pub async fn doc_delete(params: DeleteDocParams) -> Result<RpcOutcome<serde_json::Value>, String> {
let client = MemoryClient::new_local()?;
let client = active_memory_client().await?;
let result = client
.delete_document(&params.namespace, &params.document_id)
.await?;
@@ -160,7 +579,7 @@ pub async fn doc_delete(params: DeleteDocParams) -> Result<RpcOutcome<serde_json
}
pub async fn context_query(params: QueryNamespaceParams) -> Result<RpcOutcome<String>, String> {
let client = MemoryClient::new_local()?;
let client = active_memory_client().await?;
let result = client
.query_namespace(&params.namespace, &params.query, params.limit.unwrap_or(10))
.await?;
@@ -170,7 +589,7 @@ pub async fn context_query(params: QueryNamespaceParams) -> Result<RpcOutcome<St
pub async fn context_recall(
params: RecallNamespaceParams,
) -> Result<RpcOutcome<Option<String>>, String> {
let client = MemoryClient::new_local()?;
let client = active_memory_client().await?;
let result = client
.recall_namespace(&params.namespace, params.limit.unwrap_or(10))
.await?;
@@ -178,7 +597,7 @@ pub async fn context_recall(
}
pub async fn kv_set(params: KvSetParams) -> Result<RpcOutcome<bool>, String> {
let client = MemoryClient::new_local()?;
let client = active_memory_client().await?;
client
.kv_set(params.namespace.as_deref(), &params.key, &params.value)
.await?;
@@ -188,7 +607,7 @@ pub async fn kv_set(params: KvSetParams) -> Result<RpcOutcome<bool>, String> {
pub async fn kv_get(
params: KvGetDeleteParams,
) -> Result<RpcOutcome<Option<serde_json::Value>>, String> {
let client = MemoryClient::new_local()?;
let client = active_memory_client().await?;
let value = client
.kv_get(params.namespace.as_deref(), &params.key)
.await?;
@@ -196,7 +615,7 @@ pub async fn kv_get(
}
pub async fn kv_delete(params: KvGetDeleteParams) -> Result<RpcOutcome<bool>, String> {
let client = MemoryClient::new_local()?;
let client = active_memory_client().await?;
let deleted = client
.kv_delete(params.namespace.as_deref(), &params.key)
.await?;
@@ -206,13 +625,13 @@ pub async fn kv_delete(params: KvGetDeleteParams) -> Result<RpcOutcome<bool>, St
pub async fn kv_list_namespace(
params: NamespaceOnlyParams,
) -> Result<RpcOutcome<Vec<serde_json::Value>>, String> {
let client = MemoryClient::new_local()?;
let client = active_memory_client().await?;
let rows = client.kv_list_namespace(&params.namespace).await?;
Ok(RpcOutcome::single_log(rows, "memory namespace kv listed"))
}
pub async fn graph_upsert(params: GraphUpsertParams) -> Result<RpcOutcome<bool>, String> {
let client = MemoryClient::new_local()?;
let client = active_memory_client().await?;
client
.graph_upsert(
params.namespace.as_deref(),
@@ -228,7 +647,7 @@ pub async fn graph_upsert(params: GraphUpsertParams) -> Result<RpcOutcome<bool>,
pub async fn graph_query(
params: GraphQueryParams,
) -> Result<RpcOutcome<Vec<serde_json::Value>>, String> {
let client = MemoryClient::new_local()?;
let client = active_memory_client().await?;
let rows = client
.graph_query(
params.namespace.as_deref(),
@@ -238,3 +657,329 @@ pub async fn graph_query(
.await?;
Ok(RpcOutcome::single_log(rows, "memory graph queried"))
}
pub async fn memory_init(
request: MemoryInitRequest,
) -> Result<RpcOutcome<ApiEnvelope<MemoryInitResponse>>, String> {
if request.jwt_token.trim().is_empty() {
return Err("jwt_token must not be empty".to_string());
}
let workspace_dir = current_workspace_dir().await?;
let client = Arc::new(MemoryClient::from_workspace_dir(workspace_dir.clone())?);
*lock_memory_client_state()? = Some(client);
let memory_dir = workspace_dir.join("memory");
Ok(envelope(
MemoryInitResponse {
initialized: true,
workspace_dir: workspace_dir.display().to_string(),
memory_dir: memory_dir.display().to_string(),
},
None,
None,
))
}
pub async fn memory_list_documents(
request: ListDocumentsRequest,
) -> Result<RpcOutcome<ApiEnvelope<ListDocumentsResponse>>, String> {
let client = active_memory_client().await?;
let raw = client.list_documents(request.namespace.as_deref()).await?;
let documents = parse_memory_document_summaries(raw)?;
let count = documents.len();
Ok(envelope(
ListDocumentsResponse {
namespace: request.namespace,
documents,
count,
},
Some(memory_counts([("num_documents", count)])),
Some(PaginationMeta {
limit: count,
offset: 0,
count,
}),
))
}
pub async fn memory_list_namespaces(
_request: EmptyRequest,
) -> Result<RpcOutcome<ApiEnvelope<ListNamespacesResponse>>, String> {
let client = active_memory_client().await?;
let namespaces = client.list_namespaces().await?;
let count = namespaces.len();
Ok(envelope(
ListNamespacesResponse { namespaces, count },
Some(memory_counts([("num_namespaces", count)])),
None,
))
}
pub async fn memory_delete_document(
request: DeleteDocumentRequest,
) -> Result<RpcOutcome<ApiEnvelope<DeleteDocumentResponse>>, String> {
let client = active_memory_client().await?;
let raw = client
.delete_document(&request.namespace, &request.document_id)
.await?;
let parsed: RawDeleteDocumentResult =
serde_json::from_value(raw).map_err(|e| format!("decode delete document result: {e}"))?;
Ok(envelope(
DeleteDocumentResponse {
status: if parsed.deleted {
"completed".to_string()
} else {
"not_found".to_string()
},
namespace: parsed.namespace,
document_id: parsed.document_id,
deleted: parsed.deleted,
},
None,
None,
))
}
pub async fn memory_query_namespace(
request: QueryNamespaceRequest,
) -> Result<RpcOutcome<ApiEnvelope<QueryNamespaceResponse>>, String> {
let include_references = request.include_references.unwrap_or(true);
let result = async {
let client = active_memory_client().await?;
let retrieval_limit = query_limit_for_request(client.as_ref(), &request).await?;
let mut context = client
.query_namespace_context_data(&request.namespace, &request.query, retrieval_limit)
.await?;
context.hits = filter_hits_by_document_ids(context.hits, request.document_ids.as_deref());
Ok::<NamespaceRetrievalContext, String>(context)
}
.await;
match result {
Ok(context) => {
let retrieval_context = build_retrieval_context(&context.hits);
let counts = memory_counts([
("num_entities", retrieval_context.entities.len()),
("num_relations", retrieval_context.relations.len()),
("num_chunks", retrieval_context.chunks.len()),
]);
let llm_context_message =
format_llm_context_message(Some(&request.query), &context.hits);
Ok(envelope(
QueryNamespaceResponse {
context: maybe_retrieval_context(include_references, retrieval_context),
llm_context_message,
},
Some(counts),
None,
))
}
Err(message) => Ok(error_envelope("memory.query_namespace_failed", message)),
}
}
pub async fn memory_recall_context(
request: RecallContextRequest,
) -> Result<RpcOutcome<ApiEnvelope<RecallContextResponse>>, String> {
let include_references = request.include_references.unwrap_or(true);
let result = async {
let client = active_memory_client().await?;
client
.recall_namespace_context_data(&request.namespace, request.resolved_limit())
.await
}
.await;
match result {
Ok(context) => {
let retrieval_context = build_retrieval_context(&context.hits);
let counts = memory_counts([
("num_entities", retrieval_context.entities.len()),
("num_relations", retrieval_context.relations.len()),
("num_chunks", retrieval_context.chunks.len()),
]);
let llm_context_message = format_llm_context_message(None, &context.hits);
Ok(envelope(
RecallContextResponse {
context: maybe_retrieval_context(include_references, retrieval_context),
llm_context_message,
},
Some(counts),
None,
))
}
Err(message) => Ok(error_envelope("memory.recall_context_failed", message)),
}
}
pub async fn memory_recall_memories(
request: RecallMemoriesRequest,
) -> Result<RpcOutcome<ApiEnvelope<RecallMemoriesResponse>>, String> {
let result = async {
let client = active_memory_client().await?;
client
.recall_namespace_memories(&request.namespace, request.resolved_limit())
.await
}
.await;
match result {
Ok(hits) => {
let memories = hits
.into_iter()
.map(|hit| MemoryRecallItem {
kind: memory_kind_label(&hit.kind).to_string(),
id: hit.id,
content: hit.content,
score: hit.score,
retention: None,
last_accessed_at: None,
access_count: None,
stability_days: None,
})
.collect::<Vec<_>>();
let count = memories.len();
Ok(envelope(
RecallMemoriesResponse { memories },
Some(memory_counts([("num_memories", count)])),
None,
))
}
Err(message) => Ok(error_envelope("memory.recall_memories_failed", message)),
}
}
pub async fn ai_list_memory_files(
request: ListMemoryFilesRequest,
) -> Result<RpcOutcome<ApiEnvelope<ListMemoryFilesResponse>>, String> {
validate_memory_relative_path(&request.relative_dir)?;
let directory = resolve_existing_memory_path(&request.relative_dir).await?;
if !directory.is_dir() {
return Err(format!(
"memory directory not found: {}",
directory.display()
));
}
let mut files = Vec::new();
for entry in std::fs::read_dir(&directory)
.map_err(|e| format!("read memory directory {}: {e}", directory.display()))?
{
let entry = entry.map_err(|e| format!("read memory directory entry: {e}"))?;
let file_name = entry.file_name();
let file_name = file_name.to_string_lossy();
if !file_name.is_empty() {
files.push(file_name.to_string());
}
}
files.sort();
let count = files.len();
Ok(envelope(
ListMemoryFilesResponse {
relative_dir: request.relative_dir,
files,
count,
},
Some(memory_counts([("num_files", count)])),
None,
))
}
pub async fn ai_read_memory_file(
request: ReadMemoryFileRequest,
) -> Result<RpcOutcome<ApiEnvelope<ReadMemoryFileResponse>>, String> {
let path = resolve_existing_memory_path(&request.relative_path).await?;
let content = std::fs::read_to_string(&path)
.map_err(|e| format!("read memory file {}: {e}", path.display()))?;
Ok(envelope(
ReadMemoryFileResponse {
relative_path: request.relative_path,
content,
},
None,
None,
))
}
pub async fn ai_write_memory_file(
request: WriteMemoryFileRequest,
) -> Result<RpcOutcome<ApiEnvelope<WriteMemoryFileResponse>>, String> {
let path = resolve_writable_memory_path(&request.relative_path).await?;
std::fs::write(&path, request.content.as_bytes())
.map_err(|e| format!("write memory file {}: {e}", path.display()))?;
let bytes_written = request.content.len();
Ok(envelope(
WriteMemoryFileResponse {
relative_path: request.relative_path,
written: true,
bytes_written,
},
None,
None,
))
}
#[cfg(test)]
mod tests {
use serde_json::json;
use super::{build_retrieval_context, filter_hits_by_document_ids, format_llm_context_message};
use crate::openhuman::memory::store::GraphRelationRecord;
use crate::openhuman::memory::{MemoryItemKind, NamespaceMemoryHit, RetrievalScoreBreakdown};
fn sample_hit() -> NamespaceMemoryHit {
NamespaceMemoryHit {
id: "doc-1".to_string(),
kind: MemoryItemKind::Document,
namespace: "team".to_string(),
key: "atlas-status".to_string(),
title: Some("Atlas status".to_string()),
content: "Project Atlas is owned by Alice.".to_string(),
category: "core".to_string(),
source_type: Some("doc".to_string()),
updated_at: 1_700_000_000.0,
score: 0.92,
score_breakdown: RetrievalScoreBreakdown {
keyword_relevance: 0.3,
vector_similarity: 0.4,
graph_relevance: 0.9,
freshness: 0.0,
final_score: 0.92,
},
document_id: Some("doc-1".to_string()),
chunk_id: Some("doc-1#chunk-1".to_string()),
supporting_relations: vec![GraphRelationRecord {
namespace: Some("team".to_string()),
subject: "Alice".to_string(),
predicate: "OWNS".to_string(),
object: "Atlas".to_string(),
attrs: json!({"source": "graph"}),
updated_at: 1_700_000_000.0,
evidence_count: 2,
order_index: Some(1),
document_ids: vec!["doc-1".to_string()],
chunk_ids: vec!["doc-1#chunk-1".to_string()],
}],
}
}
#[test]
fn build_retrieval_context_projects_hits_into_relations_and_chunks() {
let context = build_retrieval_context(&[sample_hit()]);
assert_eq!(context.entities.len(), 2);
assert_eq!(context.relations.len(), 1);
assert_eq!(context.chunks.len(), 1);
assert_eq!(context.chunks[0].document_id.as_deref(), Some("doc-1"));
assert_eq!(context.relations[0].predicate, "OWNS");
}
#[test]
fn helpers_filter_document_ids_and_format_context_message() {
let hit = sample_hit();
let filtered = filter_hits_by_document_ids(vec![hit.clone()], Some(&["doc-2".to_string()]));
assert!(filtered.is_empty());
let message = format_llm_context_message(Some("who owns atlas"), &[hit])
.expect("context message should exist");
assert!(message.contains("Query: who owns atlas"));
assert!(message.contains("Alice -[OWNS]-> Atlas"));
}
}
+337
View File
@@ -0,0 +1,337 @@
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ApiError {
pub code: String,
pub message: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub details: Option<serde_json::Value>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PaginationMeta {
pub limit: usize,
pub offset: usize,
pub count: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ApiMeta {
pub request_id: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub latency_seconds: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cached: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub counts: Option<BTreeMap<String, usize>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub pagination: Option<PaginationMeta>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ApiEnvelope<T> {
#[serde(skip_serializing_if = "Option::is_none")]
pub data: Option<T>,
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<ApiError>,
pub meta: ApiMeta,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct EmptyRequest {}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MemoryInitRequest {
pub jwt_token: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemoryInitResponse {
pub initialized: bool,
pub workspace_dir: String,
pub memory_dir: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ListDocumentsRequest {
#[serde(default)]
pub namespace: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemoryDocumentSummary {
pub document_id: String,
pub namespace: String,
pub key: String,
pub title: String,
pub source_type: String,
pub priority: String,
pub created_at: f64,
pub updated_at: f64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ListDocumentsResponse {
#[serde(default)]
pub namespace: Option<String>,
pub documents: Vec<MemoryDocumentSummary>,
pub count: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ListNamespacesResponse {
pub namespaces: Vec<String>,
pub count: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct DeleteDocumentRequest {
pub namespace: String,
pub document_id: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DeleteDocumentResponse {
pub status: String,
pub namespace: String,
pub document_id: String,
pub deleted: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct QueryNamespaceRequest {
pub namespace: String,
pub query: String,
#[serde(default)]
pub include_references: Option<bool>,
#[serde(default)]
pub document_ids: Option<Vec<String>>,
#[serde(default)]
pub limit: Option<u32>,
#[serde(default)]
pub max_chunks: Option<u32>,
}
impl QueryNamespaceRequest {
pub fn resolved_limit(&self) -> u32 {
self.max_chunks.or(self.limit).unwrap_or(10)
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct QueryNamespaceResponse {
#[serde(skip_serializing_if = "Option::is_none")]
pub context: Option<MemoryRetrievalContext>,
#[serde(skip_serializing_if = "Option::is_none")]
pub llm_context_message: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct RecallContextRequest {
pub namespace: String,
#[serde(default)]
pub include_references: Option<bool>,
#[serde(default)]
pub limit: Option<u32>,
#[serde(default)]
pub max_chunks: Option<u32>,
}
impl RecallContextRequest {
pub fn resolved_limit(&self) -> u32 {
self.max_chunks.or(self.limit).unwrap_or(10)
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RecallContextResponse {
#[serde(skip_serializing_if = "Option::is_none")]
pub context: Option<MemoryRetrievalContext>,
#[serde(skip_serializing_if = "Option::is_none")]
pub llm_context_message: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct RecallMemoriesRequest {
pub namespace: String,
/// Accepted for forward compatibility and currently ignored until the core
/// recall path exposes real retention-based filtering semantics.
#[serde(default)]
pub min_retention: Option<f32>,
/// Accepted for forward compatibility and currently ignored until the core
/// recall path exposes real as-of / temporal recall semantics.
#[serde(default)]
pub as_of: Option<f64>,
#[serde(default)]
pub limit: Option<u32>,
#[serde(default)]
pub max_chunks: Option<u32>,
#[serde(default)]
pub top_k: Option<u32>,
}
impl RecallMemoriesRequest {
pub fn resolved_limit(&self) -> u32 {
self.top_k.or(self.max_chunks).or(self.limit).unwrap_or(10)
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemoryRetrievalEntity {
#[serde(skip_serializing_if = "Option::is_none")]
pub id: Option<String>,
pub name: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub entity_type: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub score: Option<f64>,
#[serde(default)]
pub metadata: serde_json::Value,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemoryRetrievalRelation {
pub subject: String,
pub predicate: String,
pub object: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub score: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub evidence_count: Option<u32>,
#[serde(default)]
pub metadata: serde_json::Value,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemoryRetrievalChunk {
#[serde(skip_serializing_if = "Option::is_none")]
pub chunk_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub document_id: Option<String>,
pub content: String,
pub score: f64,
#[serde(default)]
pub metadata: serde_json::Value,
#[serde(skip_serializing_if = "Option::is_none")]
pub created_at: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub updated_at: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemoryRetrievalContext {
pub entities: Vec<MemoryRetrievalEntity>,
pub relations: Vec<MemoryRetrievalRelation>,
pub chunks: Vec<MemoryRetrievalChunk>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemoryRecallItem {
#[serde(rename = "type")]
pub kind: String,
pub id: String,
pub content: String,
pub score: f64,
#[serde(skip_serializing_if = "Option::is_none")]
pub retention: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub last_accessed_at: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub access_count: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub stability_days: Option<f64>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RecallMemoriesResponse {
pub memories: Vec<MemoryRecallItem>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ListMemoryFilesRequest {
#[serde(default = "default_memory_relative_dir")]
pub relative_dir: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ListMemoryFilesResponse {
pub relative_dir: String,
pub files: Vec<String>,
pub count: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ReadMemoryFileRequest {
pub relative_path: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ReadMemoryFileResponse {
pub relative_path: String,
pub content: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct WriteMemoryFileRequest {
pub relative_path: String,
pub content: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WriteMemoryFileResponse {
pub relative_path: String,
pub written: bool,
pub bytes_written: usize,
}
fn default_memory_relative_dir() -> String {
"memory".to_string()
}
#[cfg(test)]
mod tests {
use serde_json::json;
use super::RecallMemoriesRequest;
#[test]
fn recall_memories_request_accepts_compatibility_noop_params() {
let request: RecallMemoriesRequest = serde_json::from_value(json!({
"namespace": "team",
"top_k": 7,
"min_retention": 0.8,
"as_of": 1700000000.0
}))
.expect("compatibility params should deserialize");
assert_eq!(request.namespace, "team");
assert_eq!(request.top_k, Some(7));
assert_eq!(request.min_retention, Some(0.8));
assert_eq!(request.as_of, Some(1_700_000_000.0));
}
#[test]
fn recall_memories_request_limit_resolution_ignores_compatibility_noop_params() {
let request: RecallMemoriesRequest = serde_json::from_value(json!({
"namespace": "team",
"limit": 3,
"min_retention": 0.5,
"as_of": 1700000000.0
}))
.expect("request should deserialize");
assert_eq!(request.resolved_limit(), 3);
}
}
+38 -2
View File
@@ -1,8 +1,11 @@
use serde_json::json;
use std::path::PathBuf;
use std::sync::Arc;
use crate::openhuman::memory::embeddings::{self, EmbeddingProvider};
use crate::openhuman::memory::store::types::NamespaceDocumentInput;
use crate::openhuman::memory::store::types::{
NamespaceDocumentInput, NamespaceMemoryHit, NamespaceRetrievalContext,
};
use crate::openhuman::memory::store::unified::UnifiedMemory;
pub type MemoryClientRef = Arc<MemoryClient>;
@@ -24,9 +27,13 @@ impl MemoryClient {
.ok_or_else(|| "Failed to resolve home directory".to_string())?
.join(".openhuman")
.join("workspace");
Self::from_workspace_dir(workspace_dir)
}
pub fn from_workspace_dir(workspace_dir: PathBuf) -> Result<Self, String> {
std::fs::create_dir_all(&workspace_dir)
.map_err(|e| format!("Create workspace dir {}: {e}", workspace_dir.display()))?;
let embedder: Arc<dyn EmbeddingProvider> = Arc::new(embeddings::NoopEmbedding);
let embedder: Arc<dyn EmbeddingProvider> = embeddings::default_local_embedding_provider();
let memory =
UnifiedMemory::new(&workspace_dir, embedder, None).map_err(|e| format!("{e}"))?;
Ok(Self {
@@ -121,6 +128,17 @@ impl MemoryClient {
.await
}
pub async fn query_namespace_context_data(
&self,
namespace: &str,
query: &str,
max_chunks: u32,
) -> Result<NamespaceRetrievalContext, String> {
self.inner
.query_namespace_context_data(namespace, query, max_chunks)
.await
}
pub async fn recall_namespace(
&self,
namespace: &str,
@@ -131,6 +149,24 @@ impl MemoryClient {
.await
}
pub async fn recall_namespace_context_data(
&self,
namespace: &str,
max_chunks: u32,
) -> Result<NamespaceRetrievalContext, String> {
self.inner
.recall_namespace_context_data(namespace, max_chunks)
.await
}
pub async fn recall_namespace_memories(
&self,
namespace: &str,
limit: u32,
) -> Result<Vec<NamespaceMemoryHit>, String> {
self.inner.recall_namespace_memories(namespace, limit).await
}
pub async fn kv_set(
&self,
namespace: Option<&str>,
+1 -1
View File
@@ -70,7 +70,7 @@ impl Memory for UnifiedMemory {
category: memory_category_from_stored(&r.category),
timestamp: Utc::now().to_rfc3339(),
session_id: None,
score: Some(r.score * 100.0),
score: Some(r.score),
})
.collect();
Ok(out)
+5 -1
View File
@@ -12,5 +12,9 @@ pub use factories::{
create_memory, create_memory_for_migration, create_memory_with_storage,
create_memory_with_storage_and_routes, effective_memory_backend_name,
};
pub use types::{NamespaceDocumentInput, NamespaceQueryResult};
pub use types::{
GraphRelationRecord, MemoryItemKind, MemoryKvRecord, NamespaceDocumentInput,
NamespaceMemoryHit, NamespaceQueryResult, NamespaceRetrievalContext, RetrievalScoreBreakdown,
StoredMemoryDocument,
};
pub use unified::UnifiedMemory;
+85
View File
@@ -31,3 +31,88 @@ pub struct NamespaceQueryResult {
/// Stored category string (e.g. `core`, `daily`, or custom label).
pub category: String,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum MemoryItemKind {
Document,
Kv,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StoredMemoryDocument {
pub document_id: String,
pub namespace: String,
pub key: String,
pub title: String,
pub content: String,
pub source_type: String,
pub priority: String,
pub tags: Vec<String>,
pub metadata: serde_json::Value,
pub category: String,
pub session_id: Option<String>,
pub created_at: f64,
pub updated_at: f64,
pub markdown_rel_path: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemoryKvRecord {
pub namespace: Option<String>,
pub key: String,
pub value: serde_json::Value,
pub updated_at: f64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GraphRelationRecord {
pub namespace: Option<String>,
pub subject: String,
pub predicate: String,
pub object: String,
pub attrs: serde_json::Value,
pub updated_at: f64,
pub evidence_count: u32,
pub order_index: Option<i64>,
pub document_ids: Vec<String>,
pub chunk_ids: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct RetrievalScoreBreakdown {
pub keyword_relevance: f64,
pub vector_similarity: f64,
pub graph_relevance: f64,
pub freshness: f64,
pub final_score: f64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NamespaceMemoryHit {
pub id: String,
pub kind: MemoryItemKind,
pub namespace: String,
pub key: String,
pub title: Option<String>,
pub content: String,
pub category: String,
pub source_type: Option<String>,
pub updated_at: f64,
pub score: f64,
pub score_breakdown: RetrievalScoreBreakdown,
#[serde(default)]
pub document_id: Option<String>,
#[serde(default)]
pub chunk_id: Option<String>,
#[serde(default)]
pub supporting_relations: Vec<GraphRelationRecord>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NamespaceRetrievalContext {
pub namespace: String,
pub query: Option<String>,
pub context_text: String,
pub hits: Vec<NamespaceMemoryHit>,
}
+105 -17
View File
@@ -3,7 +3,7 @@ use serde_json::{json, Value};
use std::collections::BTreeSet;
use uuid::Uuid;
use crate::openhuman::memory::store::types::NamespaceDocumentInput;
use crate::openhuman::memory::store::types::{NamespaceDocumentInput, StoredMemoryDocument};
use super::UnifiedMemory;
@@ -14,11 +14,32 @@ impl UnifiedMemory {
if key.is_empty() {
return Err("document key cannot be empty".to_string());
}
let existing_document_id = {
let conn = self.conn.lock();
conn.query_row(
"SELECT document_id FROM memory_docs WHERE namespace = ?1 AND key = ?2 LIMIT 1",
params![namespace, key],
|row| row.get::<_, String>(0),
)
.optional()
.map_err(|e| format!("lookup existing document_id: {e}"))?
};
let document_id = input
.document_id
.or(existing_document_id)
.unwrap_or_else(|| Uuid::new_v4().to_string());
let now = Self::now_ts();
let created_at = now;
let created_at = {
let conn = self.conn.lock();
conn.query_row(
"SELECT created_at FROM memory_docs WHERE namespace = ?1 AND key = ?2 LIMIT 1",
params![namespace, key],
|row| row.get::<_, f64>(0),
)
.optional()
.map_err(|e| format!("lookup existing created_at: {e}"))?
.unwrap_or(now)
};
let updated_at = now;
let markdown_rel = self
.write_markdown_doc(
@@ -84,7 +105,7 @@ impl UnifiedMemory {
tx.commit().map_err(|e| format!("commit tx: {e}"))?;
}
let chunks = Self::split_chunks(&input.content, 900);
let chunks = Self::chunk_document_content(&input.content, 225);
for (idx, chunk) in chunks.iter().enumerate() {
let embedding = self
.embedder
@@ -115,6 +136,64 @@ impl UnifiedMemory {
Ok(document_id)
}
pub(crate) async fn load_documents_for_scope(
&self,
namespace: &str,
) -> Result<Vec<StoredMemoryDocument>, String> {
let conn = self.conn.lock();
let ns = Self::sanitize_namespace(namespace);
let mut stmt = conn
.prepare(
"SELECT
document_id,
namespace,
key,
title,
content,
source_type,
priority,
tags_json,
metadata_json,
category,
session_id,
created_at,
updated_at,
markdown_rel_path
FROM memory_docs
WHERE namespace = ?1
ORDER BY updated_at DESC",
)
.map_err(|e| format!("prepare load_documents_for_scope: {e}"))?;
let mut rows = stmt
.query(params![ns])
.map_err(|e| format!("query load_documents_for_scope: {e}"))?;
let mut docs = Vec::new();
while let Some(row) = rows
.next()
.map_err(|e| format!("row load_documents_for_scope: {e}"))?
{
let tags_json: String = row.get(7).map_err(|e| e.to_string())?;
let metadata_json: String = row.get(8).map_err(|e| e.to_string())?;
docs.push(StoredMemoryDocument {
document_id: row.get(0).map_err(|e| e.to_string())?,
namespace: row.get(1).map_err(|e| e.to_string())?,
key: row.get(2).map_err(|e| e.to_string())?,
title: row.get(3).map_err(|e| e.to_string())?,
content: row.get(4).map_err(|e| e.to_string())?,
source_type: row.get(5).map_err(|e| e.to_string())?,
priority: row.get(6).map_err(|e| e.to_string())?,
tags: serde_json::from_str(&tags_json).unwrap_or_default(),
metadata: serde_json::from_str(&metadata_json).unwrap_or_else(|_| json!({})),
category: row.get(9).map_err(|e| e.to_string())?,
session_id: row.get(10).map_err(|e| e.to_string())?,
created_at: row.get(11).map_err(|e| e.to_string())?,
updated_at: row.get(12).map_err(|e| e.to_string())?,
markdown_rel_path: row.get(13).map_err(|e| e.to_string())?,
});
}
Ok(docs)
}
pub async fn list_documents(&self, namespace: Option<&str>) -> Result<Value, String> {
let conn = self.conn.lock();
let mut docs = Vec::new();
@@ -199,27 +278,36 @@ impl UnifiedMemory {
document_id: &str,
) -> Result<Value, String> {
let ns = Self::sanitize_namespace(namespace);
let conn = self.conn.lock();
let rel_path: Option<String> = conn
.query_row(
let rel_path: Option<String> = {
let conn = self.conn.lock();
conn.query_row(
"SELECT markdown_rel_path FROM memory_docs WHERE namespace = ?1 AND document_id = ?2",
params![ns, document_id],
|row| row.get(0),
)
.optional()
.map_err(|e| format!("query delete_document path: {e}"))?;
let deleted = conn
.execute(
"DELETE FROM memory_docs WHERE namespace = ?1 AND document_id = ?2",
.map_err(|e| format!("query delete_document path: {e}"))?
};
self.graph_remove_document_namespace(&ns, document_id)
.await?;
let deleted = {
let conn = self.conn.lock();
let deleted = conn
.execute(
"DELETE FROM memory_docs WHERE namespace = ?1 AND document_id = ?2",
params![ns, document_id],
)
.map_err(|e| format!("delete memory_doc: {e}"))?
> 0;
conn.execute(
"DELETE FROM vector_chunks WHERE namespace = ?1 AND document_id = ?2",
params![ns, document_id],
)
.map_err(|e| format!("delete memory_doc: {e}"))?
> 0;
conn.execute(
"DELETE FROM vector_chunks WHERE namespace = ?1 AND document_id = ?2",
params![ns, document_id],
)
.map_err(|e| format!("delete vector_chunks: {e}"))?;
.map_err(|e| format!("delete vector_chunks: {e}"))?;
deleted
};
if let Some(rel) = rel_path {
let abs = self.workspace_dir.join(rel);
+380 -71
View File
@@ -1,9 +1,99 @@
use rusqlite::params;
use serde_json::json;
use rusqlite::{params, OptionalExtension};
use serde_json::{json, Map, Value};
use crate::openhuman::memory::store::types::GraphRelationRecord;
use super::UnifiedMemory;
impl UnifiedMemory {
pub(crate) async fn graph_remove_document_namespace(
&self,
namespace: &str,
document_id: &str,
) -> Result<(), String> {
let relations = self
.graph_relations_namespace(namespace, None, None)
.await?;
if relations.is_empty() {
return Ok(());
}
let doc_prefix = format!("{document_id}:");
let updated_at = Self::now_ts();
let conn = self.conn.lock();
let tx = conn
.unchecked_transaction()
.map_err(|e| format!("graph_remove_document_namespace begin tx: {e}"))?;
for relation in relations {
let touches_document = relation.document_ids.iter().any(|id| id == document_id)
|| relation
.chunk_ids
.iter()
.any(|chunk_id| chunk_id.starts_with(&doc_prefix));
if !touches_document {
continue;
}
let mut attrs = relation.attrs.as_object().cloned().unwrap_or_default();
let document_ids = relation
.document_ids
.iter()
.filter(|id| id.as_str() != document_id)
.cloned()
.collect::<Vec<_>>();
let chunk_ids = relation
.chunk_ids
.iter()
.filter(|chunk_id| !chunk_id.starts_with(&doc_prefix))
.cloned()
.collect::<Vec<_>>();
if document_ids.is_empty() && chunk_ids.is_empty() {
tx.execute(
"DELETE FROM graph_namespace
WHERE namespace = ?1 AND subject = ?2 AND predicate = ?3 AND object = ?4",
params![
Self::sanitize_namespace(namespace),
relation.subject,
relation.predicate,
relation.object
],
)
.map_err(|e| format!("graph_remove_document_namespace delete: {e}"))?;
continue;
}
attrs.insert("document_ids".to_string(), json!(document_ids));
if chunk_ids.is_empty() {
attrs.remove("chunk_ids");
} else {
attrs.insert("chunk_ids".to_string(), json!(chunk_ids.clone()));
}
attrs.insert("evidence_count".to_string(), json!(chunk_ids.len().max(1)));
attrs.insert("updated_at".to_string(), json!(updated_at));
tx.execute(
"UPDATE graph_namespace
SET attrs_json = ?1, updated_at = ?2
WHERE namespace = ?3 AND subject = ?4 AND predicate = ?5 AND object = ?6",
params![
Value::Object(attrs).to_string(),
updated_at,
Self::sanitize_namespace(namespace),
relation.subject,
relation.predicate,
relation.object
],
)
.map_err(|e| format!("graph_remove_document_namespace update: {e}"))?;
}
tx.commit()
.map_err(|e| format!("graph_remove_document_namespace commit: {e}"))?;
Ok(())
}
pub async fn graph_upsert_global(
&self,
subject: &str,
@@ -11,15 +101,8 @@ impl UnifiedMemory {
object: &str,
attrs: &serde_json::Value,
) -> Result<(), String> {
let conn = self.conn.lock();
conn.execute(
"INSERT INTO graph_global (subject, predicate, object, attrs_json, updated_at)
VALUES (?1, ?2, ?3, ?4, ?5)
ON CONFLICT(subject, predicate, object) DO UPDATE SET attrs_json = excluded.attrs_json, updated_at = excluded.updated_at",
params![subject, predicate, object, attrs.to_string(), Self::now_ts()],
)
.map_err(|e| format!("graph_upsert_global: {e}"))?;
Ok(())
self.graph_upsert_internal(None, subject, predicate, object, attrs)
.await
}
pub async fn graph_upsert_namespace(
@@ -30,22 +113,8 @@ impl UnifiedMemory {
object: &str,
attrs: &serde_json::Value,
) -> Result<(), String> {
let conn = self.conn.lock();
conn.execute(
"INSERT INTO graph_namespace (namespace, subject, predicate, object, attrs_json, updated_at)
VALUES (?1, ?2, ?3, ?4, ?5, ?6)
ON CONFLICT(namespace, subject, predicate, object) DO UPDATE SET attrs_json = excluded.attrs_json, updated_at = excluded.updated_at",
params![
Self::sanitize_namespace(namespace),
subject,
predicate,
object,
attrs.to_string(),
Self::now_ts()
],
)
.map_err(|e| format!("graph_upsert_namespace: {e}"))?;
Ok(())
self.graph_upsert_internal(Some(namespace), subject, predicate, object, attrs)
.await
}
pub async fn graph_query_global(
@@ -53,7 +122,94 @@ impl UnifiedMemory {
subject: Option<&str>,
predicate: Option<&str>,
) -> Result<Vec<serde_json::Value>, String> {
let rows = self.graph_relations_global(subject, predicate).await?;
Ok(rows
.into_iter()
.map(Self::graph_relation_to_json)
.collect::<Vec<_>>())
}
pub async fn graph_query_namespace(
&self,
namespace: &str,
subject: Option<&str>,
predicate: Option<&str>,
) -> Result<Vec<serde_json::Value>, String> {
let rows = self
.graph_relations_namespace(namespace, subject, predicate)
.await?;
Ok(rows
.into_iter()
.map(Self::graph_relation_to_json)
.collect::<Vec<_>>())
}
pub(crate) async fn graph_relations_for_scope(
&self,
namespace: &str,
) -> Result<Vec<GraphRelationRecord>, String> {
let mut rows = self
.graph_relations_namespace(namespace, None, None)
.await?;
rows.extend(self.graph_relations_global(None, None).await?);
rows.sort_by(|a, b| {
b.updated_at
.partial_cmp(&a.updated_at)
.unwrap_or(std::cmp::Ordering::Equal)
});
Ok(rows)
}
pub(crate) async fn graph_relations_namespace(
&self,
namespace: &str,
subject: Option<&str>,
predicate: Option<&str>,
) -> Result<Vec<GraphRelationRecord>, String> {
let conn = self.conn.lock();
let ns = Self::sanitize_namespace(namespace);
let subject = subject.map(Self::normalize_graph_entity);
let predicate = predicate.map(Self::normalize_graph_predicate);
let mut stmt = conn
.prepare(
"SELECT subject, predicate, object, attrs_json, updated_at
FROM graph_namespace
WHERE namespace = ?1
AND (?2 IS NULL OR subject = ?2)
AND (?3 IS NULL OR predicate = ?3)
ORDER BY updated_at DESC
LIMIT 300",
)
.map_err(|e| format!("graph_relations_namespace prepare: {e}"))?;
let mut rows = stmt
.query(params![ns, subject, predicate])
.map_err(|e| format!("graph_relations_namespace query: {e}"))?;
let mut out = Vec::new();
while let Some(row) = rows
.next()
.map_err(|e| format!("graph_relations_namespace row: {e}"))?
{
let attrs_raw: String = row.get(3).map_err(|e| e.to_string())?;
out.push(Self::graph_relation_from_parts(
Some(Self::sanitize_namespace(namespace)),
row.get(0).map_err(|e| e.to_string())?,
row.get(1).map_err(|e| e.to_string())?,
row.get(2).map_err(|e| e.to_string())?,
&attrs_raw,
row.get(4).map_err(|e| e.to_string())?,
));
}
Ok(out)
}
pub(crate) async fn graph_relations_global(
&self,
subject: Option<&str>,
predicate: Option<&str>,
) -> Result<Vec<GraphRelationRecord>, String> {
let conn = self.conn.lock();
let subject = subject.map(Self::normalize_graph_entity);
let predicate = predicate.map(Self::normalize_graph_predicate);
let mut stmt = conn
.prepare(
"SELECT subject, predicate, object, attrs_json, updated_at
@@ -63,63 +219,216 @@ impl UnifiedMemory {
ORDER BY updated_at DESC
LIMIT 300",
)
.map_err(|e| format!("graph_query_global prepare: {e}"))?;
.map_err(|e| format!("graph_relations_global prepare: {e}"))?;
let mut rows = stmt
.query(params![subject, predicate])
.map_err(|e| format!("graph_query_global query: {e}"))?;
.map_err(|e| format!("graph_relations_global query: {e}"))?;
let mut out = Vec::new();
while let Some(row) = rows
.next()
.map_err(|e| format!("graph_query_global row: {e}"))?
.map_err(|e| format!("graph_relations_global row: {e}"))?
{
let attrs_raw: String = row.get(3).map_err(|e| e.to_string())?;
out.push(json!({
"subject": row.get::<_, String>(0).map_err(|e| e.to_string())?,
"predicate": row.get::<_, String>(1).map_err(|e| e.to_string())?,
"object": row.get::<_, String>(2).map_err(|e| e.to_string())?,
"attrs": serde_json::from_str::<serde_json::Value>(&attrs_raw).unwrap_or_else(|_| json!({})),
"updatedAt": row.get::<_, f64>(4).map_err(|e| e.to_string())?,
}));
out.push(Self::graph_relation_from_parts(
None,
row.get(0).map_err(|e| e.to_string())?,
row.get(1).map_err(|e| e.to_string())?,
row.get(2).map_err(|e| e.to_string())?,
&attrs_raw,
row.get(4).map_err(|e| e.to_string())?,
));
}
Ok(out)
}
pub async fn graph_query_namespace(
async fn graph_upsert_internal(
&self,
namespace: &str,
subject: Option<&str>,
predicate: Option<&str>,
) -> Result<Vec<serde_json::Value>, String> {
namespace: Option<&str>,
subject: &str,
predicate: &str,
object: &str,
attrs: &serde_json::Value,
) -> Result<(), String> {
let subject = Self::normalize_graph_entity(subject);
let predicate = Self::normalize_graph_predicate(predicate);
let object = Self::normalize_graph_entity(object);
let updated_at = Self::now_ts();
let conn = self.conn.lock();
let ns = Self::sanitize_namespace(namespace);
let mut stmt = conn
.prepare(
"SELECT subject, predicate, object, attrs_json, updated_at
FROM graph_namespace
WHERE namespace = ?1
AND (?2 IS NULL OR subject = ?2)
AND (?3 IS NULL OR predicate = ?3)
ORDER BY updated_at DESC
LIMIT 300",
)
.map_err(|e| format!("graph_query_namespace prepare: {e}"))?;
let mut rows = stmt
.query(params![ns, subject, predicate])
.map_err(|e| format!("graph_query_namespace query: {e}"))?;
let mut out = Vec::new();
while let Some(row) = rows
.next()
.map_err(|e| format!("graph_query_namespace row: {e}"))?
{
let attrs_raw: String = row.get(3).map_err(|e| e.to_string())?;
out.push(json!({
"subject": row.get::<_, String>(0).map_err(|e| e.to_string())?,
"predicate": row.get::<_, String>(1).map_err(|e| e.to_string())?,
"object": row.get::<_, String>(2).map_err(|e| e.to_string())?,
"attrs": serde_json::from_str::<serde_json::Value>(&attrs_raw).unwrap_or_else(|_| json!({})),
"updatedAt": row.get::<_, f64>(4).map_err(|e| e.to_string())?,
}));
let existing_attrs: Option<String> = match namespace {
Some(ns) => conn
.query_row(
"SELECT attrs_json
FROM graph_namespace
WHERE namespace = ?1 AND subject = ?2 AND predicate = ?3 AND object = ?4",
params![Self::sanitize_namespace(ns), subject, predicate, object],
|row| row.get(0),
)
.optional()
.map_err(|e| format!("graph_upsert_namespace lookup: {e}"))?,
None => conn
.query_row(
"SELECT attrs_json
FROM graph_global
WHERE subject = ?1 AND predicate = ?2 AND object = ?3",
params![subject, predicate, object],
|row| row.get(0),
)
.optional()
.map_err(|e| format!("graph_upsert_global lookup: {e}"))?,
};
let merged_attrs = Self::merge_graph_attrs(existing_attrs.as_deref(), attrs, updated_at);
let merged_attrs_json = merged_attrs.to_string();
match namespace {
Some(ns) => {
conn.execute(
"INSERT INTO graph_namespace (namespace, subject, predicate, object, attrs_json, updated_at)
VALUES (?1, ?2, ?3, ?4, ?5, ?6)
ON CONFLICT(namespace, subject, predicate, object)
DO UPDATE SET attrs_json = excluded.attrs_json, updated_at = excluded.updated_at",
params![
Self::sanitize_namespace(ns),
subject,
predicate,
object,
merged_attrs_json,
updated_at
],
)
.map_err(|e| format!("graph_upsert_namespace: {e}"))?;
}
None => {
conn.execute(
"INSERT INTO graph_global (subject, predicate, object, attrs_json, updated_at)
VALUES (?1, ?2, ?3, ?4, ?5)
ON CONFLICT(subject, predicate, object)
DO UPDATE SET attrs_json = excluded.attrs_json, updated_at = excluded.updated_at",
params![subject, predicate, object, merged_attrs_json, updated_at],
)
.map_err(|e| format!("graph_upsert_global: {e}"))?;
}
}
Ok(out)
Ok(())
}
fn merge_graph_attrs(
existing_attrs_raw: Option<&str>,
incoming_attrs: &Value,
updated_at: f64,
) -> Value {
let existing = existing_attrs_raw
.and_then(|raw| serde_json::from_str::<Value>(raw).ok())
.unwrap_or_else(|| json!({}));
let existing_evidence = Self::json_i64(&existing, "evidence_count")
.unwrap_or(0)
.max(0) as u64;
let existing_document_ids =
Self::json_string_array(&existing, "document_ids", "document_id");
let existing_chunk_ids = Self::json_string_array(&existing, "chunk_ids", "chunk_id");
let mut merged = match existing {
Value::Object(map) => map,
_ => Map::new(),
};
let incoming_map = incoming_attrs.as_object().cloned().unwrap_or_default();
let existing_order_index = Self::json_i64(&Value::Object(merged.clone()), "order_index");
let incoming_order_index = Self::json_i64(incoming_attrs, "order_index");
let merged_order_index = match (existing_order_index, incoming_order_index) {
(Some(left), Some(right)) => Some(left.min(right)),
(Some(left), None) => Some(left),
(None, Some(right)) => Some(right),
(None, None) => None,
};
for (key, value) in incoming_map {
merged.insert(key, value);
}
let incoming_evidence = Self::json_i64(incoming_attrs, "evidence_count")
.unwrap_or(1)
.max(0) as u64;
let evidence_count = existing_evidence.saturating_add(incoming_evidence).max(1);
merged.insert("evidence_count".to_string(), json!(evidence_count));
merged.insert("updated_at".to_string(), json!(updated_at));
let mut document_ids = existing_document_ids;
document_ids.extend(Self::json_string_array(
incoming_attrs,
"document_ids",
"document_id",
));
document_ids.sort();
document_ids.dedup();
if !document_ids.is_empty() {
merged.insert("document_ids".to_string(), json!(document_ids));
}
let mut chunk_ids = existing_chunk_ids;
chunk_ids.extend(Self::json_string_array(
incoming_attrs,
"chunk_ids",
"chunk_id",
));
chunk_ids.sort();
chunk_ids.dedup();
if !chunk_ids.is_empty() {
merged.insert("chunk_ids".to_string(), json!(chunk_ids));
}
if !merged.contains_key("created_at") {
merged.insert("created_at".to_string(), json!(updated_at));
}
if let Some(order_index) = merged_order_index {
merged.insert("order_index".to_string(), json!(order_index));
}
Value::Object(merged)
}
fn graph_relation_from_parts(
namespace: Option<String>,
subject: String,
predicate: String,
object: String,
attrs_raw: &str,
updated_at: f64,
) -> GraphRelationRecord {
let attrs = serde_json::from_str::<Value>(attrs_raw).unwrap_or_else(|_| json!({}));
let evidence_count = Self::json_i64(&attrs, "evidence_count").unwrap_or(1).max(1) as u32;
let order_index = Self::json_i64(&attrs, "order_index");
let document_ids = Self::json_string_array(&attrs, "document_ids", "document_id");
let chunk_ids = Self::json_string_array(&attrs, "chunk_ids", "chunk_id");
GraphRelationRecord {
namespace,
subject,
predicate,
object,
attrs,
updated_at,
evidence_count,
order_index,
document_ids,
chunk_ids,
}
}
fn graph_relation_to_json(record: GraphRelationRecord) -> serde_json::Value {
json!({
"namespace": record.namespace,
"subject": record.subject,
"predicate": record.predicate,
"object": record.object,
"attrs": record.attrs,
"updatedAt": record.updated_at,
"evidenceCount": record.evidence_count,
"orderIndex": record.order_index,
"documentIds": record.document_ids,
"chunkIds": record.chunk_ids,
})
}
}
+108 -21
View File
@@ -1,3 +1,5 @@
use crate::openhuman::memory::chunker::chunk_markdown;
use super::UnifiedMemory;
impl UnifiedMemory {
@@ -74,29 +76,114 @@ impl UnifiedMemory {
(dot / denom).clamp(0.0, 1.0)
}
pub(crate) fn split_chunks(content: &str, max_len: usize) -> Vec<String> {
let mut out = Vec::new();
let mut current = String::new();
for para in content.split("\n\n") {
let p = para.trim();
if p.is_empty() {
continue;
}
if current.is_empty() {
current.push_str(p);
continue;
}
if current.len() + 2 + p.len() <= max_len {
current.push_str("\n\n");
current.push_str(p);
} else {
out.push(std::mem::take(&mut current));
current.push_str(p);
pub(crate) fn chunk_document_content(content: &str, max_tokens: usize) -> Vec<String> {
let mut chunks: Vec<String> = chunk_markdown(content, max_tokens.max(1))
.into_iter()
.map(|chunk| chunk.content.trim().to_string())
.filter(|chunk: &String| !chunk.is_empty())
.collect();
if chunks.is_empty() && !content.trim().is_empty() {
chunks.push(content.trim().to_string());
}
chunks
}
pub(crate) fn collapse_whitespace(text: &str) -> String {
text.split_whitespace().collect::<Vec<_>>().join(" ")
}
pub(crate) fn normalize_search_text(text: &str) -> String {
let collapsed = Self::collapse_whitespace(text);
let mut normalized = String::with_capacity(collapsed.len());
for ch in collapsed.chars() {
if ch.is_alphanumeric() {
normalized.extend(ch.to_lowercase());
} else if ch.is_whitespace() || matches!(ch, '_' | '-' | '/' | '.') {
normalized.push(' ');
}
}
if !current.trim().is_empty() {
out.push(current);
normalized.split_whitespace().collect::<Vec<_>>().join(" ")
}
pub(crate) fn tokenize_search_terms(text: &str) -> Vec<String> {
Self::normalize_search_text(text)
.split_whitespace()
.map(ToOwned::to_owned)
.collect()
}
pub(crate) fn normalize_graph_entity(text: &str) -> String {
Self::collapse_whitespace(text.trim()).to_uppercase()
}
pub(crate) fn normalize_graph_predicate(text: &str) -> String {
let mut out = String::new();
let mut last_was_sep = false;
for ch in Self::collapse_whitespace(text.trim()).chars() {
if ch.is_alphanumeric() {
out.extend(ch.to_uppercase());
last_was_sep = false;
} else if !last_was_sep {
out.push('_');
last_was_sep = true;
}
}
out
out.trim_matches('_').to_string()
}
pub(crate) fn json_string_array(
value: &serde_json::Value,
primary_key: &str,
singular_key: &str,
) -> Vec<String> {
let mut items = Vec::new();
if let Some(array) = value.get(primary_key).and_then(serde_json::Value::as_array) {
for item in array {
if let Some(text) = item.as_str() {
let trimmed = text.trim();
if !trimmed.is_empty() {
items.push(trimmed.to_string());
}
}
}
}
if let Some(text) = value.get(singular_key).and_then(serde_json::Value::as_str) {
let trimmed = text.trim();
if !trimmed.is_empty() {
items.push(trimmed.to_string());
}
}
items.sort();
items.dedup();
items
}
pub(crate) fn merge_unique_string_arrays(
current: &serde_json::Value,
incoming: &serde_json::Value,
primary_key: &str,
singular_key: &str,
) -> Vec<String> {
let mut merged = Self::json_string_array(current, primary_key, singular_key);
merged.extend(Self::json_string_array(incoming, primary_key, singular_key));
merged.sort();
merged.dedup();
merged
}
pub(crate) fn json_i64(value: &serde_json::Value, key: &str) -> Option<i64> {
value.get(key).and_then(|raw| {
raw.as_i64().or_else(|| {
raw.as_u64()
.and_then(|v| i64::try_from(v).ok())
.or_else(|| raw.as_f64().map(|v| v as i64))
})
})
}
pub(crate) fn recency_score(updated_at: f64, now: f64) -> f64 {
let age_secs = (now - updated_at).max(0.0);
let age_hours = age_secs / 3600.0;
(1.0 / (1.0 + age_hours / 24.0)).clamp(0.0, 1.0)
}
}
@@ -23,6 +23,13 @@ impl UnifiedMemory {
let db_path = memory_dir.join("memory.db");
let conn = Connection::open(&db_path)?;
// Active storage layout for the core memory domain:
// - memory_docs: namespace-scoped source documents and markdown metadata.
// - vector_chunks: chunked document text plus optional local embedding bytes.
// - graph_namespace: namespace graph edges used for relation-first retrieval.
// - graph_global: cross-namespace graph edges used as fallback/shared memory.
// - kv_namespace: namespace-scoped durable preferences, decisions, and state.
// - kv_global: global durable key-value memories outside a namespace scope.
conn.execute_batch(
"PRAGMA journal_mode = WAL;
PRAGMA synchronous = NORMAL;
@@ -69,6 +76,7 @@ impl UnifiedMemory {
updated_at REAL NOT NULL,
PRIMARY KEY(subject, predicate, object)
);
CREATE INDEX IF NOT EXISTS idx_graph_global_subject ON graph_global(subject, predicate);
CREATE TABLE IF NOT EXISTS graph_namespace (
namespace TEXT NOT NULL,
@@ -80,6 +88,7 @@ impl UnifiedMemory {
PRIMARY KEY(namespace, subject, predicate, object)
);
CREATE INDEX IF NOT EXISTS idx_graph_namespace_ns ON graph_namespace(namespace);
CREATE INDEX IF NOT EXISTS idx_graph_namespace_subject ON graph_namespace(namespace, subject, predicate);
CREATE TABLE IF NOT EXISTS vector_chunks (
namespace TEXT NOT NULL,
+74
View File
@@ -1,6 +1,8 @@
use rusqlite::{params, OptionalExtension};
use serde_json::json;
use crate::openhuman::memory::store::types::MemoryKvRecord;
use super::UnifiedMemory;
impl UnifiedMemory {
@@ -110,4 +112,76 @@ impl UnifiedMemory {
}
Ok(out)
}
pub(crate) async fn kv_records_for_scope(
&self,
namespace: &str,
) -> Result<Vec<MemoryKvRecord>, String> {
let mut records = self.kv_records_namespace(namespace).await?;
records.extend(self.kv_records_global().await?);
records.sort_by(|a, b| {
b.updated_at
.partial_cmp(&a.updated_at)
.unwrap_or(std::cmp::Ordering::Equal)
});
Ok(records)
}
pub(crate) async fn kv_records_namespace(
&self,
namespace: &str,
) -> Result<Vec<MemoryKvRecord>, String> {
let conn = self.conn.lock();
let mut stmt = conn
.prepare(
"SELECT key, value_json, updated_at FROM kv_namespace
WHERE namespace = ?1
ORDER BY updated_at DESC",
)
.map_err(|e| format!("prepare kv_records_namespace: {e}"))?;
let mut rows = stmt
.query(params![Self::sanitize_namespace(namespace)])
.map_err(|e| format!("query kv_records_namespace: {e}"))?;
let mut out = Vec::new();
while let Some(row) = rows
.next()
.map_err(|e| format!("row kv_records_namespace: {e}"))?
{
let value_raw: String = row.get(1).map_err(|e| e.to_string())?;
out.push(MemoryKvRecord {
namespace: Some(Self::sanitize_namespace(namespace)),
key: row.get(0).map_err(|e| e.to_string())?,
value: serde_json::from_str(&value_raw).unwrap_or(serde_json::Value::Null),
updated_at: row.get(2).map_err(|e| e.to_string())?,
});
}
Ok(out)
}
pub(crate) async fn kv_records_global(&self) -> Result<Vec<MemoryKvRecord>, String> {
let conn = self.conn.lock();
let mut stmt = conn
.prepare(
"SELECT key, value_json, updated_at FROM kv_global
ORDER BY updated_at DESC",
)
.map_err(|e| format!("prepare kv_records_global: {e}"))?;
let mut rows = stmt
.query([])
.map_err(|e| format!("query kv_records_global: {e}"))?;
let mut out = Vec::new();
while let Some(row) = rows
.next()
.map_err(|e| format!("row kv_records_global: {e}"))?
{
let value_raw: String = row.get(1).map_err(|e| e.to_string())?;
out.push(MemoryKvRecord {
namespace: None,
key: row.get(0).map_err(|e| e.to_string())?,
value: serde_json::from_str(&value_raw).unwrap_or(serde_json::Value::Null),
updated_at: row.get(2).map_err(|e| e.to_string())?,
});
}
Ok(out)
}
}
File diff suppressed because it is too large Load Diff
+86
View File
@@ -21,6 +21,92 @@ pub async fn try_dispatch(
params: serde_json::Value,
) -> Option<Result<serde_json::Value, String>> {
match method {
"memory.init" => Some(
async move {
let payload: crate::openhuman::memory::MemoryInitRequest = parse_params(params)?;
rpc_json(crate::openhuman::memory::rpc::memory_init(payload).await?)
}
.await,
),
"memory.list_documents" => Some(
async move {
let payload: crate::openhuman::memory::ListDocumentsRequest = parse_params(params)?;
rpc_json(crate::openhuman::memory::rpc::memory_list_documents(payload).await?)
}
.await,
),
"memory.list_namespaces" => Some(
async move {
let payload: crate::openhuman::memory::EmptyRequest = parse_params(params)?;
rpc_json(crate::openhuman::memory::rpc::memory_list_namespaces(payload).await?)
}
.await,
),
"memory.delete_document" => Some(
async move {
let payload: crate::openhuman::memory::DeleteDocumentRequest =
parse_params(params)?;
rpc_json(crate::openhuman::memory::rpc::memory_delete_document(payload).await?)
}
.await,
),
"memory.query_namespace" => Some(
async move {
let payload: crate::openhuman::memory::QueryNamespaceRequest =
parse_params(params)?;
rpc_json(crate::openhuman::memory::rpc::memory_query_namespace(payload).await?)
}
.await,
),
"memory.recall_context" | "memory.recall_namespace" => Some(
async move {
let payload: crate::openhuman::memory::RecallContextRequest = parse_params(params)?;
rpc_json(crate::openhuman::memory::rpc::memory_recall_context(payload).await?)
}
.await,
),
"memory.recall_memories" => Some(
async move {
let payload: crate::openhuman::memory::RecallMemoriesRequest =
parse_params(params)?;
rpc_json(crate::openhuman::memory::rpc::memory_recall_memories(payload).await?)
}
.await,
),
"ai.list_memory_files" => Some(
async move {
let payload: crate::openhuman::memory::ListMemoryFilesRequest =
parse_params(params)?;
rpc_json(crate::openhuman::memory::rpc::ai_list_memory_files(payload).await?)
}
.await,
),
"ai.read_memory_file" => Some(
async move {
let payload: crate::openhuman::memory::ReadMemoryFileRequest =
parse_params(params)?;
rpc_json(crate::openhuman::memory::rpc::ai_read_memory_file(payload).await?)
}
.await,
),
"ai.write_memory_file" => Some(
async move {
let payload: crate::openhuman::memory::WriteMemoryFileRequest =
parse_params(params)?;
rpc_json(crate::openhuman::memory::rpc::ai_write_memory_file(payload).await?)
}
.await,
),
"memory.namespace.list" => Some(
async move { rpc_json(crate::openhuman::memory::rpc::namespace_list().await?) }.await,
),