mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-27 21:08:00 +00:00
Merge pull request #112 from sanil-23/issue-69-memory-rpc
Add core memory RPC surface (#69)
This commit is contained in:
Generated
+561
-31
File diff suppressed because it is too large
Load Diff
@@ -36,6 +36,8 @@ anyhow = "1.0"
|
||||
async-trait = "0.1"
|
||||
chacha20poly1305 = "0.10"
|
||||
hex = "0.4"
|
||||
fastembed = { version = "5.13", default-features = false, features = ["hf-hub-native-tls", "ort-load-dynamic"] }
|
||||
ort = { version = "=2.0.0-rc.11", default-features = false, features = ["std", "ndarray", "load-dynamic"] }
|
||||
tokio-util = { version = "0.7", features = ["rt"] }
|
||||
tokio-tungstenite = { version = "0.24", features = ["rustls-tls-webpki-roots"] }
|
||||
futures = "0.3"
|
||||
@@ -61,6 +63,8 @@ dialoguer = { version = "0.12", features = ["fuzzy-select"] }
|
||||
console = "0.16"
|
||||
glob = "0.3"
|
||||
regex = "1.10"
|
||||
ndarray = "0.17.2"
|
||||
tokenizers = "0.21.0"
|
||||
hostname = "0.4.2"
|
||||
rustyline = { version = "15", features = ["with-file-history"] }
|
||||
rustls = { version = "0.23", features = ["ring"] }
|
||||
|
||||
@@ -476,6 +476,50 @@ mod tests {
|
||||
assert!(err.contains("unknown param 'x'"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invoke_memory_init_missing_required_param_fails() {
|
||||
let err = invoke_method(default_state(), "memory.init", json!({}))
|
||||
.await
|
||||
.expect_err("missing jwt_token should fail");
|
||||
assert!(err.contains("missing field `jwt_token`") || err.contains("jwt_token"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invoke_memory_list_namespaces_rejects_unknown_param() {
|
||||
let err = invoke_method(
|
||||
default_state(),
|
||||
"memory.list_namespaces",
|
||||
json!({ "extra": true }),
|
||||
)
|
||||
.await
|
||||
.expect_err("unknown param should fail");
|
||||
assert!(err.contains("unknown field `extra`") || err.contains("extra"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invoke_memory_query_namespace_missing_namespace_fails() {
|
||||
let err = invoke_method(
|
||||
default_state(),
|
||||
"memory.query_namespace",
|
||||
json!({ "query": "who owns atlas" }),
|
||||
)
|
||||
.await
|
||||
.expect_err("missing namespace should fail");
|
||||
assert!(err.contains("missing field `namespace`") || err.contains("namespace"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invoke_memory_recall_memories_rejects_unknown_param() {
|
||||
let err = invoke_method(
|
||||
default_state(),
|
||||
"memory.recall_memories",
|
||||
json!({ "namespace": "team", "extra": true }),
|
||||
)
|
||||
.await
|
||||
.expect_err("unknown param should fail");
|
||||
assert!(err.contains("unknown field `extra`") || err.contains("extra"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invoke_migrate_openclaw_rejects_unknown_param() {
|
||||
let err = invoke_method(
|
||||
|
||||
@@ -109,7 +109,7 @@ fn default_true() -> bool {
|
||||
}
|
||||
|
||||
fn default_embedding_provider() -> String {
|
||||
"none".into()
|
||||
"fastembed".into()
|
||||
}
|
||||
fn default_hygiene_enabled() -> bool {
|
||||
true
|
||||
@@ -124,10 +124,10 @@ fn default_conversation_retention_days() -> u32 {
|
||||
30
|
||||
}
|
||||
fn default_embedding_model() -> String {
|
||||
"text-embedding-3-small".into()
|
||||
"BGESmallENV15".into()
|
||||
}
|
||||
fn default_embedding_dims() -> usize {
|
||||
1536
|
||||
384
|
||||
}
|
||||
fn default_vector_weight() -> f64 {
|
||||
0.7
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
use async_trait::async_trait;
|
||||
use parking_lot::Mutex;
|
||||
use std::env;
|
||||
use std::path::PathBuf;
|
||||
use std::str::FromStr;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub const DEFAULT_FASTEMBED_MODEL: &str = "BGESmallENV15";
|
||||
pub const DEFAULT_FASTEMBED_DIMENSIONS: usize = 384;
|
||||
const DEFAULT_MANAGED_RELEX_DIR: &str = ".openhuman/models/gliner-relex-large-v0.5-onnx";
|
||||
|
||||
/// Trait for embedding providers — convert text to vectors
|
||||
#[async_trait]
|
||||
@@ -40,6 +49,183 @@ impl EmbeddingProvider for NoopEmbedding {
|
||||
}
|
||||
}
|
||||
|
||||
enum FastembedState {
|
||||
Uninitialized,
|
||||
Ready(fastembed::TextEmbedding),
|
||||
Failed(String),
|
||||
}
|
||||
|
||||
pub struct FastembedEmbedding {
|
||||
model: String,
|
||||
dims: usize,
|
||||
state: Arc<Mutex<FastembedState>>,
|
||||
}
|
||||
|
||||
impl FastembedEmbedding {
|
||||
pub fn new(model: &str, dims: usize) -> Self {
|
||||
Self {
|
||||
model: if model.trim().is_empty() {
|
||||
DEFAULT_FASTEMBED_MODEL.to_string()
|
||||
} else {
|
||||
model.trim().to_string()
|
||||
},
|
||||
dims: if dims == 0 {
|
||||
DEFAULT_FASTEMBED_DIMENSIONS
|
||||
} else {
|
||||
dims
|
||||
},
|
||||
state: Arc::new(Mutex::new(FastembedState::Uninitialized)),
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_model(&self) -> fastembed::EmbeddingModel {
|
||||
fastembed::EmbeddingModel::from_str(&self.model)
|
||||
.unwrap_or(fastembed::EmbeddingModel::BGESmallENV15)
|
||||
}
|
||||
|
||||
fn init_model(&self) -> anyhow::Result<fastembed::TextEmbedding> {
|
||||
ensure_fastembed_ort_dylib_path();
|
||||
fastembed::TextEmbedding::try_new(
|
||||
fastembed::InitOptions::new(self.resolve_model()).with_show_download_progress(false),
|
||||
)
|
||||
.map_err(|e| anyhow::anyhow!("fastembed init failed for {}: {e}", self.model))
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_fastembed_ort_dylib_path() {
|
||||
if env::var_os("ORT_DYLIB_PATH").is_some() {
|
||||
return;
|
||||
}
|
||||
|
||||
if let Some(lib_path) = env::var_os("ORT_LIB_LOCATION") {
|
||||
let candidate = PathBuf::from(lib_path);
|
||||
if candidate.is_file() {
|
||||
env::set_var("ORT_DYLIB_PATH", candidate);
|
||||
return;
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
let runtime_lib = candidate.join("onnxruntime.dll");
|
||||
#[cfg(target_os = "macos")]
|
||||
let runtime_lib = candidate.join("libonnxruntime.dylib");
|
||||
#[cfg(target_os = "linux")]
|
||||
let runtime_lib = candidate.join("libonnxruntime.so");
|
||||
|
||||
if runtime_lib.exists() {
|
||||
env::set_var("ORT_DYLIB_PATH", runtime_lib);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(path) = env::var("OPENHUMAN_GLINER_RELEX_CACHE_DIR") {
|
||||
#[cfg(target_os = "windows")]
|
||||
let bundled = PathBuf::from(path).join("onnxruntime.dll");
|
||||
#[cfg(target_os = "macos")]
|
||||
let bundled = PathBuf::from(path).join("libonnxruntime.dylib");
|
||||
#[cfg(target_os = "linux")]
|
||||
let bundled = PathBuf::from(path).join("libonnxruntime.so");
|
||||
|
||||
if bundled.exists() {
|
||||
env::set_var("ORT_DYLIB_PATH", bundled);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(user_dirs) = directories::UserDirs::new() {
|
||||
#[cfg(target_os = "windows")]
|
||||
let bundled = user_dirs
|
||||
.home_dir()
|
||||
.join(DEFAULT_MANAGED_RELEX_DIR)
|
||||
.join("onnxruntime.dll");
|
||||
#[cfg(target_os = "macos")]
|
||||
let bundled = user_dirs
|
||||
.home_dir()
|
||||
.join(DEFAULT_MANAGED_RELEX_DIR)
|
||||
.join("libonnxruntime.dylib");
|
||||
#[cfg(target_os = "linux")]
|
||||
let bundled = user_dirs
|
||||
.home_dir()
|
||||
.join(DEFAULT_MANAGED_RELEX_DIR)
|
||||
.join("libonnxruntime.so");
|
||||
|
||||
if bundled.exists() {
|
||||
env::set_var("ORT_DYLIB_PATH", bundled);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
for candidate in [
|
||||
"/usr/lib/x86_64-linux-gnu/libonnxruntime.so",
|
||||
"/usr/local/lib/libonnxruntime.so",
|
||||
"/usr/lib/libonnxruntime.so",
|
||||
] {
|
||||
let candidate = PathBuf::from(candidate);
|
||||
if candidate.exists() {
|
||||
env::set_var("ORT_DYLIB_PATH", candidate);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl EmbeddingProvider for FastembedEmbedding {
|
||||
fn name(&self) -> &str {
|
||||
"fastembed"
|
||||
}
|
||||
|
||||
fn dimensions(&self) -> usize {
|
||||
self.dims
|
||||
}
|
||||
|
||||
async fn embed(&self, texts: &[&str]) -> anyhow::Result<Vec<Vec<f32>>> {
|
||||
if texts.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let items = texts
|
||||
.iter()
|
||||
.map(|text| (*text).to_string())
|
||||
.collect::<Vec<_>>();
|
||||
let state = Arc::clone(&self.state);
|
||||
let provider = self.model.clone();
|
||||
let join_result = tokio::task::spawn_blocking(move || {
|
||||
ensure_fastembed_ort_dylib_path();
|
||||
let mut guard = state.lock();
|
||||
if matches!(*guard, FastembedState::Uninitialized) {
|
||||
match fastembed::TextEmbedding::try_new(
|
||||
fastembed::InitOptions::new(
|
||||
fastembed::EmbeddingModel::from_str(&provider)
|
||||
.unwrap_or(fastembed::EmbeddingModel::BGESmallENV15),
|
||||
)
|
||||
.with_show_download_progress(false),
|
||||
) {
|
||||
Ok(model) => *guard = FastembedState::Ready(model),
|
||||
Err(err) => {
|
||||
let message = format!("fastembed init failed for {provider}: {err}");
|
||||
*guard = FastembedState::Failed(message.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match &mut *guard {
|
||||
FastembedState::Ready(model) => model
|
||||
.embed(items, None)
|
||||
.map_err(|e| anyhow::anyhow!("fastembed embed failed: {e}")),
|
||||
FastembedState::Failed(message) => Err(anyhow::anyhow!(message.clone())),
|
||||
FastembedState::Uninitialized => {
|
||||
Err(anyhow::anyhow!("fastembed provider did not initialize"))
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
|
||||
join_result.map_err(|e| anyhow::anyhow!("fastembed task join failed: {e}"))?
|
||||
}
|
||||
}
|
||||
|
||||
// ── OpenAI-compatible embedding provider ─────────────────────
|
||||
|
||||
pub struct OpenAiEmbedding {
|
||||
@@ -163,6 +349,7 @@ pub fn create_embedding_provider(
|
||||
dims: usize,
|
||||
) -> Box<dyn EmbeddingProvider> {
|
||||
match provider {
|
||||
"fastembed" => Box::new(FastembedEmbedding::new(model, dims)),
|
||||
"openai" => {
|
||||
let key = api_key.unwrap_or("");
|
||||
Box::new(OpenAiEmbedding::new(
|
||||
@@ -181,6 +368,13 @@ pub fn create_embedding_provider(
|
||||
}
|
||||
}
|
||||
|
||||
pub fn default_local_embedding_provider() -> Arc<dyn EmbeddingProvider> {
|
||||
Arc::new(FastembedEmbedding::new(
|
||||
DEFAULT_FASTEMBED_MODEL,
|
||||
DEFAULT_FASTEMBED_DIMENSIONS,
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -212,6 +406,13 @@ mod tests {
|
||||
assert_eq!(p.dimensions(), 1536);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn factory_fastembed() {
|
||||
let p = create_embedding_provider("fastembed", None, DEFAULT_FASTEMBED_MODEL, 384);
|
||||
assert_eq!(p.name(), "fastembed");
|
||||
assert_eq!(p.dimensions(), 384);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn factory_custom_url() {
|
||||
let p = create_embedding_provider("custom:http://localhost:1234", None, "model", 768);
|
||||
@@ -255,6 +456,13 @@ mod tests {
|
||||
assert_eq!(p.name(), "none");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_local_provider_uses_fastembed_defaults() {
|
||||
let p = default_local_embedding_provider();
|
||||
assert_eq!(p.name(), "fastembed");
|
||||
assert_eq!(p.dimensions(), DEFAULT_FASTEMBED_DIMENSIONS);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn factory_custom_empty_url() {
|
||||
// "custom:" with no URL — should still construct without panic
|
||||
|
||||
@@ -1,13 +1,17 @@
|
||||
pub mod chunker;
|
||||
pub mod embeddings;
|
||||
pub mod ops;
|
||||
pub mod rpc_models;
|
||||
pub mod store;
|
||||
pub mod traits;
|
||||
|
||||
pub use ops as rpc;
|
||||
pub use ops::*;
|
||||
pub use rpc_models::*;
|
||||
pub use store::{
|
||||
create_memory, create_memory_for_migration, create_memory_with_storage,
|
||||
create_memory_with_storage_and_routes, effective_memory_backend_name, MemoryClient,
|
||||
MemoryClientRef, MemoryState, NamespaceDocumentInput, NamespaceQueryResult, UnifiedMemory,
|
||||
MemoryClientRef, MemoryItemKind, MemoryState, NamespaceDocumentInput, NamespaceMemoryHit,
|
||||
NamespaceQueryResult, NamespaceRetrievalContext, RetrievalScoreBreakdown, UnifiedMemory,
|
||||
};
|
||||
pub use traits::{Memory, MemoryCategory, MemoryEntry};
|
||||
|
||||
+764
-19
@@ -1,6 +1,431 @@
|
||||
use crate::openhuman::memory::{MemoryClient, NamespaceDocumentInput};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
|
||||
use crate::openhuman::config::Config;
|
||||
use crate::openhuman::memory::store::GraphRelationRecord;
|
||||
use crate::openhuman::memory::{
|
||||
ApiEnvelope, ApiError, ApiMeta, DeleteDocumentRequest, DeleteDocumentResponse, EmptyRequest,
|
||||
ListDocumentsRequest, ListDocumentsResponse, ListMemoryFilesRequest, ListMemoryFilesResponse,
|
||||
ListNamespacesResponse, MemoryClient, MemoryClientRef, MemoryDocumentSummary,
|
||||
MemoryInitRequest, MemoryInitResponse, MemoryItemKind, MemoryRecallItem, MemoryRetrievalChunk,
|
||||
MemoryRetrievalContext, MemoryRetrievalEntity, MemoryRetrievalRelation, NamespaceDocumentInput,
|
||||
NamespaceMemoryHit, NamespaceRetrievalContext, PaginationMeta, QueryNamespaceRequest,
|
||||
QueryNamespaceResponse, ReadMemoryFileRequest, ReadMemoryFileResponse, RecallContextRequest,
|
||||
RecallContextResponse, RecallMemoriesRequest, RecallMemoriesResponse, WriteMemoryFileRequest,
|
||||
WriteMemoryFileResponse,
|
||||
};
|
||||
use crate::rpc::RpcOutcome;
|
||||
use chrono::TimeZone;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
static MEMORY_CLIENT_STATE: OnceLock<Mutex<Option<MemoryClientRef>>> = OnceLock::new();
|
||||
|
||||
fn memory_client_state() -> &'static Mutex<Option<MemoryClientRef>> {
|
||||
MEMORY_CLIENT_STATE.get_or_init(|| Mutex::new(None))
|
||||
}
|
||||
|
||||
fn lock_memory_client_state(
|
||||
) -> Result<std::sync::MutexGuard<'static, Option<MemoryClientRef>>, String> {
|
||||
memory_client_state()
|
||||
.lock()
|
||||
.map_err(|_| "memory client state lock poisoned".to_string())
|
||||
}
|
||||
|
||||
fn memory_request_id() -> String {
|
||||
uuid::Uuid::new_v4().to_string()
|
||||
}
|
||||
|
||||
fn memory_counts(
|
||||
entries: impl IntoIterator<Item = (&'static str, usize)>,
|
||||
) -> BTreeMap<String, usize> {
|
||||
entries
|
||||
.into_iter()
|
||||
.map(|(key, value)| (key.to_string(), value))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn envelope<T: Serialize>(
|
||||
data: T,
|
||||
counts: Option<BTreeMap<String, usize>>,
|
||||
pagination: Option<PaginationMeta>,
|
||||
) -> RpcOutcome<ApiEnvelope<T>> {
|
||||
RpcOutcome::new(
|
||||
ApiEnvelope {
|
||||
data: Some(data),
|
||||
error: None,
|
||||
meta: ApiMeta {
|
||||
request_id: memory_request_id(),
|
||||
latency_seconds: None,
|
||||
cached: None,
|
||||
counts,
|
||||
pagination,
|
||||
},
|
||||
},
|
||||
vec![],
|
||||
)
|
||||
}
|
||||
|
||||
fn error_envelope<T: Serialize>(code: &str, message: String) -> RpcOutcome<ApiEnvelope<T>> {
|
||||
RpcOutcome::new(
|
||||
ApiEnvelope {
|
||||
data: None,
|
||||
error: Some(ApiError {
|
||||
code: code.to_string(),
|
||||
message,
|
||||
details: None,
|
||||
}),
|
||||
meta: ApiMeta {
|
||||
request_id: memory_request_id(),
|
||||
latency_seconds: None,
|
||||
cached: None,
|
||||
counts: None,
|
||||
pagination: None,
|
||||
},
|
||||
},
|
||||
vec![],
|
||||
)
|
||||
}
|
||||
|
||||
fn timestamp_to_rfc3339(timestamp: f64) -> Option<String> {
|
||||
if !timestamp.is_finite() || timestamp < 0.0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let secs = timestamp.trunc() as i64;
|
||||
let nanos = ((timestamp.fract().abs()) * 1_000_000_000.0).round() as u32;
|
||||
chrono::Utc
|
||||
.timestamp_opt(secs, nanos.min(999_999_999))
|
||||
.single()
|
||||
.map(|value| value.to_rfc3339())
|
||||
}
|
||||
|
||||
fn memory_kind_label(kind: &MemoryItemKind) -> &'static str {
|
||||
match kind {
|
||||
MemoryItemKind::Document => "document",
|
||||
MemoryItemKind::Kv => "kv",
|
||||
}
|
||||
}
|
||||
|
||||
fn relation_identity(relation: &GraphRelationRecord) -> String {
|
||||
format!(
|
||||
"{}|{}|{}|{}",
|
||||
relation.namespace.as_deref().unwrap_or("global"),
|
||||
relation.subject.as_str(),
|
||||
relation.predicate.as_str(),
|
||||
relation.object.as_str()
|
||||
)
|
||||
}
|
||||
|
||||
fn relation_metadata(relation: &GraphRelationRecord) -> Value {
|
||||
json!({
|
||||
"namespace": relation.namespace.clone(),
|
||||
"attrs": relation.attrs.clone(),
|
||||
"order_index": relation.order_index,
|
||||
"document_ids": relation.document_ids.clone(),
|
||||
"chunk_ids": relation.chunk_ids.clone(),
|
||||
"updated_at": timestamp_to_rfc3339(relation.updated_at),
|
||||
})
|
||||
}
|
||||
|
||||
fn chunk_metadata(hit: &NamespaceMemoryHit) -> Value {
|
||||
json!({
|
||||
"kind": memory_kind_label(&hit.kind),
|
||||
"namespace": hit.namespace.clone(),
|
||||
"key": hit.key.clone(),
|
||||
"title": hit.title.clone(),
|
||||
"category": hit.category.clone(),
|
||||
"source_type": hit.source_type.clone(),
|
||||
"score_breakdown": {
|
||||
"keyword_relevance": hit.score_breakdown.keyword_relevance,
|
||||
"vector_similarity": hit.score_breakdown.vector_similarity,
|
||||
"graph_relevance": hit.score_breakdown.graph_relevance,
|
||||
"freshness": hit.score_breakdown.freshness,
|
||||
"final_score": hit.score_breakdown.final_score,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn build_retrieval_context(hits: &[NamespaceMemoryHit]) -> MemoryRetrievalContext {
|
||||
let mut entities = BTreeSet::new();
|
||||
let mut relations = BTreeMap::new();
|
||||
let chunks = hits
|
||||
.iter()
|
||||
.map(|hit| {
|
||||
for relation in &hit.supporting_relations {
|
||||
if !relation.subject.trim().is_empty() {
|
||||
entities.insert(relation.subject.clone());
|
||||
}
|
||||
if !relation.object.trim().is_empty() {
|
||||
entities.insert(relation.object.clone());
|
||||
}
|
||||
relations
|
||||
.entry(relation_identity(relation))
|
||||
.or_insert_with(|| MemoryRetrievalRelation {
|
||||
subject: relation.subject.clone(),
|
||||
predicate: relation.predicate.clone(),
|
||||
object: relation.object.clone(),
|
||||
score: None,
|
||||
evidence_count: Some(relation.evidence_count),
|
||||
metadata: relation_metadata(relation),
|
||||
});
|
||||
}
|
||||
|
||||
MemoryRetrievalChunk {
|
||||
chunk_id: hit.chunk_id.clone(),
|
||||
document_id: hit.document_id.clone(),
|
||||
content: hit.content.clone(),
|
||||
score: hit.score,
|
||||
metadata: chunk_metadata(hit),
|
||||
created_at: None,
|
||||
updated_at: timestamp_to_rfc3339(hit.updated_at),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
MemoryRetrievalContext {
|
||||
entities: entities
|
||||
.into_iter()
|
||||
.map(|name| MemoryRetrievalEntity {
|
||||
id: None,
|
||||
name,
|
||||
entity_type: None,
|
||||
score: None,
|
||||
metadata: json!({}),
|
||||
})
|
||||
.collect(),
|
||||
relations: relations.into_values().collect(),
|
||||
chunks,
|
||||
}
|
||||
}
|
||||
|
||||
fn format_llm_context_message(query: Option<&str>, hits: &[NamespaceMemoryHit]) -> Option<String> {
|
||||
if hits.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut parts = Vec::new();
|
||||
if let Some(query) = query {
|
||||
parts.push(format!("Query: {query}"));
|
||||
}
|
||||
|
||||
for hit in hits {
|
||||
let summary = match hit.kind {
|
||||
MemoryItemKind::Document => {
|
||||
let title = hit.title.clone().unwrap_or_else(|| hit.key.clone());
|
||||
format!("{title}: {}", hit.content.trim())
|
||||
}
|
||||
MemoryItemKind::Kv => format!("[kv:{}] {}", hit.key, hit.content.trim()),
|
||||
};
|
||||
parts.push(summary);
|
||||
|
||||
if !hit.supporting_relations.is_empty() {
|
||||
let relations = hit
|
||||
.supporting_relations
|
||||
.iter()
|
||||
.map(|relation| {
|
||||
format!(
|
||||
"{} -[{}]-> {}",
|
||||
relation.subject.as_str(),
|
||||
relation.predicate.as_str(),
|
||||
relation.object.as_str()
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("; ");
|
||||
parts.push(format!("Relations: {relations}"));
|
||||
}
|
||||
}
|
||||
|
||||
Some(parts.join("\n\n"))
|
||||
}
|
||||
|
||||
fn filter_hits_by_document_ids(
|
||||
hits: Vec<NamespaceMemoryHit>,
|
||||
document_ids: Option<&[String]>,
|
||||
) -> Vec<NamespaceMemoryHit> {
|
||||
let Some(document_ids) = document_ids else {
|
||||
return hits;
|
||||
};
|
||||
let allowed = document_ids.iter().cloned().collect::<BTreeSet<_>>();
|
||||
hits.into_iter()
|
||||
.filter(|hit| {
|
||||
hit.document_id
|
||||
.as_ref()
|
||||
.map(|document_id| allowed.contains(document_id))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn maybe_retrieval_context(
|
||||
include_references: bool,
|
||||
context: MemoryRetrievalContext,
|
||||
) -> Option<MemoryRetrievalContext> {
|
||||
if !include_references {
|
||||
return None;
|
||||
}
|
||||
if context.entities.is_empty() && context.relations.is_empty() && context.chunks.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(context)
|
||||
}
|
||||
|
||||
async fn current_workspace_dir() -> Result<PathBuf, String> {
|
||||
Config::load_or_init()
|
||||
.await
|
||||
.map(|config| config.workspace_dir)
|
||||
.map_err(|e| format!("load config: {e}"))
|
||||
}
|
||||
|
||||
async fn active_memory_client() -> Result<MemoryClientRef, String> {
|
||||
if let Some(client) = lock_memory_client_state()?.clone() {
|
||||
return Ok(client);
|
||||
}
|
||||
|
||||
let workspace_dir = current_workspace_dir().await?;
|
||||
Ok(Arc::new(MemoryClient::from_workspace_dir(workspace_dir)?))
|
||||
}
|
||||
|
||||
fn validate_memory_relative_path(path: &str) -> Result<(), String> {
|
||||
let candidate = Path::new(path);
|
||||
if candidate.as_os_str().is_empty() {
|
||||
return Err("memory path must not be empty".to_string());
|
||||
}
|
||||
if candidate.is_absolute() {
|
||||
return Err("absolute paths are not allowed".to_string());
|
||||
}
|
||||
for component in candidate.components() {
|
||||
match component {
|
||||
Component::ParentDir | Component::RootDir | Component::Prefix(_) => {
|
||||
return Err("path traversal is not allowed".to_string());
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn resolve_memory_root() -> Result<PathBuf, String> {
|
||||
let workspace_dir = current_workspace_dir().await?;
|
||||
let memory_root = workspace_dir.join("memory");
|
||||
std::fs::create_dir_all(&memory_root)
|
||||
.map_err(|e| format!("create memory dir {}: {e}", memory_root.display()))?;
|
||||
memory_root
|
||||
.canonicalize()
|
||||
.map_err(|e| format!("resolve memory dir {}: {e}", memory_root.display()))
|
||||
}
|
||||
|
||||
async fn resolve_existing_memory_path(relative_path: &str) -> Result<PathBuf, String> {
|
||||
validate_memory_relative_path(relative_path)?;
|
||||
let memory_root = resolve_memory_root().await?;
|
||||
let workspace_root = memory_root
|
||||
.parent()
|
||||
.ok_or_else(|| "memory root is missing a parent workspace".to_string())?;
|
||||
let full_path = workspace_root.join(relative_path);
|
||||
let resolved = full_path
|
||||
.canonicalize()
|
||||
.map_err(|e| format!("resolve memory path {}: {e}", full_path.display()))?;
|
||||
if !resolved.starts_with(&memory_root) {
|
||||
return Err("memory path escapes the memory directory".to_string());
|
||||
}
|
||||
Ok(resolved)
|
||||
}
|
||||
|
||||
async fn resolve_writable_memory_path(relative_path: &str) -> Result<PathBuf, String> {
|
||||
validate_memory_relative_path(relative_path)?;
|
||||
let memory_root = resolve_memory_root().await?;
|
||||
let workspace_root = memory_root
|
||||
.parent()
|
||||
.ok_or_else(|| "memory root is missing a parent workspace".to_string())?;
|
||||
let full_path = workspace_root.join(relative_path);
|
||||
let parent = full_path
|
||||
.parent()
|
||||
.ok_or_else(|| "memory path must include a file name".to_string())?;
|
||||
std::fs::create_dir_all(parent)
|
||||
.map_err(|e| format!("create memory path {}: {e}", parent.display()))?;
|
||||
let resolved_parent = parent
|
||||
.canonicalize()
|
||||
.map_err(|e| format!("resolve memory parent {}: {e}", parent.display()))?;
|
||||
if !resolved_parent.starts_with(&memory_root) {
|
||||
return Err("memory path escapes the memory directory".to_string());
|
||||
}
|
||||
let file_name = full_path
|
||||
.file_name()
|
||||
.ok_or_else(|| "memory path must include a file name".to_string())?;
|
||||
let resolved = resolved_parent.join(file_name);
|
||||
if let Ok(metadata) = std::fs::symlink_metadata(&resolved) {
|
||||
if metadata.file_type().is_symlink() {
|
||||
return Err(format!(
|
||||
"refusing to write through symlink: {}",
|
||||
resolved.display()
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(resolved)
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct RawMemoryDocumentSummary {
|
||||
document_id: String,
|
||||
namespace: String,
|
||||
key: String,
|
||||
title: String,
|
||||
source_type: String,
|
||||
priority: String,
|
||||
created_at: f64,
|
||||
updated_at: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct RawDeleteDocumentResult {
|
||||
deleted: bool,
|
||||
namespace: String,
|
||||
document_id: String,
|
||||
}
|
||||
|
||||
fn parse_memory_document_summaries(raw: Value) -> Result<Vec<MemoryDocumentSummary>, String> {
|
||||
let documents = raw
|
||||
.get("documents")
|
||||
.and_then(Value::as_array)
|
||||
.ok_or_else(|| "memory document list missing 'documents' array".to_string())?;
|
||||
documents
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|value| {
|
||||
let raw: RawMemoryDocumentSummary = serde_json::from_value(value)
|
||||
.map_err(|e| format!("decode memory document: {e}"))?;
|
||||
Ok(MemoryDocumentSummary {
|
||||
document_id: raw.document_id,
|
||||
namespace: raw.namespace,
|
||||
key: raw.key,
|
||||
title: raw.title,
|
||||
source_type: raw.source_type,
|
||||
priority: raw.priority,
|
||||
created_at: raw.created_at,
|
||||
updated_at: raw.updated_at,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn query_limit_for_request(
|
||||
client: &MemoryClient,
|
||||
request: &QueryNamespaceRequest,
|
||||
) -> Result<u32, String> {
|
||||
let requested = request.resolved_limit();
|
||||
if request.document_ids.is_none() {
|
||||
return Ok(requested);
|
||||
}
|
||||
|
||||
let raw = client.list_documents(Some(&request.namespace)).await?;
|
||||
let documents = parse_memory_document_summaries(raw)?;
|
||||
let total_documents = u32::try_from(documents.len()).unwrap_or(u32::MAX);
|
||||
Ok(requested.max(total_documents))
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct PutDocParams {
|
||||
@@ -29,12 +454,6 @@ pub struct NamespaceOnlyParams {
|
||||
pub namespace: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct OptionalNamespaceParams {
|
||||
#[serde(default)]
|
||||
pub namespace: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct DeleteDocParams {
|
||||
pub namespace: String,
|
||||
@@ -110,7 +529,7 @@ fn default_category() -> String {
|
||||
}
|
||||
|
||||
pub async fn namespace_list() -> Result<RpcOutcome<Vec<String>>, String> {
|
||||
let client = MemoryClient::new_local()?;
|
||||
let client = active_memory_client().await?;
|
||||
let namespaces = client.list_namespaces().await?;
|
||||
Ok(RpcOutcome::single_log(
|
||||
namespaces,
|
||||
@@ -119,7 +538,7 @@ pub async fn namespace_list() -> Result<RpcOutcome<Vec<String>>, String> {
|
||||
}
|
||||
|
||||
pub async fn doc_put(params: PutDocParams) -> Result<RpcOutcome<PutDocResult>, String> {
|
||||
let client = MemoryClient::new_local()?;
|
||||
let client = active_memory_client().await?;
|
||||
let document_id = client
|
||||
.put_doc(NamespaceDocumentInput {
|
||||
namespace: params.namespace,
|
||||
@@ -144,7 +563,7 @@ pub async fn doc_put(params: PutDocParams) -> Result<RpcOutcome<PutDocResult>, S
|
||||
pub async fn doc_list(
|
||||
params: Option<NamespaceOnlyParams>,
|
||||
) -> Result<RpcOutcome<serde_json::Value>, String> {
|
||||
let client = MemoryClient::new_local()?;
|
||||
let client = active_memory_client().await?;
|
||||
let docs = client
|
||||
.list_documents(params.as_ref().map(|v| v.namespace.as_str()))
|
||||
.await?;
|
||||
@@ -152,7 +571,7 @@ pub async fn doc_list(
|
||||
}
|
||||
|
||||
pub async fn doc_delete(params: DeleteDocParams) -> Result<RpcOutcome<serde_json::Value>, String> {
|
||||
let client = MemoryClient::new_local()?;
|
||||
let client = active_memory_client().await?;
|
||||
let result = client
|
||||
.delete_document(¶ms.namespace, ¶ms.document_id)
|
||||
.await?;
|
||||
@@ -160,7 +579,7 @@ pub async fn doc_delete(params: DeleteDocParams) -> Result<RpcOutcome<serde_json
|
||||
}
|
||||
|
||||
pub async fn context_query(params: QueryNamespaceParams) -> Result<RpcOutcome<String>, String> {
|
||||
let client = MemoryClient::new_local()?;
|
||||
let client = active_memory_client().await?;
|
||||
let result = client
|
||||
.query_namespace(¶ms.namespace, ¶ms.query, params.limit.unwrap_or(10))
|
||||
.await?;
|
||||
@@ -170,7 +589,7 @@ pub async fn context_query(params: QueryNamespaceParams) -> Result<RpcOutcome<St
|
||||
pub async fn context_recall(
|
||||
params: RecallNamespaceParams,
|
||||
) -> Result<RpcOutcome<Option<String>>, String> {
|
||||
let client = MemoryClient::new_local()?;
|
||||
let client = active_memory_client().await?;
|
||||
let result = client
|
||||
.recall_namespace(¶ms.namespace, params.limit.unwrap_or(10))
|
||||
.await?;
|
||||
@@ -178,7 +597,7 @@ pub async fn context_recall(
|
||||
}
|
||||
|
||||
pub async fn kv_set(params: KvSetParams) -> Result<RpcOutcome<bool>, String> {
|
||||
let client = MemoryClient::new_local()?;
|
||||
let client = active_memory_client().await?;
|
||||
client
|
||||
.kv_set(params.namespace.as_deref(), ¶ms.key, ¶ms.value)
|
||||
.await?;
|
||||
@@ -188,7 +607,7 @@ pub async fn kv_set(params: KvSetParams) -> Result<RpcOutcome<bool>, String> {
|
||||
pub async fn kv_get(
|
||||
params: KvGetDeleteParams,
|
||||
) -> Result<RpcOutcome<Option<serde_json::Value>>, String> {
|
||||
let client = MemoryClient::new_local()?;
|
||||
let client = active_memory_client().await?;
|
||||
let value = client
|
||||
.kv_get(params.namespace.as_deref(), ¶ms.key)
|
||||
.await?;
|
||||
@@ -196,7 +615,7 @@ pub async fn kv_get(
|
||||
}
|
||||
|
||||
pub async fn kv_delete(params: KvGetDeleteParams) -> Result<RpcOutcome<bool>, String> {
|
||||
let client = MemoryClient::new_local()?;
|
||||
let client = active_memory_client().await?;
|
||||
let deleted = client
|
||||
.kv_delete(params.namespace.as_deref(), ¶ms.key)
|
||||
.await?;
|
||||
@@ -206,13 +625,13 @@ pub async fn kv_delete(params: KvGetDeleteParams) -> Result<RpcOutcome<bool>, St
|
||||
pub async fn kv_list_namespace(
|
||||
params: NamespaceOnlyParams,
|
||||
) -> Result<RpcOutcome<Vec<serde_json::Value>>, String> {
|
||||
let client = MemoryClient::new_local()?;
|
||||
let client = active_memory_client().await?;
|
||||
let rows = client.kv_list_namespace(¶ms.namespace).await?;
|
||||
Ok(RpcOutcome::single_log(rows, "memory namespace kv listed"))
|
||||
}
|
||||
|
||||
pub async fn graph_upsert(params: GraphUpsertParams) -> Result<RpcOutcome<bool>, String> {
|
||||
let client = MemoryClient::new_local()?;
|
||||
let client = active_memory_client().await?;
|
||||
client
|
||||
.graph_upsert(
|
||||
params.namespace.as_deref(),
|
||||
@@ -228,7 +647,7 @@ pub async fn graph_upsert(params: GraphUpsertParams) -> Result<RpcOutcome<bool>,
|
||||
pub async fn graph_query(
|
||||
params: GraphQueryParams,
|
||||
) -> Result<RpcOutcome<Vec<serde_json::Value>>, String> {
|
||||
let client = MemoryClient::new_local()?;
|
||||
let client = active_memory_client().await?;
|
||||
let rows = client
|
||||
.graph_query(
|
||||
params.namespace.as_deref(),
|
||||
@@ -238,3 +657,329 @@ pub async fn graph_query(
|
||||
.await?;
|
||||
Ok(RpcOutcome::single_log(rows, "memory graph queried"))
|
||||
}
|
||||
|
||||
pub async fn memory_init(
|
||||
request: MemoryInitRequest,
|
||||
) -> Result<RpcOutcome<ApiEnvelope<MemoryInitResponse>>, String> {
|
||||
if request.jwt_token.trim().is_empty() {
|
||||
return Err("jwt_token must not be empty".to_string());
|
||||
}
|
||||
let workspace_dir = current_workspace_dir().await?;
|
||||
let client = Arc::new(MemoryClient::from_workspace_dir(workspace_dir.clone())?);
|
||||
*lock_memory_client_state()? = Some(client);
|
||||
let memory_dir = workspace_dir.join("memory");
|
||||
Ok(envelope(
|
||||
MemoryInitResponse {
|
||||
initialized: true,
|
||||
workspace_dir: workspace_dir.display().to_string(),
|
||||
memory_dir: memory_dir.display().to_string(),
|
||||
},
|
||||
None,
|
||||
None,
|
||||
))
|
||||
}
|
||||
|
||||
pub async fn memory_list_documents(
|
||||
request: ListDocumentsRequest,
|
||||
) -> Result<RpcOutcome<ApiEnvelope<ListDocumentsResponse>>, String> {
|
||||
let client = active_memory_client().await?;
|
||||
let raw = client.list_documents(request.namespace.as_deref()).await?;
|
||||
let documents = parse_memory_document_summaries(raw)?;
|
||||
let count = documents.len();
|
||||
Ok(envelope(
|
||||
ListDocumentsResponse {
|
||||
namespace: request.namespace,
|
||||
documents,
|
||||
count,
|
||||
},
|
||||
Some(memory_counts([("num_documents", count)])),
|
||||
Some(PaginationMeta {
|
||||
limit: count,
|
||||
offset: 0,
|
||||
count,
|
||||
}),
|
||||
))
|
||||
}
|
||||
|
||||
pub async fn memory_list_namespaces(
|
||||
_request: EmptyRequest,
|
||||
) -> Result<RpcOutcome<ApiEnvelope<ListNamespacesResponse>>, String> {
|
||||
let client = active_memory_client().await?;
|
||||
let namespaces = client.list_namespaces().await?;
|
||||
let count = namespaces.len();
|
||||
Ok(envelope(
|
||||
ListNamespacesResponse { namespaces, count },
|
||||
Some(memory_counts([("num_namespaces", count)])),
|
||||
None,
|
||||
))
|
||||
}
|
||||
|
||||
pub async fn memory_delete_document(
|
||||
request: DeleteDocumentRequest,
|
||||
) -> Result<RpcOutcome<ApiEnvelope<DeleteDocumentResponse>>, String> {
|
||||
let client = active_memory_client().await?;
|
||||
let raw = client
|
||||
.delete_document(&request.namespace, &request.document_id)
|
||||
.await?;
|
||||
let parsed: RawDeleteDocumentResult =
|
||||
serde_json::from_value(raw).map_err(|e| format!("decode delete document result: {e}"))?;
|
||||
Ok(envelope(
|
||||
DeleteDocumentResponse {
|
||||
status: if parsed.deleted {
|
||||
"completed".to_string()
|
||||
} else {
|
||||
"not_found".to_string()
|
||||
},
|
||||
namespace: parsed.namespace,
|
||||
document_id: parsed.document_id,
|
||||
deleted: parsed.deleted,
|
||||
},
|
||||
None,
|
||||
None,
|
||||
))
|
||||
}
|
||||
|
||||
pub async fn memory_query_namespace(
|
||||
request: QueryNamespaceRequest,
|
||||
) -> Result<RpcOutcome<ApiEnvelope<QueryNamespaceResponse>>, String> {
|
||||
let include_references = request.include_references.unwrap_or(true);
|
||||
let result = async {
|
||||
let client = active_memory_client().await?;
|
||||
let retrieval_limit = query_limit_for_request(client.as_ref(), &request).await?;
|
||||
let mut context = client
|
||||
.query_namespace_context_data(&request.namespace, &request.query, retrieval_limit)
|
||||
.await?;
|
||||
context.hits = filter_hits_by_document_ids(context.hits, request.document_ids.as_deref());
|
||||
Ok::<NamespaceRetrievalContext, String>(context)
|
||||
}
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(context) => {
|
||||
let retrieval_context = build_retrieval_context(&context.hits);
|
||||
let counts = memory_counts([
|
||||
("num_entities", retrieval_context.entities.len()),
|
||||
("num_relations", retrieval_context.relations.len()),
|
||||
("num_chunks", retrieval_context.chunks.len()),
|
||||
]);
|
||||
let llm_context_message =
|
||||
format_llm_context_message(Some(&request.query), &context.hits);
|
||||
Ok(envelope(
|
||||
QueryNamespaceResponse {
|
||||
context: maybe_retrieval_context(include_references, retrieval_context),
|
||||
llm_context_message,
|
||||
},
|
||||
Some(counts),
|
||||
None,
|
||||
))
|
||||
}
|
||||
Err(message) => Ok(error_envelope("memory.query_namespace_failed", message)),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn memory_recall_context(
|
||||
request: RecallContextRequest,
|
||||
) -> Result<RpcOutcome<ApiEnvelope<RecallContextResponse>>, String> {
|
||||
let include_references = request.include_references.unwrap_or(true);
|
||||
let result = async {
|
||||
let client = active_memory_client().await?;
|
||||
client
|
||||
.recall_namespace_context_data(&request.namespace, request.resolved_limit())
|
||||
.await
|
||||
}
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(context) => {
|
||||
let retrieval_context = build_retrieval_context(&context.hits);
|
||||
let counts = memory_counts([
|
||||
("num_entities", retrieval_context.entities.len()),
|
||||
("num_relations", retrieval_context.relations.len()),
|
||||
("num_chunks", retrieval_context.chunks.len()),
|
||||
]);
|
||||
let llm_context_message = format_llm_context_message(None, &context.hits);
|
||||
Ok(envelope(
|
||||
RecallContextResponse {
|
||||
context: maybe_retrieval_context(include_references, retrieval_context),
|
||||
llm_context_message,
|
||||
},
|
||||
Some(counts),
|
||||
None,
|
||||
))
|
||||
}
|
||||
Err(message) => Ok(error_envelope("memory.recall_context_failed", message)),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn memory_recall_memories(
|
||||
request: RecallMemoriesRequest,
|
||||
) -> Result<RpcOutcome<ApiEnvelope<RecallMemoriesResponse>>, String> {
|
||||
let result = async {
|
||||
let client = active_memory_client().await?;
|
||||
client
|
||||
.recall_namespace_memories(&request.namespace, request.resolved_limit())
|
||||
.await
|
||||
}
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(hits) => {
|
||||
let memories = hits
|
||||
.into_iter()
|
||||
.map(|hit| MemoryRecallItem {
|
||||
kind: memory_kind_label(&hit.kind).to_string(),
|
||||
id: hit.id,
|
||||
content: hit.content,
|
||||
score: hit.score,
|
||||
retention: None,
|
||||
last_accessed_at: None,
|
||||
access_count: None,
|
||||
stability_days: None,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let count = memories.len();
|
||||
Ok(envelope(
|
||||
RecallMemoriesResponse { memories },
|
||||
Some(memory_counts([("num_memories", count)])),
|
||||
None,
|
||||
))
|
||||
}
|
||||
Err(message) => Ok(error_envelope("memory.recall_memories_failed", message)),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn ai_list_memory_files(
|
||||
request: ListMemoryFilesRequest,
|
||||
) -> Result<RpcOutcome<ApiEnvelope<ListMemoryFilesResponse>>, String> {
|
||||
validate_memory_relative_path(&request.relative_dir)?;
|
||||
let directory = resolve_existing_memory_path(&request.relative_dir).await?;
|
||||
if !directory.is_dir() {
|
||||
return Err(format!(
|
||||
"memory directory not found: {}",
|
||||
directory.display()
|
||||
));
|
||||
}
|
||||
let mut files = Vec::new();
|
||||
for entry in std::fs::read_dir(&directory)
|
||||
.map_err(|e| format!("read memory directory {}: {e}", directory.display()))?
|
||||
{
|
||||
let entry = entry.map_err(|e| format!("read memory directory entry: {e}"))?;
|
||||
let file_name = entry.file_name();
|
||||
let file_name = file_name.to_string_lossy();
|
||||
if !file_name.is_empty() {
|
||||
files.push(file_name.to_string());
|
||||
}
|
||||
}
|
||||
files.sort();
|
||||
let count = files.len();
|
||||
Ok(envelope(
|
||||
ListMemoryFilesResponse {
|
||||
relative_dir: request.relative_dir,
|
||||
files,
|
||||
count,
|
||||
},
|
||||
Some(memory_counts([("num_files", count)])),
|
||||
None,
|
||||
))
|
||||
}
|
||||
|
||||
pub async fn ai_read_memory_file(
|
||||
request: ReadMemoryFileRequest,
|
||||
) -> Result<RpcOutcome<ApiEnvelope<ReadMemoryFileResponse>>, String> {
|
||||
let path = resolve_existing_memory_path(&request.relative_path).await?;
|
||||
let content = std::fs::read_to_string(&path)
|
||||
.map_err(|e| format!("read memory file {}: {e}", path.display()))?;
|
||||
Ok(envelope(
|
||||
ReadMemoryFileResponse {
|
||||
relative_path: request.relative_path,
|
||||
content,
|
||||
},
|
||||
None,
|
||||
None,
|
||||
))
|
||||
}
|
||||
|
||||
pub async fn ai_write_memory_file(
|
||||
request: WriteMemoryFileRequest,
|
||||
) -> Result<RpcOutcome<ApiEnvelope<WriteMemoryFileResponse>>, String> {
|
||||
let path = resolve_writable_memory_path(&request.relative_path).await?;
|
||||
std::fs::write(&path, request.content.as_bytes())
|
||||
.map_err(|e| format!("write memory file {}: {e}", path.display()))?;
|
||||
let bytes_written = request.content.len();
|
||||
Ok(envelope(
|
||||
WriteMemoryFileResponse {
|
||||
relative_path: request.relative_path,
|
||||
written: true,
|
||||
bytes_written,
|
||||
},
|
||||
None,
|
||||
None,
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::json;
|
||||
|
||||
use super::{build_retrieval_context, filter_hits_by_document_ids, format_llm_context_message};
|
||||
use crate::openhuman::memory::store::GraphRelationRecord;
|
||||
use crate::openhuman::memory::{MemoryItemKind, NamespaceMemoryHit, RetrievalScoreBreakdown};
|
||||
|
||||
fn sample_hit() -> NamespaceMemoryHit {
|
||||
NamespaceMemoryHit {
|
||||
id: "doc-1".to_string(),
|
||||
kind: MemoryItemKind::Document,
|
||||
namespace: "team".to_string(),
|
||||
key: "atlas-status".to_string(),
|
||||
title: Some("Atlas status".to_string()),
|
||||
content: "Project Atlas is owned by Alice.".to_string(),
|
||||
category: "core".to_string(),
|
||||
source_type: Some("doc".to_string()),
|
||||
updated_at: 1_700_000_000.0,
|
||||
score: 0.92,
|
||||
score_breakdown: RetrievalScoreBreakdown {
|
||||
keyword_relevance: 0.3,
|
||||
vector_similarity: 0.4,
|
||||
graph_relevance: 0.9,
|
||||
freshness: 0.0,
|
||||
final_score: 0.92,
|
||||
},
|
||||
document_id: Some("doc-1".to_string()),
|
||||
chunk_id: Some("doc-1#chunk-1".to_string()),
|
||||
supporting_relations: vec![GraphRelationRecord {
|
||||
namespace: Some("team".to_string()),
|
||||
subject: "Alice".to_string(),
|
||||
predicate: "OWNS".to_string(),
|
||||
object: "Atlas".to_string(),
|
||||
attrs: json!({"source": "graph"}),
|
||||
updated_at: 1_700_000_000.0,
|
||||
evidence_count: 2,
|
||||
order_index: Some(1),
|
||||
document_ids: vec!["doc-1".to_string()],
|
||||
chunk_ids: vec!["doc-1#chunk-1".to_string()],
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_retrieval_context_projects_hits_into_relations_and_chunks() {
|
||||
let context = build_retrieval_context(&[sample_hit()]);
|
||||
assert_eq!(context.entities.len(), 2);
|
||||
assert_eq!(context.relations.len(), 1);
|
||||
assert_eq!(context.chunks.len(), 1);
|
||||
assert_eq!(context.chunks[0].document_id.as_deref(), Some("doc-1"));
|
||||
assert_eq!(context.relations[0].predicate, "OWNS");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn helpers_filter_document_ids_and_format_context_message() {
|
||||
let hit = sample_hit();
|
||||
let filtered = filter_hits_by_document_ids(vec![hit.clone()], Some(&["doc-2".to_string()]));
|
||||
assert!(filtered.is_empty());
|
||||
|
||||
let message = format_llm_context_message(Some("who owns atlas"), &[hit])
|
||||
.expect("context message should exist");
|
||||
assert!(message.contains("Query: who owns atlas"));
|
||||
assert!(message.contains("Alice -[OWNS]-> Atlas"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,337 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ApiError {
|
||||
pub code: String,
|
||||
pub message: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub details: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct PaginationMeta {
|
||||
pub limit: usize,
|
||||
pub offset: usize,
|
||||
pub count: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ApiMeta {
|
||||
pub request_id: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub latency_seconds: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cached: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub counts: Option<BTreeMap<String, usize>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub pagination: Option<PaginationMeta>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ApiEnvelope<T> {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub data: Option<T>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<ApiError>,
|
||||
pub meta: ApiMeta,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct EmptyRequest {}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MemoryInitRequest {
|
||||
pub jwt_token: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MemoryInitResponse {
|
||||
pub initialized: bool,
|
||||
pub workspace_dir: String,
|
||||
pub memory_dir: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ListDocumentsRequest {
|
||||
#[serde(default)]
|
||||
pub namespace: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MemoryDocumentSummary {
|
||||
pub document_id: String,
|
||||
pub namespace: String,
|
||||
pub key: String,
|
||||
pub title: String,
|
||||
pub source_type: String,
|
||||
pub priority: String,
|
||||
pub created_at: f64,
|
||||
pub updated_at: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ListDocumentsResponse {
|
||||
#[serde(default)]
|
||||
pub namespace: Option<String>,
|
||||
pub documents: Vec<MemoryDocumentSummary>,
|
||||
pub count: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ListNamespacesResponse {
|
||||
pub namespaces: Vec<String>,
|
||||
pub count: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct DeleteDocumentRequest {
|
||||
pub namespace: String,
|
||||
pub document_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct DeleteDocumentResponse {
|
||||
pub status: String,
|
||||
pub namespace: String,
|
||||
pub document_id: String,
|
||||
pub deleted: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct QueryNamespaceRequest {
|
||||
pub namespace: String,
|
||||
pub query: String,
|
||||
#[serde(default)]
|
||||
pub include_references: Option<bool>,
|
||||
#[serde(default)]
|
||||
pub document_ids: Option<Vec<String>>,
|
||||
#[serde(default)]
|
||||
pub limit: Option<u32>,
|
||||
#[serde(default)]
|
||||
pub max_chunks: Option<u32>,
|
||||
}
|
||||
|
||||
impl QueryNamespaceRequest {
|
||||
pub fn resolved_limit(&self) -> u32 {
|
||||
self.max_chunks.or(self.limit).unwrap_or(10)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct QueryNamespaceResponse {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub context: Option<MemoryRetrievalContext>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub llm_context_message: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RecallContextRequest {
|
||||
pub namespace: String,
|
||||
#[serde(default)]
|
||||
pub include_references: Option<bool>,
|
||||
#[serde(default)]
|
||||
pub limit: Option<u32>,
|
||||
#[serde(default)]
|
||||
pub max_chunks: Option<u32>,
|
||||
}
|
||||
|
||||
impl RecallContextRequest {
|
||||
pub fn resolved_limit(&self) -> u32 {
|
||||
self.max_chunks.or(self.limit).unwrap_or(10)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RecallContextResponse {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub context: Option<MemoryRetrievalContext>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub llm_context_message: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RecallMemoriesRequest {
|
||||
pub namespace: String,
|
||||
/// Accepted for forward compatibility and currently ignored until the core
|
||||
/// recall path exposes real retention-based filtering semantics.
|
||||
#[serde(default)]
|
||||
pub min_retention: Option<f32>,
|
||||
/// Accepted for forward compatibility and currently ignored until the core
|
||||
/// recall path exposes real as-of / temporal recall semantics.
|
||||
#[serde(default)]
|
||||
pub as_of: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub limit: Option<u32>,
|
||||
#[serde(default)]
|
||||
pub max_chunks: Option<u32>,
|
||||
#[serde(default)]
|
||||
pub top_k: Option<u32>,
|
||||
}
|
||||
|
||||
impl RecallMemoriesRequest {
|
||||
pub fn resolved_limit(&self) -> u32 {
|
||||
self.top_k.or(self.max_chunks).or(self.limit).unwrap_or(10)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MemoryRetrievalEntity {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub id: Option<String>,
|
||||
pub name: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub entity_type: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub score: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub metadata: serde_json::Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MemoryRetrievalRelation {
|
||||
pub subject: String,
|
||||
pub predicate: String,
|
||||
pub object: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub score: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub evidence_count: Option<u32>,
|
||||
#[serde(default)]
|
||||
pub metadata: serde_json::Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MemoryRetrievalChunk {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub chunk_id: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub document_id: Option<String>,
|
||||
pub content: String,
|
||||
pub score: f64,
|
||||
#[serde(default)]
|
||||
pub metadata: serde_json::Value,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub created_at: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub updated_at: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MemoryRetrievalContext {
|
||||
pub entities: Vec<MemoryRetrievalEntity>,
|
||||
pub relations: Vec<MemoryRetrievalRelation>,
|
||||
pub chunks: Vec<MemoryRetrievalChunk>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MemoryRecallItem {
|
||||
#[serde(rename = "type")]
|
||||
pub kind: String,
|
||||
pub id: String,
|
||||
pub content: String,
|
||||
pub score: f64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub retention: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub last_accessed_at: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub access_count: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stability_days: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RecallMemoriesResponse {
|
||||
pub memories: Vec<MemoryRecallItem>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ListMemoryFilesRequest {
|
||||
#[serde(default = "default_memory_relative_dir")]
|
||||
pub relative_dir: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ListMemoryFilesResponse {
|
||||
pub relative_dir: String,
|
||||
pub files: Vec<String>,
|
||||
pub count: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ReadMemoryFileRequest {
|
||||
pub relative_path: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ReadMemoryFileResponse {
|
||||
pub relative_path: String,
|
||||
pub content: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct WriteMemoryFileRequest {
|
||||
pub relative_path: String,
|
||||
pub content: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct WriteMemoryFileResponse {
|
||||
pub relative_path: String,
|
||||
pub written: bool,
|
||||
pub bytes_written: usize,
|
||||
}
|
||||
|
||||
fn default_memory_relative_dir() -> String {
|
||||
"memory".to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::json;
|
||||
|
||||
use super::RecallMemoriesRequest;
|
||||
|
||||
#[test]
|
||||
fn recall_memories_request_accepts_compatibility_noop_params() {
|
||||
let request: RecallMemoriesRequest = serde_json::from_value(json!({
|
||||
"namespace": "team",
|
||||
"top_k": 7,
|
||||
"min_retention": 0.8,
|
||||
"as_of": 1700000000.0
|
||||
}))
|
||||
.expect("compatibility params should deserialize");
|
||||
|
||||
assert_eq!(request.namespace, "team");
|
||||
assert_eq!(request.top_k, Some(7));
|
||||
assert_eq!(request.min_retention, Some(0.8));
|
||||
assert_eq!(request.as_of, Some(1_700_000_000.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recall_memories_request_limit_resolution_ignores_compatibility_noop_params() {
|
||||
let request: RecallMemoriesRequest = serde_json::from_value(json!({
|
||||
"namespace": "team",
|
||||
"limit": 3,
|
||||
"min_retention": 0.5,
|
||||
"as_of": 1700000000.0
|
||||
}))
|
||||
.expect("request should deserialize");
|
||||
|
||||
assert_eq!(request.resolved_limit(), 3);
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,11 @@
|
||||
use serde_json::json;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::openhuman::memory::embeddings::{self, EmbeddingProvider};
|
||||
use crate::openhuman::memory::store::types::NamespaceDocumentInput;
|
||||
use crate::openhuman::memory::store::types::{
|
||||
NamespaceDocumentInput, NamespaceMemoryHit, NamespaceRetrievalContext,
|
||||
};
|
||||
use crate::openhuman::memory::store::unified::UnifiedMemory;
|
||||
|
||||
pub type MemoryClientRef = Arc<MemoryClient>;
|
||||
@@ -24,9 +27,13 @@ impl MemoryClient {
|
||||
.ok_or_else(|| "Failed to resolve home directory".to_string())?
|
||||
.join(".openhuman")
|
||||
.join("workspace");
|
||||
Self::from_workspace_dir(workspace_dir)
|
||||
}
|
||||
|
||||
pub fn from_workspace_dir(workspace_dir: PathBuf) -> Result<Self, String> {
|
||||
std::fs::create_dir_all(&workspace_dir)
|
||||
.map_err(|e| format!("Create workspace dir {}: {e}", workspace_dir.display()))?;
|
||||
let embedder: Arc<dyn EmbeddingProvider> = Arc::new(embeddings::NoopEmbedding);
|
||||
let embedder: Arc<dyn EmbeddingProvider> = embeddings::default_local_embedding_provider();
|
||||
let memory =
|
||||
UnifiedMemory::new(&workspace_dir, embedder, None).map_err(|e| format!("{e}"))?;
|
||||
Ok(Self {
|
||||
@@ -121,6 +128,17 @@ impl MemoryClient {
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn query_namespace_context_data(
|
||||
&self,
|
||||
namespace: &str,
|
||||
query: &str,
|
||||
max_chunks: u32,
|
||||
) -> Result<NamespaceRetrievalContext, String> {
|
||||
self.inner
|
||||
.query_namespace_context_data(namespace, query, max_chunks)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn recall_namespace(
|
||||
&self,
|
||||
namespace: &str,
|
||||
@@ -131,6 +149,24 @@ impl MemoryClient {
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn recall_namespace_context_data(
|
||||
&self,
|
||||
namespace: &str,
|
||||
max_chunks: u32,
|
||||
) -> Result<NamespaceRetrievalContext, String> {
|
||||
self.inner
|
||||
.recall_namespace_context_data(namespace, max_chunks)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn recall_namespace_memories(
|
||||
&self,
|
||||
namespace: &str,
|
||||
limit: u32,
|
||||
) -> Result<Vec<NamespaceMemoryHit>, String> {
|
||||
self.inner.recall_namespace_memories(namespace, limit).await
|
||||
}
|
||||
|
||||
pub async fn kv_set(
|
||||
&self,
|
||||
namespace: Option<&str>,
|
||||
|
||||
@@ -70,7 +70,7 @@ impl Memory for UnifiedMemory {
|
||||
category: memory_category_from_stored(&r.category),
|
||||
timestamp: Utc::now().to_rfc3339(),
|
||||
session_id: None,
|
||||
score: Some(r.score * 100.0),
|
||||
score: Some(r.score),
|
||||
})
|
||||
.collect();
|
||||
Ok(out)
|
||||
|
||||
@@ -12,5 +12,9 @@ pub use factories::{
|
||||
create_memory, create_memory_for_migration, create_memory_with_storage,
|
||||
create_memory_with_storage_and_routes, effective_memory_backend_name,
|
||||
};
|
||||
pub use types::{NamespaceDocumentInput, NamespaceQueryResult};
|
||||
pub use types::{
|
||||
GraphRelationRecord, MemoryItemKind, MemoryKvRecord, NamespaceDocumentInput,
|
||||
NamespaceMemoryHit, NamespaceQueryResult, NamespaceRetrievalContext, RetrievalScoreBreakdown,
|
||||
StoredMemoryDocument,
|
||||
};
|
||||
pub use unified::UnifiedMemory;
|
||||
|
||||
@@ -31,3 +31,88 @@ pub struct NamespaceQueryResult {
|
||||
/// Stored category string (e.g. `core`, `daily`, or custom label).
|
||||
pub category: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum MemoryItemKind {
|
||||
Document,
|
||||
Kv,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct StoredMemoryDocument {
|
||||
pub document_id: String,
|
||||
pub namespace: String,
|
||||
pub key: String,
|
||||
pub title: String,
|
||||
pub content: String,
|
||||
pub source_type: String,
|
||||
pub priority: String,
|
||||
pub tags: Vec<String>,
|
||||
pub metadata: serde_json::Value,
|
||||
pub category: String,
|
||||
pub session_id: Option<String>,
|
||||
pub created_at: f64,
|
||||
pub updated_at: f64,
|
||||
pub markdown_rel_path: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MemoryKvRecord {
|
||||
pub namespace: Option<String>,
|
||||
pub key: String,
|
||||
pub value: serde_json::Value,
|
||||
pub updated_at: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GraphRelationRecord {
|
||||
pub namespace: Option<String>,
|
||||
pub subject: String,
|
||||
pub predicate: String,
|
||||
pub object: String,
|
||||
pub attrs: serde_json::Value,
|
||||
pub updated_at: f64,
|
||||
pub evidence_count: u32,
|
||||
pub order_index: Option<i64>,
|
||||
pub document_ids: Vec<String>,
|
||||
pub chunk_ids: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct RetrievalScoreBreakdown {
|
||||
pub keyword_relevance: f64,
|
||||
pub vector_similarity: f64,
|
||||
pub graph_relevance: f64,
|
||||
pub freshness: f64,
|
||||
pub final_score: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct NamespaceMemoryHit {
|
||||
pub id: String,
|
||||
pub kind: MemoryItemKind,
|
||||
pub namespace: String,
|
||||
pub key: String,
|
||||
pub title: Option<String>,
|
||||
pub content: String,
|
||||
pub category: String,
|
||||
pub source_type: Option<String>,
|
||||
pub updated_at: f64,
|
||||
pub score: f64,
|
||||
pub score_breakdown: RetrievalScoreBreakdown,
|
||||
#[serde(default)]
|
||||
pub document_id: Option<String>,
|
||||
#[serde(default)]
|
||||
pub chunk_id: Option<String>,
|
||||
#[serde(default)]
|
||||
pub supporting_relations: Vec<GraphRelationRecord>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct NamespaceRetrievalContext {
|
||||
pub namespace: String,
|
||||
pub query: Option<String>,
|
||||
pub context_text: String,
|
||||
pub hits: Vec<NamespaceMemoryHit>,
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ use serde_json::{json, Value};
|
||||
use std::collections::BTreeSet;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::openhuman::memory::store::types::NamespaceDocumentInput;
|
||||
use crate::openhuman::memory::store::types::{NamespaceDocumentInput, StoredMemoryDocument};
|
||||
|
||||
use super::UnifiedMemory;
|
||||
|
||||
@@ -14,11 +14,32 @@ impl UnifiedMemory {
|
||||
if key.is_empty() {
|
||||
return Err("document key cannot be empty".to_string());
|
||||
}
|
||||
let existing_document_id = {
|
||||
let conn = self.conn.lock();
|
||||
conn.query_row(
|
||||
"SELECT document_id FROM memory_docs WHERE namespace = ?1 AND key = ?2 LIMIT 1",
|
||||
params![namespace, key],
|
||||
|row| row.get::<_, String>(0),
|
||||
)
|
||||
.optional()
|
||||
.map_err(|e| format!("lookup existing document_id: {e}"))?
|
||||
};
|
||||
let document_id = input
|
||||
.document_id
|
||||
.or(existing_document_id)
|
||||
.unwrap_or_else(|| Uuid::new_v4().to_string());
|
||||
let now = Self::now_ts();
|
||||
let created_at = now;
|
||||
let created_at = {
|
||||
let conn = self.conn.lock();
|
||||
conn.query_row(
|
||||
"SELECT created_at FROM memory_docs WHERE namespace = ?1 AND key = ?2 LIMIT 1",
|
||||
params![namespace, key],
|
||||
|row| row.get::<_, f64>(0),
|
||||
)
|
||||
.optional()
|
||||
.map_err(|e| format!("lookup existing created_at: {e}"))?
|
||||
.unwrap_or(now)
|
||||
};
|
||||
let updated_at = now;
|
||||
let markdown_rel = self
|
||||
.write_markdown_doc(
|
||||
@@ -84,7 +105,7 @@ impl UnifiedMemory {
|
||||
tx.commit().map_err(|e| format!("commit tx: {e}"))?;
|
||||
}
|
||||
|
||||
let chunks = Self::split_chunks(&input.content, 900);
|
||||
let chunks = Self::chunk_document_content(&input.content, 225);
|
||||
for (idx, chunk) in chunks.iter().enumerate() {
|
||||
let embedding = self
|
||||
.embedder
|
||||
@@ -115,6 +136,64 @@ impl UnifiedMemory {
|
||||
Ok(document_id)
|
||||
}
|
||||
|
||||
pub(crate) async fn load_documents_for_scope(
|
||||
&self,
|
||||
namespace: &str,
|
||||
) -> Result<Vec<StoredMemoryDocument>, String> {
|
||||
let conn = self.conn.lock();
|
||||
let ns = Self::sanitize_namespace(namespace);
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT
|
||||
document_id,
|
||||
namespace,
|
||||
key,
|
||||
title,
|
||||
content,
|
||||
source_type,
|
||||
priority,
|
||||
tags_json,
|
||||
metadata_json,
|
||||
category,
|
||||
session_id,
|
||||
created_at,
|
||||
updated_at,
|
||||
markdown_rel_path
|
||||
FROM memory_docs
|
||||
WHERE namespace = ?1
|
||||
ORDER BY updated_at DESC",
|
||||
)
|
||||
.map_err(|e| format!("prepare load_documents_for_scope: {e}"))?;
|
||||
let mut rows = stmt
|
||||
.query(params![ns])
|
||||
.map_err(|e| format!("query load_documents_for_scope: {e}"))?;
|
||||
let mut docs = Vec::new();
|
||||
while let Some(row) = rows
|
||||
.next()
|
||||
.map_err(|e| format!("row load_documents_for_scope: {e}"))?
|
||||
{
|
||||
let tags_json: String = row.get(7).map_err(|e| e.to_string())?;
|
||||
let metadata_json: String = row.get(8).map_err(|e| e.to_string())?;
|
||||
docs.push(StoredMemoryDocument {
|
||||
document_id: row.get(0).map_err(|e| e.to_string())?,
|
||||
namespace: row.get(1).map_err(|e| e.to_string())?,
|
||||
key: row.get(2).map_err(|e| e.to_string())?,
|
||||
title: row.get(3).map_err(|e| e.to_string())?,
|
||||
content: row.get(4).map_err(|e| e.to_string())?,
|
||||
source_type: row.get(5).map_err(|e| e.to_string())?,
|
||||
priority: row.get(6).map_err(|e| e.to_string())?,
|
||||
tags: serde_json::from_str(&tags_json).unwrap_or_default(),
|
||||
metadata: serde_json::from_str(&metadata_json).unwrap_or_else(|_| json!({})),
|
||||
category: row.get(9).map_err(|e| e.to_string())?,
|
||||
session_id: row.get(10).map_err(|e| e.to_string())?,
|
||||
created_at: row.get(11).map_err(|e| e.to_string())?,
|
||||
updated_at: row.get(12).map_err(|e| e.to_string())?,
|
||||
markdown_rel_path: row.get(13).map_err(|e| e.to_string())?,
|
||||
});
|
||||
}
|
||||
Ok(docs)
|
||||
}
|
||||
|
||||
pub async fn list_documents(&self, namespace: Option<&str>) -> Result<Value, String> {
|
||||
let conn = self.conn.lock();
|
||||
let mut docs = Vec::new();
|
||||
@@ -199,27 +278,36 @@ impl UnifiedMemory {
|
||||
document_id: &str,
|
||||
) -> Result<Value, String> {
|
||||
let ns = Self::sanitize_namespace(namespace);
|
||||
let conn = self.conn.lock();
|
||||
let rel_path: Option<String> = conn
|
||||
.query_row(
|
||||
let rel_path: Option<String> = {
|
||||
let conn = self.conn.lock();
|
||||
conn.query_row(
|
||||
"SELECT markdown_rel_path FROM memory_docs WHERE namespace = ?1 AND document_id = ?2",
|
||||
params![ns, document_id],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.optional()
|
||||
.map_err(|e| format!("query delete_document path: {e}"))?;
|
||||
let deleted = conn
|
||||
.execute(
|
||||
"DELETE FROM memory_docs WHERE namespace = ?1 AND document_id = ?2",
|
||||
.map_err(|e| format!("query delete_document path: {e}"))?
|
||||
};
|
||||
|
||||
self.graph_remove_document_namespace(&ns, document_id)
|
||||
.await?;
|
||||
|
||||
let deleted = {
|
||||
let conn = self.conn.lock();
|
||||
let deleted = conn
|
||||
.execute(
|
||||
"DELETE FROM memory_docs WHERE namespace = ?1 AND document_id = ?2",
|
||||
params![ns, document_id],
|
||||
)
|
||||
.map_err(|e| format!("delete memory_doc: {e}"))?
|
||||
> 0;
|
||||
conn.execute(
|
||||
"DELETE FROM vector_chunks WHERE namespace = ?1 AND document_id = ?2",
|
||||
params![ns, document_id],
|
||||
)
|
||||
.map_err(|e| format!("delete memory_doc: {e}"))?
|
||||
> 0;
|
||||
conn.execute(
|
||||
"DELETE FROM vector_chunks WHERE namespace = ?1 AND document_id = ?2",
|
||||
params![ns, document_id],
|
||||
)
|
||||
.map_err(|e| format!("delete vector_chunks: {e}"))?;
|
||||
.map_err(|e| format!("delete vector_chunks: {e}"))?;
|
||||
deleted
|
||||
};
|
||||
|
||||
if let Some(rel) = rel_path {
|
||||
let abs = self.workspace_dir.join(rel);
|
||||
|
||||
@@ -1,9 +1,99 @@
|
||||
use rusqlite::params;
|
||||
use serde_json::json;
|
||||
use rusqlite::{params, OptionalExtension};
|
||||
use serde_json::{json, Map, Value};
|
||||
|
||||
use crate::openhuman::memory::store::types::GraphRelationRecord;
|
||||
|
||||
use super::UnifiedMemory;
|
||||
|
||||
impl UnifiedMemory {
|
||||
pub(crate) async fn graph_remove_document_namespace(
|
||||
&self,
|
||||
namespace: &str,
|
||||
document_id: &str,
|
||||
) -> Result<(), String> {
|
||||
let relations = self
|
||||
.graph_relations_namespace(namespace, None, None)
|
||||
.await?;
|
||||
if relations.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let doc_prefix = format!("{document_id}:");
|
||||
let updated_at = Self::now_ts();
|
||||
let conn = self.conn.lock();
|
||||
let tx = conn
|
||||
.unchecked_transaction()
|
||||
.map_err(|e| format!("graph_remove_document_namespace begin tx: {e}"))?;
|
||||
|
||||
for relation in relations {
|
||||
let touches_document = relation.document_ids.iter().any(|id| id == document_id)
|
||||
|| relation
|
||||
.chunk_ids
|
||||
.iter()
|
||||
.any(|chunk_id| chunk_id.starts_with(&doc_prefix));
|
||||
if !touches_document {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut attrs = relation.attrs.as_object().cloned().unwrap_or_default();
|
||||
let document_ids = relation
|
||||
.document_ids
|
||||
.iter()
|
||||
.filter(|id| id.as_str() != document_id)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
let chunk_ids = relation
|
||||
.chunk_ids
|
||||
.iter()
|
||||
.filter(|chunk_id| !chunk_id.starts_with(&doc_prefix))
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if document_ids.is_empty() && chunk_ids.is_empty() {
|
||||
tx.execute(
|
||||
"DELETE FROM graph_namespace
|
||||
WHERE namespace = ?1 AND subject = ?2 AND predicate = ?3 AND object = ?4",
|
||||
params![
|
||||
Self::sanitize_namespace(namespace),
|
||||
relation.subject,
|
||||
relation.predicate,
|
||||
relation.object
|
||||
],
|
||||
)
|
||||
.map_err(|e| format!("graph_remove_document_namespace delete: {e}"))?;
|
||||
continue;
|
||||
}
|
||||
|
||||
attrs.insert("document_ids".to_string(), json!(document_ids));
|
||||
if chunk_ids.is_empty() {
|
||||
attrs.remove("chunk_ids");
|
||||
} else {
|
||||
attrs.insert("chunk_ids".to_string(), json!(chunk_ids.clone()));
|
||||
}
|
||||
attrs.insert("evidence_count".to_string(), json!(chunk_ids.len().max(1)));
|
||||
attrs.insert("updated_at".to_string(), json!(updated_at));
|
||||
|
||||
tx.execute(
|
||||
"UPDATE graph_namespace
|
||||
SET attrs_json = ?1, updated_at = ?2
|
||||
WHERE namespace = ?3 AND subject = ?4 AND predicate = ?5 AND object = ?6",
|
||||
params![
|
||||
Value::Object(attrs).to_string(),
|
||||
updated_at,
|
||||
Self::sanitize_namespace(namespace),
|
||||
relation.subject,
|
||||
relation.predicate,
|
||||
relation.object
|
||||
],
|
||||
)
|
||||
.map_err(|e| format!("graph_remove_document_namespace update: {e}"))?;
|
||||
}
|
||||
|
||||
tx.commit()
|
||||
.map_err(|e| format!("graph_remove_document_namespace commit: {e}"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn graph_upsert_global(
|
||||
&self,
|
||||
subject: &str,
|
||||
@@ -11,15 +101,8 @@ impl UnifiedMemory {
|
||||
object: &str,
|
||||
attrs: &serde_json::Value,
|
||||
) -> Result<(), String> {
|
||||
let conn = self.conn.lock();
|
||||
conn.execute(
|
||||
"INSERT INTO graph_global (subject, predicate, object, attrs_json, updated_at)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5)
|
||||
ON CONFLICT(subject, predicate, object) DO UPDATE SET attrs_json = excluded.attrs_json, updated_at = excluded.updated_at",
|
||||
params![subject, predicate, object, attrs.to_string(), Self::now_ts()],
|
||||
)
|
||||
.map_err(|e| format!("graph_upsert_global: {e}"))?;
|
||||
Ok(())
|
||||
self.graph_upsert_internal(None, subject, predicate, object, attrs)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn graph_upsert_namespace(
|
||||
@@ -30,22 +113,8 @@ impl UnifiedMemory {
|
||||
object: &str,
|
||||
attrs: &serde_json::Value,
|
||||
) -> Result<(), String> {
|
||||
let conn = self.conn.lock();
|
||||
conn.execute(
|
||||
"INSERT INTO graph_namespace (namespace, subject, predicate, object, attrs_json, updated_at)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6)
|
||||
ON CONFLICT(namespace, subject, predicate, object) DO UPDATE SET attrs_json = excluded.attrs_json, updated_at = excluded.updated_at",
|
||||
params![
|
||||
Self::sanitize_namespace(namespace),
|
||||
subject,
|
||||
predicate,
|
||||
object,
|
||||
attrs.to_string(),
|
||||
Self::now_ts()
|
||||
],
|
||||
)
|
||||
.map_err(|e| format!("graph_upsert_namespace: {e}"))?;
|
||||
Ok(())
|
||||
self.graph_upsert_internal(Some(namespace), subject, predicate, object, attrs)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn graph_query_global(
|
||||
@@ -53,7 +122,94 @@ impl UnifiedMemory {
|
||||
subject: Option<&str>,
|
||||
predicate: Option<&str>,
|
||||
) -> Result<Vec<serde_json::Value>, String> {
|
||||
let rows = self.graph_relations_global(subject, predicate).await?;
|
||||
Ok(rows
|
||||
.into_iter()
|
||||
.map(Self::graph_relation_to_json)
|
||||
.collect::<Vec<_>>())
|
||||
}
|
||||
|
||||
pub async fn graph_query_namespace(
|
||||
&self,
|
||||
namespace: &str,
|
||||
subject: Option<&str>,
|
||||
predicate: Option<&str>,
|
||||
) -> Result<Vec<serde_json::Value>, String> {
|
||||
let rows = self
|
||||
.graph_relations_namespace(namespace, subject, predicate)
|
||||
.await?;
|
||||
Ok(rows
|
||||
.into_iter()
|
||||
.map(Self::graph_relation_to_json)
|
||||
.collect::<Vec<_>>())
|
||||
}
|
||||
|
||||
pub(crate) async fn graph_relations_for_scope(
|
||||
&self,
|
||||
namespace: &str,
|
||||
) -> Result<Vec<GraphRelationRecord>, String> {
|
||||
let mut rows = self
|
||||
.graph_relations_namespace(namespace, None, None)
|
||||
.await?;
|
||||
rows.extend(self.graph_relations_global(None, None).await?);
|
||||
rows.sort_by(|a, b| {
|
||||
b.updated_at
|
||||
.partial_cmp(&a.updated_at)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
});
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
pub(crate) async fn graph_relations_namespace(
|
||||
&self,
|
||||
namespace: &str,
|
||||
subject: Option<&str>,
|
||||
predicate: Option<&str>,
|
||||
) -> Result<Vec<GraphRelationRecord>, String> {
|
||||
let conn = self.conn.lock();
|
||||
let ns = Self::sanitize_namespace(namespace);
|
||||
let subject = subject.map(Self::normalize_graph_entity);
|
||||
let predicate = predicate.map(Self::normalize_graph_predicate);
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT subject, predicate, object, attrs_json, updated_at
|
||||
FROM graph_namespace
|
||||
WHERE namespace = ?1
|
||||
AND (?2 IS NULL OR subject = ?2)
|
||||
AND (?3 IS NULL OR predicate = ?3)
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT 300",
|
||||
)
|
||||
.map_err(|e| format!("graph_relations_namespace prepare: {e}"))?;
|
||||
let mut rows = stmt
|
||||
.query(params![ns, subject, predicate])
|
||||
.map_err(|e| format!("graph_relations_namespace query: {e}"))?;
|
||||
let mut out = Vec::new();
|
||||
while let Some(row) = rows
|
||||
.next()
|
||||
.map_err(|e| format!("graph_relations_namespace row: {e}"))?
|
||||
{
|
||||
let attrs_raw: String = row.get(3).map_err(|e| e.to_string())?;
|
||||
out.push(Self::graph_relation_from_parts(
|
||||
Some(Self::sanitize_namespace(namespace)),
|
||||
row.get(0).map_err(|e| e.to_string())?,
|
||||
row.get(1).map_err(|e| e.to_string())?,
|
||||
row.get(2).map_err(|e| e.to_string())?,
|
||||
&attrs_raw,
|
||||
row.get(4).map_err(|e| e.to_string())?,
|
||||
));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub(crate) async fn graph_relations_global(
|
||||
&self,
|
||||
subject: Option<&str>,
|
||||
predicate: Option<&str>,
|
||||
) -> Result<Vec<GraphRelationRecord>, String> {
|
||||
let conn = self.conn.lock();
|
||||
let subject = subject.map(Self::normalize_graph_entity);
|
||||
let predicate = predicate.map(Self::normalize_graph_predicate);
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT subject, predicate, object, attrs_json, updated_at
|
||||
@@ -63,63 +219,216 @@ impl UnifiedMemory {
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT 300",
|
||||
)
|
||||
.map_err(|e| format!("graph_query_global prepare: {e}"))?;
|
||||
.map_err(|e| format!("graph_relations_global prepare: {e}"))?;
|
||||
let mut rows = stmt
|
||||
.query(params![subject, predicate])
|
||||
.map_err(|e| format!("graph_query_global query: {e}"))?;
|
||||
.map_err(|e| format!("graph_relations_global query: {e}"))?;
|
||||
let mut out = Vec::new();
|
||||
while let Some(row) = rows
|
||||
.next()
|
||||
.map_err(|e| format!("graph_query_global row: {e}"))?
|
||||
.map_err(|e| format!("graph_relations_global row: {e}"))?
|
||||
{
|
||||
let attrs_raw: String = row.get(3).map_err(|e| e.to_string())?;
|
||||
out.push(json!({
|
||||
"subject": row.get::<_, String>(0).map_err(|e| e.to_string())?,
|
||||
"predicate": row.get::<_, String>(1).map_err(|e| e.to_string())?,
|
||||
"object": row.get::<_, String>(2).map_err(|e| e.to_string())?,
|
||||
"attrs": serde_json::from_str::<serde_json::Value>(&attrs_raw).unwrap_or_else(|_| json!({})),
|
||||
"updatedAt": row.get::<_, f64>(4).map_err(|e| e.to_string())?,
|
||||
}));
|
||||
out.push(Self::graph_relation_from_parts(
|
||||
None,
|
||||
row.get(0).map_err(|e| e.to_string())?,
|
||||
row.get(1).map_err(|e| e.to_string())?,
|
||||
row.get(2).map_err(|e| e.to_string())?,
|
||||
&attrs_raw,
|
||||
row.get(4).map_err(|e| e.to_string())?,
|
||||
));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub async fn graph_query_namespace(
|
||||
async fn graph_upsert_internal(
|
||||
&self,
|
||||
namespace: &str,
|
||||
subject: Option<&str>,
|
||||
predicate: Option<&str>,
|
||||
) -> Result<Vec<serde_json::Value>, String> {
|
||||
namespace: Option<&str>,
|
||||
subject: &str,
|
||||
predicate: &str,
|
||||
object: &str,
|
||||
attrs: &serde_json::Value,
|
||||
) -> Result<(), String> {
|
||||
let subject = Self::normalize_graph_entity(subject);
|
||||
let predicate = Self::normalize_graph_predicate(predicate);
|
||||
let object = Self::normalize_graph_entity(object);
|
||||
let updated_at = Self::now_ts();
|
||||
let conn = self.conn.lock();
|
||||
let ns = Self::sanitize_namespace(namespace);
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT subject, predicate, object, attrs_json, updated_at
|
||||
FROM graph_namespace
|
||||
WHERE namespace = ?1
|
||||
AND (?2 IS NULL OR subject = ?2)
|
||||
AND (?3 IS NULL OR predicate = ?3)
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT 300",
|
||||
)
|
||||
.map_err(|e| format!("graph_query_namespace prepare: {e}"))?;
|
||||
let mut rows = stmt
|
||||
.query(params![ns, subject, predicate])
|
||||
.map_err(|e| format!("graph_query_namespace query: {e}"))?;
|
||||
let mut out = Vec::new();
|
||||
while let Some(row) = rows
|
||||
.next()
|
||||
.map_err(|e| format!("graph_query_namespace row: {e}"))?
|
||||
{
|
||||
let attrs_raw: String = row.get(3).map_err(|e| e.to_string())?;
|
||||
out.push(json!({
|
||||
"subject": row.get::<_, String>(0).map_err(|e| e.to_string())?,
|
||||
"predicate": row.get::<_, String>(1).map_err(|e| e.to_string())?,
|
||||
"object": row.get::<_, String>(2).map_err(|e| e.to_string())?,
|
||||
"attrs": serde_json::from_str::<serde_json::Value>(&attrs_raw).unwrap_or_else(|_| json!({})),
|
||||
"updatedAt": row.get::<_, f64>(4).map_err(|e| e.to_string())?,
|
||||
}));
|
||||
|
||||
let existing_attrs: Option<String> = match namespace {
|
||||
Some(ns) => conn
|
||||
.query_row(
|
||||
"SELECT attrs_json
|
||||
FROM graph_namespace
|
||||
WHERE namespace = ?1 AND subject = ?2 AND predicate = ?3 AND object = ?4",
|
||||
params![Self::sanitize_namespace(ns), subject, predicate, object],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.optional()
|
||||
.map_err(|e| format!("graph_upsert_namespace lookup: {e}"))?,
|
||||
None => conn
|
||||
.query_row(
|
||||
"SELECT attrs_json
|
||||
FROM graph_global
|
||||
WHERE subject = ?1 AND predicate = ?2 AND object = ?3",
|
||||
params![subject, predicate, object],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.optional()
|
||||
.map_err(|e| format!("graph_upsert_global lookup: {e}"))?,
|
||||
};
|
||||
|
||||
let merged_attrs = Self::merge_graph_attrs(existing_attrs.as_deref(), attrs, updated_at);
|
||||
let merged_attrs_json = merged_attrs.to_string();
|
||||
|
||||
match namespace {
|
||||
Some(ns) => {
|
||||
conn.execute(
|
||||
"INSERT INTO graph_namespace (namespace, subject, predicate, object, attrs_json, updated_at)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6)
|
||||
ON CONFLICT(namespace, subject, predicate, object)
|
||||
DO UPDATE SET attrs_json = excluded.attrs_json, updated_at = excluded.updated_at",
|
||||
params![
|
||||
Self::sanitize_namespace(ns),
|
||||
subject,
|
||||
predicate,
|
||||
object,
|
||||
merged_attrs_json,
|
||||
updated_at
|
||||
],
|
||||
)
|
||||
.map_err(|e| format!("graph_upsert_namespace: {e}"))?;
|
||||
}
|
||||
None => {
|
||||
conn.execute(
|
||||
"INSERT INTO graph_global (subject, predicate, object, attrs_json, updated_at)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5)
|
||||
ON CONFLICT(subject, predicate, object)
|
||||
DO UPDATE SET attrs_json = excluded.attrs_json, updated_at = excluded.updated_at",
|
||||
params![subject, predicate, object, merged_attrs_json, updated_at],
|
||||
)
|
||||
.map_err(|e| format!("graph_upsert_global: {e}"))?;
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn merge_graph_attrs(
|
||||
existing_attrs_raw: Option<&str>,
|
||||
incoming_attrs: &Value,
|
||||
updated_at: f64,
|
||||
) -> Value {
|
||||
let existing = existing_attrs_raw
|
||||
.and_then(|raw| serde_json::from_str::<Value>(raw).ok())
|
||||
.unwrap_or_else(|| json!({}));
|
||||
let existing_evidence = Self::json_i64(&existing, "evidence_count")
|
||||
.unwrap_or(0)
|
||||
.max(0) as u64;
|
||||
let existing_document_ids =
|
||||
Self::json_string_array(&existing, "document_ids", "document_id");
|
||||
let existing_chunk_ids = Self::json_string_array(&existing, "chunk_ids", "chunk_id");
|
||||
|
||||
let mut merged = match existing {
|
||||
Value::Object(map) => map,
|
||||
_ => Map::new(),
|
||||
};
|
||||
let incoming_map = incoming_attrs.as_object().cloned().unwrap_or_default();
|
||||
let existing_order_index = Self::json_i64(&Value::Object(merged.clone()), "order_index");
|
||||
let incoming_order_index = Self::json_i64(incoming_attrs, "order_index");
|
||||
let merged_order_index = match (existing_order_index, incoming_order_index) {
|
||||
(Some(left), Some(right)) => Some(left.min(right)),
|
||||
(Some(left), None) => Some(left),
|
||||
(None, Some(right)) => Some(right),
|
||||
(None, None) => None,
|
||||
};
|
||||
|
||||
for (key, value) in incoming_map {
|
||||
merged.insert(key, value);
|
||||
}
|
||||
|
||||
let incoming_evidence = Self::json_i64(incoming_attrs, "evidence_count")
|
||||
.unwrap_or(1)
|
||||
.max(0) as u64;
|
||||
let evidence_count = existing_evidence.saturating_add(incoming_evidence).max(1);
|
||||
|
||||
merged.insert("evidence_count".to_string(), json!(evidence_count));
|
||||
merged.insert("updated_at".to_string(), json!(updated_at));
|
||||
|
||||
let mut document_ids = existing_document_ids;
|
||||
document_ids.extend(Self::json_string_array(
|
||||
incoming_attrs,
|
||||
"document_ids",
|
||||
"document_id",
|
||||
));
|
||||
document_ids.sort();
|
||||
document_ids.dedup();
|
||||
if !document_ids.is_empty() {
|
||||
merged.insert("document_ids".to_string(), json!(document_ids));
|
||||
}
|
||||
|
||||
let mut chunk_ids = existing_chunk_ids;
|
||||
chunk_ids.extend(Self::json_string_array(
|
||||
incoming_attrs,
|
||||
"chunk_ids",
|
||||
"chunk_id",
|
||||
));
|
||||
chunk_ids.sort();
|
||||
chunk_ids.dedup();
|
||||
if !chunk_ids.is_empty() {
|
||||
merged.insert("chunk_ids".to_string(), json!(chunk_ids));
|
||||
}
|
||||
|
||||
if !merged.contains_key("created_at") {
|
||||
merged.insert("created_at".to_string(), json!(updated_at));
|
||||
}
|
||||
if let Some(order_index) = merged_order_index {
|
||||
merged.insert("order_index".to_string(), json!(order_index));
|
||||
}
|
||||
|
||||
Value::Object(merged)
|
||||
}
|
||||
|
||||
fn graph_relation_from_parts(
|
||||
namespace: Option<String>,
|
||||
subject: String,
|
||||
predicate: String,
|
||||
object: String,
|
||||
attrs_raw: &str,
|
||||
updated_at: f64,
|
||||
) -> GraphRelationRecord {
|
||||
let attrs = serde_json::from_str::<Value>(attrs_raw).unwrap_or_else(|_| json!({}));
|
||||
let evidence_count = Self::json_i64(&attrs, "evidence_count").unwrap_or(1).max(1) as u32;
|
||||
let order_index = Self::json_i64(&attrs, "order_index");
|
||||
let document_ids = Self::json_string_array(&attrs, "document_ids", "document_id");
|
||||
let chunk_ids = Self::json_string_array(&attrs, "chunk_ids", "chunk_id");
|
||||
|
||||
GraphRelationRecord {
|
||||
namespace,
|
||||
subject,
|
||||
predicate,
|
||||
object,
|
||||
attrs,
|
||||
updated_at,
|
||||
evidence_count,
|
||||
order_index,
|
||||
document_ids,
|
||||
chunk_ids,
|
||||
}
|
||||
}
|
||||
|
||||
fn graph_relation_to_json(record: GraphRelationRecord) -> serde_json::Value {
|
||||
json!({
|
||||
"namespace": record.namespace,
|
||||
"subject": record.subject,
|
||||
"predicate": record.predicate,
|
||||
"object": record.object,
|
||||
"attrs": record.attrs,
|
||||
"updatedAt": record.updated_at,
|
||||
"evidenceCount": record.evidence_count,
|
||||
"orderIndex": record.order_index,
|
||||
"documentIds": record.document_ids,
|
||||
"chunkIds": record.chunk_ids,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
use crate::openhuman::memory::chunker::chunk_markdown;
|
||||
|
||||
use super::UnifiedMemory;
|
||||
|
||||
impl UnifiedMemory {
|
||||
@@ -74,29 +76,114 @@ impl UnifiedMemory {
|
||||
(dot / denom).clamp(0.0, 1.0)
|
||||
}
|
||||
|
||||
pub(crate) fn split_chunks(content: &str, max_len: usize) -> Vec<String> {
|
||||
let mut out = Vec::new();
|
||||
let mut current = String::new();
|
||||
for para in content.split("\n\n") {
|
||||
let p = para.trim();
|
||||
if p.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if current.is_empty() {
|
||||
current.push_str(p);
|
||||
continue;
|
||||
}
|
||||
if current.len() + 2 + p.len() <= max_len {
|
||||
current.push_str("\n\n");
|
||||
current.push_str(p);
|
||||
} else {
|
||||
out.push(std::mem::take(&mut current));
|
||||
current.push_str(p);
|
||||
pub(crate) fn chunk_document_content(content: &str, max_tokens: usize) -> Vec<String> {
|
||||
let mut chunks: Vec<String> = chunk_markdown(content, max_tokens.max(1))
|
||||
.into_iter()
|
||||
.map(|chunk| chunk.content.trim().to_string())
|
||||
.filter(|chunk: &String| !chunk.is_empty())
|
||||
.collect();
|
||||
if chunks.is_empty() && !content.trim().is_empty() {
|
||||
chunks.push(content.trim().to_string());
|
||||
}
|
||||
chunks
|
||||
}
|
||||
|
||||
pub(crate) fn collapse_whitespace(text: &str) -> String {
|
||||
text.split_whitespace().collect::<Vec<_>>().join(" ")
|
||||
}
|
||||
|
||||
pub(crate) fn normalize_search_text(text: &str) -> String {
|
||||
let collapsed = Self::collapse_whitespace(text);
|
||||
let mut normalized = String::with_capacity(collapsed.len());
|
||||
for ch in collapsed.chars() {
|
||||
if ch.is_alphanumeric() {
|
||||
normalized.extend(ch.to_lowercase());
|
||||
} else if ch.is_whitespace() || matches!(ch, '_' | '-' | '/' | '.') {
|
||||
normalized.push(' ');
|
||||
}
|
||||
}
|
||||
if !current.trim().is_empty() {
|
||||
out.push(current);
|
||||
normalized.split_whitespace().collect::<Vec<_>>().join(" ")
|
||||
}
|
||||
|
||||
pub(crate) fn tokenize_search_terms(text: &str) -> Vec<String> {
|
||||
Self::normalize_search_text(text)
|
||||
.split_whitespace()
|
||||
.map(ToOwned::to_owned)
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn normalize_graph_entity(text: &str) -> String {
|
||||
Self::collapse_whitespace(text.trim()).to_uppercase()
|
||||
}
|
||||
|
||||
pub(crate) fn normalize_graph_predicate(text: &str) -> String {
|
||||
let mut out = String::new();
|
||||
let mut last_was_sep = false;
|
||||
for ch in Self::collapse_whitespace(text.trim()).chars() {
|
||||
if ch.is_alphanumeric() {
|
||||
out.extend(ch.to_uppercase());
|
||||
last_was_sep = false;
|
||||
} else if !last_was_sep {
|
||||
out.push('_');
|
||||
last_was_sep = true;
|
||||
}
|
||||
}
|
||||
out
|
||||
out.trim_matches('_').to_string()
|
||||
}
|
||||
|
||||
pub(crate) fn json_string_array(
|
||||
value: &serde_json::Value,
|
||||
primary_key: &str,
|
||||
singular_key: &str,
|
||||
) -> Vec<String> {
|
||||
let mut items = Vec::new();
|
||||
if let Some(array) = value.get(primary_key).and_then(serde_json::Value::as_array) {
|
||||
for item in array {
|
||||
if let Some(text) = item.as_str() {
|
||||
let trimmed = text.trim();
|
||||
if !trimmed.is_empty() {
|
||||
items.push(trimmed.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(text) = value.get(singular_key).and_then(serde_json::Value::as_str) {
|
||||
let trimmed = text.trim();
|
||||
if !trimmed.is_empty() {
|
||||
items.push(trimmed.to_string());
|
||||
}
|
||||
}
|
||||
items.sort();
|
||||
items.dedup();
|
||||
items
|
||||
}
|
||||
|
||||
pub(crate) fn merge_unique_string_arrays(
|
||||
current: &serde_json::Value,
|
||||
incoming: &serde_json::Value,
|
||||
primary_key: &str,
|
||||
singular_key: &str,
|
||||
) -> Vec<String> {
|
||||
let mut merged = Self::json_string_array(current, primary_key, singular_key);
|
||||
merged.extend(Self::json_string_array(incoming, primary_key, singular_key));
|
||||
merged.sort();
|
||||
merged.dedup();
|
||||
merged
|
||||
}
|
||||
|
||||
pub(crate) fn json_i64(value: &serde_json::Value, key: &str) -> Option<i64> {
|
||||
value.get(key).and_then(|raw| {
|
||||
raw.as_i64().or_else(|| {
|
||||
raw.as_u64()
|
||||
.and_then(|v| i64::try_from(v).ok())
|
||||
.or_else(|| raw.as_f64().map(|v| v as i64))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn recency_score(updated_at: f64, now: f64) -> f64 {
|
||||
let age_secs = (now - updated_at).max(0.0);
|
||||
let age_hours = age_secs / 3600.0;
|
||||
(1.0 / (1.0 + age_hours / 24.0)).clamp(0.0, 1.0)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,6 +23,13 @@ impl UnifiedMemory {
|
||||
|
||||
let db_path = memory_dir.join("memory.db");
|
||||
let conn = Connection::open(&db_path)?;
|
||||
// Active storage layout for the core memory domain:
|
||||
// - memory_docs: namespace-scoped source documents and markdown metadata.
|
||||
// - vector_chunks: chunked document text plus optional local embedding bytes.
|
||||
// - graph_namespace: namespace graph edges used for relation-first retrieval.
|
||||
// - graph_global: cross-namespace graph edges used as fallback/shared memory.
|
||||
// - kv_namespace: namespace-scoped durable preferences, decisions, and state.
|
||||
// - kv_global: global durable key-value memories outside a namespace scope.
|
||||
conn.execute_batch(
|
||||
"PRAGMA journal_mode = WAL;
|
||||
PRAGMA synchronous = NORMAL;
|
||||
@@ -69,6 +76,7 @@ impl UnifiedMemory {
|
||||
updated_at REAL NOT NULL,
|
||||
PRIMARY KEY(subject, predicate, object)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_graph_global_subject ON graph_global(subject, predicate);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS graph_namespace (
|
||||
namespace TEXT NOT NULL,
|
||||
@@ -80,6 +88,7 @@ impl UnifiedMemory {
|
||||
PRIMARY KEY(namespace, subject, predicate, object)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_graph_namespace_ns ON graph_namespace(namespace);
|
||||
CREATE INDEX IF NOT EXISTS idx_graph_namespace_subject ON graph_namespace(namespace, subject, predicate);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS vector_chunks (
|
||||
namespace TEXT NOT NULL,
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
use rusqlite::{params, OptionalExtension};
|
||||
use serde_json::json;
|
||||
|
||||
use crate::openhuman::memory::store::types::MemoryKvRecord;
|
||||
|
||||
use super::UnifiedMemory;
|
||||
|
||||
impl UnifiedMemory {
|
||||
@@ -110,4 +112,76 @@ impl UnifiedMemory {
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub(crate) async fn kv_records_for_scope(
|
||||
&self,
|
||||
namespace: &str,
|
||||
) -> Result<Vec<MemoryKvRecord>, String> {
|
||||
let mut records = self.kv_records_namespace(namespace).await?;
|
||||
records.extend(self.kv_records_global().await?);
|
||||
records.sort_by(|a, b| {
|
||||
b.updated_at
|
||||
.partial_cmp(&a.updated_at)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
});
|
||||
Ok(records)
|
||||
}
|
||||
|
||||
pub(crate) async fn kv_records_namespace(
|
||||
&self,
|
||||
namespace: &str,
|
||||
) -> Result<Vec<MemoryKvRecord>, String> {
|
||||
let conn = self.conn.lock();
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT key, value_json, updated_at FROM kv_namespace
|
||||
WHERE namespace = ?1
|
||||
ORDER BY updated_at DESC",
|
||||
)
|
||||
.map_err(|e| format!("prepare kv_records_namespace: {e}"))?;
|
||||
let mut rows = stmt
|
||||
.query(params![Self::sanitize_namespace(namespace)])
|
||||
.map_err(|e| format!("query kv_records_namespace: {e}"))?;
|
||||
let mut out = Vec::new();
|
||||
while let Some(row) = rows
|
||||
.next()
|
||||
.map_err(|e| format!("row kv_records_namespace: {e}"))?
|
||||
{
|
||||
let value_raw: String = row.get(1).map_err(|e| e.to_string())?;
|
||||
out.push(MemoryKvRecord {
|
||||
namespace: Some(Self::sanitize_namespace(namespace)),
|
||||
key: row.get(0).map_err(|e| e.to_string())?,
|
||||
value: serde_json::from_str(&value_raw).unwrap_or(serde_json::Value::Null),
|
||||
updated_at: row.get(2).map_err(|e| e.to_string())?,
|
||||
});
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub(crate) async fn kv_records_global(&self) -> Result<Vec<MemoryKvRecord>, String> {
|
||||
let conn = self.conn.lock();
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT key, value_json, updated_at FROM kv_global
|
||||
ORDER BY updated_at DESC",
|
||||
)
|
||||
.map_err(|e| format!("prepare kv_records_global: {e}"))?;
|
||||
let mut rows = stmt
|
||||
.query([])
|
||||
.map_err(|e| format!("query kv_records_global: {e}"))?;
|
||||
let mut out = Vec::new();
|
||||
while let Some(row) = rows
|
||||
.next()
|
||||
.map_err(|e| format!("row kv_records_global: {e}"))?
|
||||
{
|
||||
let value_raw: String = row.get(1).map_err(|e| e.to_string())?;
|
||||
out.push(MemoryKvRecord {
|
||||
namespace: None,
|
||||
key: row.get(0).map_err(|e| e.to_string())?,
|
||||
value: serde_json::from_str(&value_raw).unwrap_or(serde_json::Value::Null),
|
||||
updated_at: row.get(2).map_err(|e| e.to_string())?,
|
||||
});
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -21,6 +21,92 @@ pub async fn try_dispatch(
|
||||
params: serde_json::Value,
|
||||
) -> Option<Result<serde_json::Value, String>> {
|
||||
match method {
|
||||
"memory.init" => Some(
|
||||
async move {
|
||||
let payload: crate::openhuman::memory::MemoryInitRequest = parse_params(params)?;
|
||||
rpc_json(crate::openhuman::memory::rpc::memory_init(payload).await?)
|
||||
}
|
||||
.await,
|
||||
),
|
||||
|
||||
"memory.list_documents" => Some(
|
||||
async move {
|
||||
let payload: crate::openhuman::memory::ListDocumentsRequest = parse_params(params)?;
|
||||
rpc_json(crate::openhuman::memory::rpc::memory_list_documents(payload).await?)
|
||||
}
|
||||
.await,
|
||||
),
|
||||
|
||||
"memory.list_namespaces" => Some(
|
||||
async move {
|
||||
let payload: crate::openhuman::memory::EmptyRequest = parse_params(params)?;
|
||||
rpc_json(crate::openhuman::memory::rpc::memory_list_namespaces(payload).await?)
|
||||
}
|
||||
.await,
|
||||
),
|
||||
|
||||
"memory.delete_document" => Some(
|
||||
async move {
|
||||
let payload: crate::openhuman::memory::DeleteDocumentRequest =
|
||||
parse_params(params)?;
|
||||
rpc_json(crate::openhuman::memory::rpc::memory_delete_document(payload).await?)
|
||||
}
|
||||
.await,
|
||||
),
|
||||
|
||||
"memory.query_namespace" => Some(
|
||||
async move {
|
||||
let payload: crate::openhuman::memory::QueryNamespaceRequest =
|
||||
parse_params(params)?;
|
||||
rpc_json(crate::openhuman::memory::rpc::memory_query_namespace(payload).await?)
|
||||
}
|
||||
.await,
|
||||
),
|
||||
|
||||
"memory.recall_context" | "memory.recall_namespace" => Some(
|
||||
async move {
|
||||
let payload: crate::openhuman::memory::RecallContextRequest = parse_params(params)?;
|
||||
rpc_json(crate::openhuman::memory::rpc::memory_recall_context(payload).await?)
|
||||
}
|
||||
.await,
|
||||
),
|
||||
|
||||
"memory.recall_memories" => Some(
|
||||
async move {
|
||||
let payload: crate::openhuman::memory::RecallMemoriesRequest =
|
||||
parse_params(params)?;
|
||||
rpc_json(crate::openhuman::memory::rpc::memory_recall_memories(payload).await?)
|
||||
}
|
||||
.await,
|
||||
),
|
||||
|
||||
"ai.list_memory_files" => Some(
|
||||
async move {
|
||||
let payload: crate::openhuman::memory::ListMemoryFilesRequest =
|
||||
parse_params(params)?;
|
||||
rpc_json(crate::openhuman::memory::rpc::ai_list_memory_files(payload).await?)
|
||||
}
|
||||
.await,
|
||||
),
|
||||
|
||||
"ai.read_memory_file" => Some(
|
||||
async move {
|
||||
let payload: crate::openhuman::memory::ReadMemoryFileRequest =
|
||||
parse_params(params)?;
|
||||
rpc_json(crate::openhuman::memory::rpc::ai_read_memory_file(payload).await?)
|
||||
}
|
||||
.await,
|
||||
),
|
||||
|
||||
"ai.write_memory_file" => Some(
|
||||
async move {
|
||||
let payload: crate::openhuman::memory::WriteMemoryFileRequest =
|
||||
parse_params(params)?;
|
||||
rpc_json(crate::openhuman::memory::rpc::ai_write_memory_file(payload).await?)
|
||||
}
|
||||
.await,
|
||||
),
|
||||
|
||||
"memory.namespace.list" => Some(
|
||||
async move { rpc_json(crate::openhuman::memory::rpc::namespace_list().await?) }.await,
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user