mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-28 13:32:23 +00:00
196 lines
6.3 KiB
Bash
Executable File
196 lines
6.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# bench-memory-walk.sh — benchmark memory tree walking and retrieval performance.
|
|
#
|
|
# Calls the core CLI with a set of test queries against the memory tree,
|
|
# measures latency per query, and reports summary statistics.
|
|
#
|
|
# Usage:
|
|
# ./scripts/bench-memory-walk.sh # defaults
|
|
# ./scripts/bench-memory-walk.sh --query "what is X?" # single query
|
|
# ./scripts/bench-memory-walk.sh --content-root /path/to/tree # custom root
|
|
# ./scripts/bench-memory-walk.sh --max-turns 20 # more turns
|
|
# ./scripts/bench-memory-walk.sh --model "deepseek:deepseek-chat"
|
|
# ./scripts/bench-memory-walk.sh --verbose # show full output
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
|
|
# Defaults
|
|
CONTENT_ROOT="${OPENHUMAN_MEMORY_CONTENT_ROOT:-$HOME/.openhuman-staging/users/69d9cb73e61f755583c3671f/workspace/memory_tree/content}"
|
|
MAX_TURNS=12
|
|
NAMESPACE="default"
|
|
MODEL=""
|
|
VERBOSE=0
|
|
CUSTOM_QUERY=""
|
|
|
|
# Default benchmark queries covering different retrieval patterns
|
|
DEFAULT_QUERIES=(
|
|
"What projects am I working on?"
|
|
"What did I discuss in my most recent conversations?"
|
|
"What are my preferences and settings?"
|
|
"Find any mentions of GitHub or pull requests"
|
|
"What people have I interacted with recently?"
|
|
)
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--content-root) CONTENT_ROOT="$2"; shift 2 ;;
|
|
--max-turns) MAX_TURNS="$2"; shift 2 ;;
|
|
--namespace) NAMESPACE="$2"; shift 2 ;;
|
|
--model) MODEL="$2"; shift 2 ;;
|
|
--query) CUSTOM_QUERY="$2"; shift 2 ;;
|
|
--verbose) VERBOSE=1; shift ;;
|
|
-h|--help)
|
|
echo "Usage: $0 [OPTIONS]"
|
|
echo ""
|
|
echo "Options:"
|
|
echo " --content-root PATH Memory tree content root (default: staging)"
|
|
echo " --max-turns N Max LLM turns per query (default: 12)"
|
|
echo " --namespace NS Memory namespace (default: 'default')"
|
|
echo " --model MODEL Provider:model override"
|
|
echo " --query TEXT Run a single custom query instead of defaults"
|
|
echo " --verbose Show full tool output"
|
|
echo " -h, --help Show this help"
|
|
exit 0
|
|
;;
|
|
*) echo "Unknown option: $1"; exit 1 ;;
|
|
esac
|
|
done
|
|
|
|
# Verify content root exists
|
|
if [[ ! -d "$CONTENT_ROOT" ]]; then
|
|
echo "ERROR: Content root does not exist: $CONTENT_ROOT"
|
|
echo "Set OPENHUMAN_MEMORY_CONTENT_ROOT or use --content-root"
|
|
exit 1
|
|
fi
|
|
|
|
# Count files in the tree
|
|
FILE_COUNT=$(find "$CONTENT_ROOT" -type f -name "*.md" 2>/dev/null | wc -l | tr -d ' ')
|
|
DIR_COUNT=$(find "$CONTENT_ROOT" -type d 2>/dev/null | wc -l | tr -d ' ')
|
|
TOTAL_SIZE=$(du -sh "$CONTENT_ROOT" 2>/dev/null | cut -f1)
|
|
|
|
echo "=============================================="
|
|
echo " Memory Tree Walk Benchmark"
|
|
echo "=============================================="
|
|
echo ""
|
|
echo "Content root: $CONTENT_ROOT"
|
|
echo "Files: $FILE_COUNT markdown files"
|
|
echo "Directories: $DIR_COUNT"
|
|
echo "Total size: $TOTAL_SIZE"
|
|
echo "Max turns: $MAX_TURNS"
|
|
echo "Namespace: $NAMESPACE"
|
|
if [[ -n "$MODEL" ]]; then
|
|
echo "Model: $MODEL"
|
|
fi
|
|
echo ""
|
|
|
|
# Build the queries array
|
|
if [[ -n "$CUSTOM_QUERY" ]]; then
|
|
QUERIES=("$CUSTOM_QUERY")
|
|
else
|
|
QUERIES=("${DEFAULT_QUERIES[@]}")
|
|
fi
|
|
|
|
# Check if the core binary exists
|
|
CORE_BIN="$REPO_ROOT/target/debug/openhuman-core"
|
|
if [[ ! -x "$CORE_BIN" ]]; then
|
|
echo "Building openhuman-core..."
|
|
cargo build --manifest-path "$REPO_ROOT/Cargo.toml" --bin openhuman-core 2>&1 | tail -3
|
|
echo ""
|
|
fi
|
|
|
|
# Results storage
|
|
RESULTS_DIR="$REPO_ROOT/target/bench-memory"
|
|
mkdir -p "$RESULTS_DIR"
|
|
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
|
|
RESULTS_FILE="$RESULTS_DIR/bench-$TIMESTAMP.jsonl"
|
|
|
|
echo "----------------------------------------------"
|
|
echo " Running ${#QUERIES[@]} queries"
|
|
echo "----------------------------------------------"
|
|
echo ""
|
|
|
|
TOTAL_START=$(date +%s%N 2>/dev/null || python3 -c "import time; print(int(time.time()*1e9))")
|
|
PASS=0
|
|
FAIL=0
|
|
|
|
for i in "${!QUERIES[@]}"; do
|
|
query="${QUERIES[$i]}"
|
|
idx=$((i + 1))
|
|
echo "[$idx/${#QUERIES[@]}] $query"
|
|
|
|
QUERY_START=$(date +%s%N 2>/dev/null || python3 -c "import time; print(int(time.time()*1e9))")
|
|
|
|
# Call the core CLI with the memory_smart_walk RPC
|
|
# Use python3 to safely build the JSON payload and avoid query injection
|
|
RPC_PAYLOAD=$(python3 -c "
|
|
import json, sys
|
|
payload = {
|
|
'jsonrpc': '2.0',
|
|
'id': 'bench-$idx',
|
|
'method': 'openhuman.memory_smart_walk',
|
|
'params': {
|
|
'query': sys.argv[1],
|
|
'namespace': sys.argv[2],
|
|
'max_turns': $MAX_TURNS
|
|
}
|
|
}
|
|
print(json.dumps(payload))
|
|
" "$query" "$NAMESPACE")
|
|
|
|
# Use the CLI's rpc subcommand if available, otherwise use the tool directly
|
|
if [[ $VERBOSE -eq 1 ]]; then
|
|
OUTPUT=$("$CORE_BIN" rpc --stdin <<< "$RPC_PAYLOAD" 2>&1) || true
|
|
echo "$OUTPUT"
|
|
else
|
|
OUTPUT=$("$CORE_BIN" rpc --stdin <<< "$RPC_PAYLOAD" 2>/dev/null) || true
|
|
fi
|
|
|
|
QUERY_END=$(date +%s%N 2>/dev/null || python3 -c "import time; print(int(time.time()*1e9))")
|
|
ELAPSED_MS=$(( (QUERY_END - QUERY_START) / 1000000 ))
|
|
|
|
if echo "$OUTPUT" | grep -q '"result"'; then
|
|
PASS=$((PASS + 1))
|
|
STATUS="OK"
|
|
else
|
|
FAIL=$((FAIL + 1))
|
|
STATUS="FAIL"
|
|
fi
|
|
|
|
echo " -> ${STATUS} in ${ELAPSED_MS}ms"
|
|
|
|
# Log to JSONL — use python3 to safely encode the query string
|
|
python3 -c "
|
|
import json, sys
|
|
record = {
|
|
'query': sys.argv[1],
|
|
'elapsed_ms': $ELAPSED_MS,
|
|
'status': sys.argv[2],
|
|
'timestamp': sys.argv[3]
|
|
}
|
|
print(json.dumps(record))
|
|
" "$query" "$STATUS" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$RESULTS_FILE"
|
|
echo ""
|
|
done
|
|
|
|
TOTAL_END=$(date +%s%N 2>/dev/null || python3 -c "import time; print(int(time.time()*1e9))")
|
|
TOTAL_MS=$(( (TOTAL_END - TOTAL_START) / 1000000 ))
|
|
|
|
echo "=============================================="
|
|
echo " Summary"
|
|
echo "=============================================="
|
|
echo ""
|
|
echo "Total queries: ${#QUERIES[@]}"
|
|
echo "Passed: $PASS"
|
|
echo "Failed: $FAIL"
|
|
echo "Total time: ${TOTAL_MS}ms"
|
|
echo "Results saved to: $RESULTS_FILE"
|
|
echo ""
|
|
|
|
if [[ $FAIL -gt 0 ]]; then
|
|
echo "WARNING: $FAIL queries failed. Run with --verbose to see errors."
|
|
exit 1
|
|
fi
|