Feature/twitter bot (#259)

scripts/index_docs.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python3
+"""Index OpenJarvis docs (README.md + docs/**/*.md) into a DenseMemory backend.
+
+Usage:
+    python scripts/index_docs.py              # print retrieval smoke test
+    python scripts/index_docs.py --query "can i run this on cpu?"
+
+This script is idempotent: it builds a fresh in-memory index each run.
+There is no disk persistence by design — dense vectors are cheap to
+rebuild and the docs corpus is small.
+
+Embedding model: ``nomic-embed-text`` via Ollama. Pull it with
+``ollama pull nomic-embed-text`` if you don't have it. Expected
+indexing time for the full corpus: ~30s on a warm Ollama server.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+import time
+from pathlib import Path
+
+from openjarvis.tools.storage.dense import (
+    DenseMemory,
+    MdChunk,
+    chunk_markdown,
+    dedupe_chunks,
+)
+
+
+def discover_md_files(repo_root: Path) -> list[Path]:
+    """README + every markdown file under docs/. Sorted for determinism."""
+    files: list[Path] = []
+    readme = repo_root / "README.md"
+    if readme.exists():
+        files.append(readme)
+    docs_dir = repo_root / "docs"
+    if docs_dir.is_dir():
+        files.extend(sorted(docs_dir.rglob("*.md")))
+    return files
+
+
+def build_index(
+    repo_root: Path,
+    *,
+    max_section_tokens: int = 1000,
+    paragraph_overlap_tokens: int = 100,
+    dedupe: bool = True,
+    # Empirical: on the actual OpenJarvis docs the boilerplate that
+    # crowds retrieval ("OpenJarvis runs entirely on your hardware...")
+    # appears in exactly 2 files (downloads.md ↔ installation.md).
+    # Spec'd 3+ removes 0 chunks; 2+ removes 15 (1.3%) — all genuine
+    # cross-file boilerplate. See the dry-run audit logged at index time.
+    dedupe_min_files: int = 2,
+    dedupe_threshold: float = 0.7,
+) -> DenseMemory:
+    """Chunk all markdown under *repo_root* and build a DenseMemory index.
+
+    When ``dedupe`` is True (default), runs cross-file boilerplate
+    deduplication after chunking and before embedding. The dedupe
+    report is printed to stderr so reviewers can spot over-aggressive
+    drops; if it removes >20% of the corpus a warning is emitted.
+    """
+    backend = DenseMemory()
+    md_files = discover_md_files(repo_root)
+    if not md_files:
+        raise RuntimeError(f"No markdown files found under {repo_root}")
+
+    all_chunks: list[MdChunk] = []
+    for fpath in md_files:
+        try:
+            text = fpath.read_text(encoding="utf-8")
+        except Exception as exc:
+            print(f"  WARN: could not read {fpath}: {exc}", file=sys.stderr)
+            continue
+        rel = str(fpath.relative_to(repo_root))
+        all_chunks.extend(
+            chunk_markdown(
+                text,
+                source=rel,
+                max_section_tokens=max_section_tokens,
+                paragraph_overlap_tokens=paragraph_overlap_tokens,
+            )
+        )
+
+    print(
+        f"Chunked {len(md_files)} files into {len(all_chunks)} chunks",
+        file=sys.stderr,
+    )
+
+    if dedupe:
+        before = len(all_chunks)
+        all_chunks, report = dedupe_chunks(
+            all_chunks,
+            similarity_threshold=dedupe_threshold,
+            min_files_for_dup=dedupe_min_files,
+        )
+        pct = report.removed_fraction * 100
+        print(
+            f"Dedupe: {before} -> {len(all_chunks)} chunks "
+            f"({report.removed_count} removed, {pct:.1f}%) "
+            f"across {len(report.groups)} clusters",
+            file=sys.stderr,
+        )
+        for g in report.groups:
+            dropped = sorted(set(g.dropped_sources))
+            print(
+                f"  KEPT  {g.kept_source}\n"
+                f"  DROP  {len(g.dropped_indices)} from {dropped}\n"
+                f"  TEXT  {g.sample_text!r}",
+                file=sys.stderr,
+            )
+        if report.removed_fraction > 0.20:
+            print(
+                f"  WARNING: dedupe removed {pct:.1f}% of chunks (>20% threshold). "
+                f"Review the list above before trusting the index.",
+                file=sys.stderr,
+            )
+
+    print(
+        f"Embedding {len(all_chunks)} chunks via nomic-embed-text...",
+        file=sys.stderr,
+    )
+    t0 = time.time()
+    backend.store_many(
+        [c.content for c in all_chunks],
+        sources=[c.source for c in all_chunks],
+        metadatas=[{"breadcrumb": c.breadcrumb} for c in all_chunks],
+    )
+    print(
+        f"Indexed {backend.count()} chunks in {time.time() - t0:.1f}s",
+        file=sys.stderr,
+    )
+    return backend
+
+
+def _print_hits(query: str, backend: DenseMemory, top_k: int = 3) -> None:
+    print(f"\nQ: {query}")
+    print("-" * 80)
+    hits = backend.retrieve(query, top_k=top_k)
+    if not hits:
+        print("  (no hits)")
+        return
+    for i, h in enumerate(hits, 1):
+        preview = h.content.replace("\n", " ")[:200]
+        print(f"  [{i}] score={h.score:.3f}  src={h.source}")
+        print(f"      breadcrumb={h.metadata.get('breadcrumb', '')}")
+        print(f"      {preview}{'...' if len(h.content) > 200 else ''}")
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__.strip().splitlines()[0])
+    p.add_argument(
+        "--repo-root",
+        default=str(Path(__file__).resolve().parents[1]),
+        help="Repository root (default: script's parent)",
+    )
+    p.add_argument(
+        "--query",
+        "-q",
+        action="append",
+        default=None,
+        help="Query to test against the built index (can be given multiple times)",
+    )
+    p.add_argument("--top-k", type=int, default=3, help="Top-K results per query")
+    args = p.parse_args()
+
+    repo_root = Path(args.repo_root).resolve()
+    backend = build_index(repo_root)
+
+    queries = args.query or [
+        "can I run the orchestrator agent on a laptop without a gpu?",
+        "what inference engines does openjarvis support?",
+        "how do I add a new channel integration?",
+        "why would I choose the dense memory backend over sqlite?",
+    ]
+    for q in queries:
+        _print_hits(q, backend, top_k=args.top_k)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())