From 370be96565da7f0dab09937b5314f35d79b36862 Mon Sep 17 00:00:00 2001
From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com>
Date: Tue, 31 Mar 2026 21:31:04 -0700
Subject: [PATCH] feat: add document import via paste text or file upload

Adds a new Upload / Paste data source that lets users paste text or upload
documents (.txt, .md, .pdf, .docx, .csv) directly into the knowledge base.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 frontend/src/pages/DataSourcesPage.tsx | 206 ++++++++++++++++++++++-
 frontend/src/types/connectors.ts       |  19 ++-
 src/openjarvis/server/app.py           |   2 +
 src/openjarvis/server/upload_router.py | 221 +++++++++++++++++++++++++
 4 files changed, 444 insertions(+), 4 deletions(-)
 create mode 100644 src/openjarvis/server/upload_router.py

diff --git a/frontend/src/pages/DataSourcesPage.tsx b/frontend/src/pages/DataSourcesPage.tsx
index 1033cc2f..be729b74 100644
--- a/frontend/src/pages/DataSourcesPage.tsx
+++ b/frontend/src/pages/DataSourcesPage.tsx
@@ -10,6 +10,7 @@ import {
   sendblueHealth,
 } from '../lib/api';
 import type { ChannelBinding, ManagedAgent } from '../lib/api';
+import { getBase } from '../lib/api';
 import { Database, MessageSquare, Loader2 } from 'lucide-react';
 import { SOURCE_CATALOG } from '../types/connectors';
 import type { ConnectRequest } from '../types/connectors';
@@ -87,6 +88,191 @@ function InlineConnectForm({
   );
 }
 
+// ---------------------------------------------------------------------------
+// Upload / Paste form
+// ---------------------------------------------------------------------------
+
+const ACCEPTED_EXTENSIONS = '.txt,.md,.pdf,.docx,.csv';
+
+function UploadForm({ onDone }: { onDone?: () => void }) {
+  const [tab, setTab] = useState<'paste' | 'upload'>('paste');
+  const [title, setTitle] = useState('');
+  const [content, setContent] = useState('');
+  const [files, setFiles] = useState<File[]>([]);
+  const [busy, setBusy] = useState(false);
+  const [result, setResult] = useState('');
+  const [error, setError] = useState('');
+
+  const handlePaste = async () => {
+    if (!content.trim()) return;
+    setBusy(true);
+    setError('');
+    setResult('');
+    try {
+      const res = await fetch(`${getBase()}/v1/connectors/upload/ingest`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ title: title.trim(), content }),
+      });
+      if (!res.ok) {
+        const err = await res.json().catch(() => ({ detail: res.statusText }));
+        throw new Error(err.detail || `Upload failed: ${res.status}`);
+      }
+      const data = await res.json();
+      setResult(`Added ${data.chunks_added} chunk${data.chunks_added !== 1 ? 's' : ''} to knowledge base`);
+      setTitle('');
+      setContent('');
+      onDone?.();
+    } catch (err: any) {
+      setError(err.message || 'Upload failed');
+    } finally {
+      setBusy(false);
+    }
+  };
+
+  const handleUpload = async () => {
+    if (files.length === 0) return;
+    setBusy(true);
+    setError('');
+    setResult('');
+    try {
+      const formData = new FormData();
+      for (const f of files) formData.append('files', f);
+      if (title.trim()) formData.append('title', title.trim());
+
+      const res = await fetch(`${getBase()}/v1/connectors/upload/ingest/files`, {
+        method: 'POST',
+        body: formData,
+      });
+      if (!res.ok) {
+        const err = await res.json().catch(() => ({ detail: res.statusText }));
+        throw new Error(err.detail || `Upload failed: ${res.status}`);
+      }
+      const data = await res.json();
+      setResult(`Added ${data.chunks_added} chunk${data.chunks_added !== 1 ? 's' : ''} from ${files.length} file${files.length !== 1 ? 's' : ''}`);
+      setFiles([]);
+      setTitle('');
+      onDone?.();
+    } catch (err: any) {
+      setError(err.message || 'Upload failed');
+    } finally {
+      setBusy(false);
+    }
+  };
+
+  const tabStyle = (active: boolean): React.CSSProperties => ({
+    flex: 1, padding: '6px 0', textAlign: 'center',
+    fontSize: 12, fontWeight: 600, cursor: 'pointer',
+    background: active ? '#7c3aed' : 'transparent',
+    color: active ? 'white' : 'var(--color-text-secondary)',
+    border: 'none', borderRadius: 4,
+  });
+
+  const inputStyle: React.CSSProperties = {
+    width: '100%', padding: '7px 10px',
+    background: 'var(--color-bg)',
+    border: '1px solid var(--color-border)',
+    borderRadius: 4, color: 'var(--color-text)',
+    fontSize: 12, marginBottom: 6,
+    boxSizing: 'border-box' as const,
+  };
+
+  return (
+    <div>
+      {/* Tab bar */}
+      <div style={{ display: 'flex', gap: 4, marginBottom: 10,
+        background: 'var(--color-bg)', borderRadius: 6, padding: 2 }}>
+        <button style={tabStyle(tab === 'paste')} onClick={() => setTab('paste')}>
+          Paste Text
+        </button>
+        <button style={tabStyle(tab === 'upload')} onClick={() => setTab('upload')}>
+          Upload Files
+        </button>
+      </div>
+
+      {/* Title input (shared) */}
+      <input
+        value={title}
+        onChange={(e) => setTitle(e.target.value)}
+        placeholder="Title (optional)"
+        style={inputStyle}
+      />
+
+      {tab === 'paste' && (
+        <>
+          <textarea
+            value={content}
+            onChange={(e) => setContent(e.target.value)}
+            placeholder="Paste your text here..."
+            rows={6}
+            style={{
+              ...inputStyle,
+              resize: 'vertical',
+              fontFamily: 'inherit',
+              minHeight: 100,
+            }}
+          />
+          <button
+            onClick={handlePaste}
+            disabled={busy || !content.trim()}
+            style={{
+              width: '100%', padding: 8,
+              background: busy || !content.trim() ? '#444' : '#7c3aed',
+              color: 'white', border: 'none',
+              borderRadius: 6, fontSize: 12, cursor: 'pointer',
+            }}
+          >
+            {busy ? 'Adding...' : 'Add to Knowledge Base'}
+          </button>
+        </>
+      )}
+
+      {tab === 'upload' && (
+        <>
+          <input
+            type="file"
+            multiple
+            accept={ACCEPTED_EXTENSIONS}
+            onChange={(e) => {
+              const selected = Array.from(e.target.files || []);
+              setFiles(selected);
+            }}
+            style={{ ...inputStyle, padding: 6 }}
+          />
+          {files.length > 0 && (
+            <div style={{ fontSize: 11, color: 'var(--color-text-secondary)', marginBottom: 6 }}>
+              {files.map((f) => f.name).join(', ')}
+            </div>
+          )}
+          <button
+            onClick={handleUpload}
+            disabled={busy || files.length === 0}
+            style={{
+              width: '100%', padding: 8,
+              background: busy || files.length === 0 ? '#444' : '#7c3aed',
+              color: 'white', border: 'none',
+              borderRadius: 6, fontSize: 12, cursor: 'pointer',
+            }}
+          >
+            {busy ? 'Uploading...' : 'Upload & Index'}
+          </button>
+        </>
+      )}
+
+      {result && (
+        <div style={{ fontSize: 12, color: '#4ade80', marginTop: 8 }}>
+          {result}
+        </div>
+      )}
+      {error && (
+        <div style={{ fontSize: 12, color: '#ef4444', marginTop: 8 }}>
+          {error}
+        </div>
+      )}
+    </div>
+  );
+}
+
 // ---------------------------------------------------------------------------
 // Icon map
 // ---------------------------------------------------------------------------
@@ -96,7 +282,7 @@ const iconMap: Record<string, string> = {
   imessage: '\uD83D\uDCAC', gdrive: '\uD83D\uDCC1', notion: '\uD83D\uDCC4',
   obsidian: '\uD83D\uDCC1', granola: '\uD83C\uDF99\uFE0F', gcalendar: '\uD83D\uDCC5',
   gcontacts: '\uD83D\uDCC7', outlook: '\u2709\uFE0F', apple_notes: '\uD83C\uDF4E',
-  dropbox: '\uD83D\uDCE6', whatsapp: '\uD83D\uDCF1',
+  dropbox: '\uD83D\uDCE6', whatsapp: '\uD83D\uDCF1', upload: '\uD83D\uDCC2',
 };
 
 // ---------------------------------------------------------------------------
@@ -383,7 +569,12 @@ function DataSourcesSection() {
   };
 
   const connected = connectors.filter((c) => c.connected);
-  const notConnected = connectors.filter((c) => !c.connected);
+  const notConnectedBase = connectors.filter((c) => !c.connected);
+  // Always show the upload card in the not-connected list (it has no backend connector)
+  const uploadEntry = { connector_id: 'upload', display_name: 'Upload / Paste', connected: false, chunks: 0 };
+  const notConnected = notConnectedBase.some((c) => c.connector_id === 'upload')
+    ? notConnectedBase
+    : [...notConnectedBase, uploadEntry];
 
   return (
     <div>
@@ -531,7 +722,16 @@ function DataSourcesSection() {
                   </span>
                 </div>
 
-                {isExpanded && meta?.steps && (
+                {isExpanded && c.connector_id === 'upload' && (
+                  <div style={{ borderTop: '1px solid var(--color-border)', padding: 12 }}>
+                    <div style={{ fontSize: 12, color: 'var(--color-text-secondary)', marginBottom: 10 }}>
+                      Paste text or upload files (.txt, .md, .pdf, .docx, .csv) to add them to your knowledge base.
+                    </div>
+                    <UploadForm onDone={loadConnectors} />
+                  </div>
+                )}
+
+                {isExpanded && c.connector_id !== 'upload' && meta?.steps && (
                   <div style={{ borderTop: '1px solid var(--color-border)', padding: 12 }}>
                     {meta.steps.map((step, i) => (
                       <div
diff --git a/frontend/src/types/connectors.ts b/frontend/src/types/connectors.ts
index d74c5046..48af7daa 100644
--- a/frontend/src/types/connectors.ts
+++ b/frontend/src/types/connectors.ts
@@ -8,7 +8,7 @@ export interface ConnectorMeta {
   connector_id: string;
   display_name: string;
   auth_type: 'oauth' | 'local' | 'bridge' | 'filesystem';
-  category: 'communication' | 'documents' | 'pim';
+  category: 'communication' | 'documents' | 'pim' | 'other';
   icon: string;
   color: string;
   description: string;
@@ -53,7 +53,24 @@ export type WizardStep = "pick" | "connect" | "ingest" | "ready";
 // Backward-compatible alias
 export type SourceCard = ConnectorMeta;
 
+export type ConnectorCategory = ConnectorMeta['category'];
+
 export const SOURCE_CATALOG: ConnectorMeta[] = [
+  // ── Upload / Paste ─────────────────────────────────────────────────
+  {
+    connector_id: 'upload',
+    display_name: 'Upload / Paste',
+    auth_type: 'filesystem',
+    category: 'other',
+    icon: 'FileUp',
+    color: 'text-blue-400',
+    description: 'Paste text or upload documents',
+    unitLabel: 'documents',
+    steps: [
+      { label: 'Paste text or upload files (.txt, .md, .pdf, .docx, .csv) to add them to your knowledge base.' },
+    ],
+    inputFields: [],
+  },
   // ── Communication ──────────────────────────────────────────────────
   {
     connector_id: 'gmail_imap',
diff --git a/src/openjarvis/server/app.py b/src/openjarvis/server/app.py
index 549e589c..5868b204 100644
--- a/src/openjarvis/server/app.py
+++ b/src/openjarvis/server/app.py
@@ -15,6 +15,7 @@ from openjarvis.server.comparison import comparison_router
 from openjarvis.server.connectors_router import create_connectors_router
 from openjarvis.server.dashboard import dashboard_router
 from openjarvis.server.routes import router
+from openjarvis.server.upload_router import router as upload_router
 
 logger = logging.getLogger(__name__)
 
@@ -222,6 +223,7 @@ def create_app(
     app.include_router(dashboard_router)
     app.include_router(comparison_router)
     app.include_router(create_connectors_router())
+    app.include_router(upload_router)
     include_all_routes(app)
 
     # Restore SendBlue channel bindings from database on startup
diff --git a/src/openjarvis/server/upload_router.py b/src/openjarvis/server/upload_router.py
new file mode 100644
index 00000000..24f02863
--- /dev/null
+++ b/src/openjarvis/server/upload_router.py
@@ -0,0 +1,221 @@
+"""Upload / Paste router for ingesting documents into the knowledge store."""
+
+from __future__ import annotations
+
+import io
+import logging
+import uuid
+from typing import List, Optional
+
+from fastapi import APIRouter, File, Form, HTTPException, UploadFile
+from pydantic import BaseModel
+
+from openjarvis.connectors.store import KnowledgeStore
+from openjarvis.core.config import DEFAULT_CONFIG_DIR
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/v1/connectors/upload", tags=["upload"])
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+_ALLOWED_EXTENSIONS = {".txt", ".md", ".csv", ".pdf", ".docx"}
+
+
+def _chunk_text(text: str, max_chars: int = 1000) -> List[str]:
+    """Split *text* into ~max_chars pieces at paragraph boundaries."""
+    paragraphs = text.split("\n\n")
+    chunks: List[str] = []
+    current = ""
+    for para in paragraphs:
+        para = para.strip()
+        if not para:
+            continue
+        if current and len(current) + len(para) + 2 > max_chars:
+            chunks.append(current.strip())
+            current = para
+        else:
+            current = f"{current}\n\n{para}" if current else para
+    if current.strip():
+        chunks.append(current.strip())
+    # Guard against very large paragraphs that exceed max_chars
+    final: List[str] = []
+    for chunk in chunks:
+        while len(chunk) > max_chars:
+            # Find last space within limit
+            split_at = chunk.rfind(" ", 0, max_chars)
+            if split_at == -1:
+                split_at = max_chars
+            final.append(chunk[:split_at].strip())
+            chunk = chunk[split_at:].strip()
+        if chunk:
+            final.append(chunk)
+    return final
+
+
+def _extract_text_from_pdf(data: bytes) -> str:
+    """Extract text from a PDF using pdfplumber or PyPDF2."""
+    # Try pdfplumber first
+    try:
+        import pdfplumber  # type: ignore[import-untyped]
+
+        with pdfplumber.open(io.BytesIO(data)) as pdf:
+            pages = [p.extract_text() or "" for p in pdf.pages]
+        return "\n\n".join(pages)
+    except ImportError:
+        pass
+
+    # Fall back to PyPDF2
+    try:
+        from PyPDF2 import PdfReader  # type: ignore[import-untyped]
+
+        reader = PdfReader(io.BytesIO(data))
+        pages = [p.extract_text() or "" for p in reader.pages]
+        return "\n\n".join(pages)
+    except ImportError:
+        raise HTTPException(
+            status_code=500,
+            detail=(
+                "PDF parsing requires pdfplumber or PyPDF2. "
+                "Install one with: pip install pdfplumber"
+            ),
+        )
+
+
+def _extract_text_from_docx(data: bytes) -> str:
+    """Extract text from a .docx file using python-docx."""
+    try:
+        from docx import Document  # type: ignore[import-untyped]
+
+        doc = Document(io.BytesIO(data))
+        return "\n\n".join(p.text for p in doc.paragraphs if p.text.strip())
+    except ImportError:
+        raise HTTPException(
+            status_code=500,
+            detail=(
+                "DOCX parsing requires python-docx. "
+                "Install with: pip install python-docx"
+            ),
+        )
+
+
+def _get_store() -> KnowledgeStore:
+    """Return a KnowledgeStore pointing at the default knowledge DB."""
+    db_path = DEFAULT_CONFIG_DIR / "knowledge.db"
+    return KnowledgeStore(db_path=db_path)
+
+
+# ---------------------------------------------------------------------------
+# Schemas
+# ---------------------------------------------------------------------------
+
+
+class PasteRequest(BaseModel):
+    title: str = ""
+    content: str
+
+
+class IngestResponse(BaseModel):
+    chunks_added: int
+    source: str = "upload"
+
+
+# ---------------------------------------------------------------------------
+# Routes
+# ---------------------------------------------------------------------------
+
+
+@router.post("/ingest", response_model=IngestResponse)
+async def ingest_paste(body: PasteRequest) -> IngestResponse:
+    """Ingest pasted text into the knowledge store."""
+    text = body.content.strip()
+    if not text:
+        raise HTTPException(status_code=400, detail="Content is empty")
+
+    store = _get_store()
+    doc_id = str(uuid.uuid4())
+    chunks = _chunk_text(text)
+
+    for idx, chunk in enumerate(chunks):
+        store.store(
+            chunk,
+            source="upload",
+            doc_type="paste",
+            doc_id=doc_id,
+            title=body.title or "Pasted text",
+            chunk_index=idx,
+        )
+
+    logger.info("Ingested %d chunks from pasted text (doc_id=%s)", len(chunks), doc_id)
+    return IngestResponse(chunks_added=len(chunks))
+
+
+@router.post("/ingest/files", response_model=IngestResponse)
+async def ingest_files(
+    files: List[UploadFile] = File(...),
+    title: Optional[str] = Form(None),
+) -> IngestResponse:
+    """Ingest uploaded files into the knowledge store."""
+    store = _get_store()
+    total_chunks = 0
+
+    for upload in files:
+        filename = upload.filename or "untitled"
+        ext = ""
+        if "." in filename:
+            ext = "." + filename.rsplit(".", 1)[-1].lower()
+
+        if ext not in _ALLOWED_EXTENSIONS:
+            allowed = ", ".join(sorted(_ALLOWED_EXTENSIONS))
+            raise HTTPException(
+                status_code=400,
+                detail=(
+                    f"Unsupported file type: {ext}. "
+                    f"Allowed: {allowed}"
+                ),
+            )
+
+        data = await upload.read()
+
+        # Parse content based on extension
+        if ext in (".txt", ".md", ".csv"):
+            try:
+                text = data.decode("utf-8")
+            except UnicodeDecodeError:
+                text = data.decode("latin-1")
+        elif ext == ".pdf":
+            text = _extract_text_from_pdf(data)
+        elif ext == ".docx":
+            text = _extract_text_from_docx(data)
+        else:
+            continue
+
+        text = text.strip()
+        if not text:
+            continue
+
+        doc_id = str(uuid.uuid4())
+        doc_title = title or filename
+        chunks = _chunk_text(text)
+
+        for idx, chunk in enumerate(chunks):
+            store.store(
+                chunk,
+                source="upload",
+                doc_type=ext.lstrip("."),
+                doc_id=doc_id,
+                title=doc_title,
+                chunk_index=idx,
+            )
+
+        total_chunks += len(chunks)
+        logger.info(
+            "Ingested %d chunks from file %s (doc_id=%s)",
+            len(chunks),
+            filename,
+            doc_id,
+        )
+
+    return IngestResponse(chunks_added=total_chunks)