mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-27 21:05:34 +00:00
feat: add document import via paste text or file upload
Adds a new Upload / Paste data source that lets users paste text or upload documents (.txt, .md, .pdf, .docx, .csv) directly into the knowledge base. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jon Saad-Falcon
committed by
mrTSB
mrTSB
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
1100057b90
commit
370be96565
@@ -10,6 +10,7 @@ import {
|
||||
sendblueHealth,
|
||||
} from '../lib/api';
|
||||
import type { ChannelBinding, ManagedAgent } from '../lib/api';
|
||||
import { getBase } from '../lib/api';
|
||||
import { Database, MessageSquare, Loader2 } from 'lucide-react';
|
||||
import { SOURCE_CATALOG } from '../types/connectors';
|
||||
import type { ConnectRequest } from '../types/connectors';
|
||||
@@ -87,6 +88,191 @@ function InlineConnectForm({
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Upload / Paste form
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const ACCEPTED_EXTENSIONS = '.txt,.md,.pdf,.docx,.csv';
|
||||
|
||||
function UploadForm({ onDone }: { onDone?: () => void }) {
|
||||
const [tab, setTab] = useState<'paste' | 'upload'>('paste');
|
||||
const [title, setTitle] = useState('');
|
||||
const [content, setContent] = useState('');
|
||||
const [files, setFiles] = useState<File[]>([]);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [result, setResult] = useState('');
|
||||
const [error, setError] = useState('');
|
||||
|
||||
const handlePaste = async () => {
|
||||
if (!content.trim()) return;
|
||||
setBusy(true);
|
||||
setError('');
|
||||
setResult('');
|
||||
try {
|
||||
const res = await fetch(`${getBase()}/v1/connectors/upload/ingest`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ title: title.trim(), content }),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const err = await res.json().catch(() => ({ detail: res.statusText }));
|
||||
throw new Error(err.detail || `Upload failed: ${res.status}`);
|
||||
}
|
||||
const data = await res.json();
|
||||
setResult(`Added ${data.chunks_added} chunk${data.chunks_added !== 1 ? 's' : ''} to knowledge base`);
|
||||
setTitle('');
|
||||
setContent('');
|
||||
onDone?.();
|
||||
} catch (err: any) {
|
||||
setError(err.message || 'Upload failed');
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleUpload = async () => {
|
||||
if (files.length === 0) return;
|
||||
setBusy(true);
|
||||
setError('');
|
||||
setResult('');
|
||||
try {
|
||||
const formData = new FormData();
|
||||
for (const f of files) formData.append('files', f);
|
||||
if (title.trim()) formData.append('title', title.trim());
|
||||
|
||||
const res = await fetch(`${getBase()}/v1/connectors/upload/ingest/files`, {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
});
|
||||
if (!res.ok) {
|
||||
const err = await res.json().catch(() => ({ detail: res.statusText }));
|
||||
throw new Error(err.detail || `Upload failed: ${res.status}`);
|
||||
}
|
||||
const data = await res.json();
|
||||
setResult(`Added ${data.chunks_added} chunk${data.chunks_added !== 1 ? 's' : ''} from ${files.length} file${files.length !== 1 ? 's' : ''}`);
|
||||
setFiles([]);
|
||||
setTitle('');
|
||||
onDone?.();
|
||||
} catch (err: any) {
|
||||
setError(err.message || 'Upload failed');
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
};
|
||||
|
||||
const tabStyle = (active: boolean): React.CSSProperties => ({
|
||||
flex: 1, padding: '6px 0', textAlign: 'center',
|
||||
fontSize: 12, fontWeight: 600, cursor: 'pointer',
|
||||
background: active ? '#7c3aed' : 'transparent',
|
||||
color: active ? 'white' : 'var(--color-text-secondary)',
|
||||
border: 'none', borderRadius: 4,
|
||||
});
|
||||
|
||||
const inputStyle: React.CSSProperties = {
|
||||
width: '100%', padding: '7px 10px',
|
||||
background: 'var(--color-bg)',
|
||||
border: '1px solid var(--color-border)',
|
||||
borderRadius: 4, color: 'var(--color-text)',
|
||||
fontSize: 12, marginBottom: 6,
|
||||
boxSizing: 'border-box' as const,
|
||||
};
|
||||
|
||||
return (
|
||||
<div>
|
||||
{/* Tab bar */}
|
||||
<div style={{ display: 'flex', gap: 4, marginBottom: 10,
|
||||
background: 'var(--color-bg)', borderRadius: 6, padding: 2 }}>
|
||||
<button style={tabStyle(tab === 'paste')} onClick={() => setTab('paste')}>
|
||||
Paste Text
|
||||
</button>
|
||||
<button style={tabStyle(tab === 'upload')} onClick={() => setTab('upload')}>
|
||||
Upload Files
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Title input (shared) */}
|
||||
<input
|
||||
value={title}
|
||||
onChange={(e) => setTitle(e.target.value)}
|
||||
placeholder="Title (optional)"
|
||||
style={inputStyle}
|
||||
/>
|
||||
|
||||
{tab === 'paste' && (
|
||||
<>
|
||||
<textarea
|
||||
value={content}
|
||||
onChange={(e) => setContent(e.target.value)}
|
||||
placeholder="Paste your text here..."
|
||||
rows={6}
|
||||
style={{
|
||||
...inputStyle,
|
||||
resize: 'vertical',
|
||||
fontFamily: 'inherit',
|
||||
minHeight: 100,
|
||||
}}
|
||||
/>
|
||||
<button
|
||||
onClick={handlePaste}
|
||||
disabled={busy || !content.trim()}
|
||||
style={{
|
||||
width: '100%', padding: 8,
|
||||
background: busy || !content.trim() ? '#444' : '#7c3aed',
|
||||
color: 'white', border: 'none',
|
||||
borderRadius: 6, fontSize: 12, cursor: 'pointer',
|
||||
}}
|
||||
>
|
||||
{busy ? 'Adding...' : 'Add to Knowledge Base'}
|
||||
</button>
|
||||
</>
|
||||
)}
|
||||
|
||||
{tab === 'upload' && (
|
||||
<>
|
||||
<input
|
||||
type="file"
|
||||
multiple
|
||||
accept={ACCEPTED_EXTENSIONS}
|
||||
onChange={(e) => {
|
||||
const selected = Array.from(e.target.files || []);
|
||||
setFiles(selected);
|
||||
}}
|
||||
style={{ ...inputStyle, padding: 6 }}
|
||||
/>
|
||||
{files.length > 0 && (
|
||||
<div style={{ fontSize: 11, color: 'var(--color-text-secondary)', marginBottom: 6 }}>
|
||||
{files.map((f) => f.name).join(', ')}
|
||||
</div>
|
||||
)}
|
||||
<button
|
||||
onClick={handleUpload}
|
||||
disabled={busy || files.length === 0}
|
||||
style={{
|
||||
width: '100%', padding: 8,
|
||||
background: busy || files.length === 0 ? '#444' : '#7c3aed',
|
||||
color: 'white', border: 'none',
|
||||
borderRadius: 6, fontSize: 12, cursor: 'pointer',
|
||||
}}
|
||||
>
|
||||
{busy ? 'Uploading...' : 'Upload & Index'}
|
||||
</button>
|
||||
</>
|
||||
)}
|
||||
|
||||
{result && (
|
||||
<div style={{ fontSize: 12, color: '#4ade80', marginTop: 8 }}>
|
||||
{result}
|
||||
</div>
|
||||
)}
|
||||
{error && (
|
||||
<div style={{ fontSize: 12, color: '#ef4444', marginTop: 8 }}>
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Icon map
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -96,7 +282,7 @@ const iconMap: Record<string, string> = {
|
||||
imessage: '\uD83D\uDCAC', gdrive: '\uD83D\uDCC1', notion: '\uD83D\uDCC4',
|
||||
obsidian: '\uD83D\uDCC1', granola: '\uD83C\uDF99\uFE0F', gcalendar: '\uD83D\uDCC5',
|
||||
gcontacts: '\uD83D\uDCC7', outlook: '\u2709\uFE0F', apple_notes: '\uD83C\uDF4E',
|
||||
dropbox: '\uD83D\uDCE6', whatsapp: '\uD83D\uDCF1',
|
||||
dropbox: '\uD83D\uDCE6', whatsapp: '\uD83D\uDCF1', upload: '\uD83D\uDCC2',
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -383,7 +569,12 @@ function DataSourcesSection() {
|
||||
};
|
||||
|
||||
const connected = connectors.filter((c) => c.connected);
|
||||
const notConnected = connectors.filter((c) => !c.connected);
|
||||
const notConnectedBase = connectors.filter((c) => !c.connected);
|
||||
// Always show the upload card in the not-connected list (it has no backend connector)
|
||||
const uploadEntry = { connector_id: 'upload', display_name: 'Upload / Paste', connected: false, chunks: 0 };
|
||||
const notConnected = notConnectedBase.some((c) => c.connector_id === 'upload')
|
||||
? notConnectedBase
|
||||
: [...notConnectedBase, uploadEntry];
|
||||
|
||||
return (
|
||||
<div>
|
||||
@@ -531,7 +722,16 @@ function DataSourcesSection() {
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{isExpanded && meta?.steps && (
|
||||
{isExpanded && c.connector_id === 'upload' && (
|
||||
<div style={{ borderTop: '1px solid var(--color-border)', padding: 12 }}>
|
||||
<div style={{ fontSize: 12, color: 'var(--color-text-secondary)', marginBottom: 10 }}>
|
||||
Paste text or upload files (.txt, .md, .pdf, .docx, .csv) to add them to your knowledge base.
|
||||
</div>
|
||||
<UploadForm onDone={loadConnectors} />
|
||||
</div>
|
||||
)}
|
||||
|
||||
{isExpanded && c.connector_id !== 'upload' && meta?.steps && (
|
||||
<div style={{ borderTop: '1px solid var(--color-border)', padding: 12 }}>
|
||||
{meta.steps.map((step, i) => (
|
||||
<div
|
||||
|
||||
@@ -8,7 +8,7 @@ export interface ConnectorMeta {
|
||||
connector_id: string;
|
||||
display_name: string;
|
||||
auth_type: 'oauth' | 'local' | 'bridge' | 'filesystem';
|
||||
category: 'communication' | 'documents' | 'pim';
|
||||
category: 'communication' | 'documents' | 'pim' | 'other';
|
||||
icon: string;
|
||||
color: string;
|
||||
description: string;
|
||||
@@ -53,7 +53,24 @@ export type WizardStep = "pick" | "connect" | "ingest" | "ready";
|
||||
// Backward-compatible alias
|
||||
export type SourceCard = ConnectorMeta;
|
||||
|
||||
export type ConnectorCategory = ConnectorMeta['category'];
|
||||
|
||||
export const SOURCE_CATALOG: ConnectorMeta[] = [
|
||||
// ── Upload / Paste ─────────────────────────────────────────────────
|
||||
{
|
||||
connector_id: 'upload',
|
||||
display_name: 'Upload / Paste',
|
||||
auth_type: 'filesystem',
|
||||
category: 'other',
|
||||
icon: 'FileUp',
|
||||
color: 'text-blue-400',
|
||||
description: 'Paste text or upload documents',
|
||||
unitLabel: 'documents',
|
||||
steps: [
|
||||
{ label: 'Paste text or upload files (.txt, .md, .pdf, .docx, .csv) to add them to your knowledge base.' },
|
||||
],
|
||||
inputFields: [],
|
||||
},
|
||||
// ── Communication ──────────────────────────────────────────────────
|
||||
{
|
||||
connector_id: 'gmail_imap',
|
||||
|
||||
@@ -15,6 +15,7 @@ from openjarvis.server.comparison import comparison_router
|
||||
from openjarvis.server.connectors_router import create_connectors_router
|
||||
from openjarvis.server.dashboard import dashboard_router
|
||||
from openjarvis.server.routes import router
|
||||
from openjarvis.server.upload_router import router as upload_router
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -222,6 +223,7 @@ def create_app(
|
||||
app.include_router(dashboard_router)
|
||||
app.include_router(comparison_router)
|
||||
app.include_router(create_connectors_router())
|
||||
app.include_router(upload_router)
|
||||
include_all_routes(app)
|
||||
|
||||
# Restore SendBlue channel bindings from database on startup
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
"""Upload / Paste router for ingesting documents into the knowledge store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import logging
|
||||
import uuid
|
||||
from typing import List, Optional
|
||||
|
||||
from fastapi import APIRouter, File, Form, HTTPException, UploadFile
|
||||
from pydantic import BaseModel
|
||||
|
||||
from openjarvis.connectors.store import KnowledgeStore
|
||||
from openjarvis.core.config import DEFAULT_CONFIG_DIR
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/v1/connectors/upload", tags=["upload"])
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_ALLOWED_EXTENSIONS = {".txt", ".md", ".csv", ".pdf", ".docx"}
|
||||
|
||||
|
||||
def _chunk_text(text: str, max_chars: int = 1000) -> List[str]:
|
||||
"""Split *text* into ~max_chars pieces at paragraph boundaries."""
|
||||
paragraphs = text.split("\n\n")
|
||||
chunks: List[str] = []
|
||||
current = ""
|
||||
for para in paragraphs:
|
||||
para = para.strip()
|
||||
if not para:
|
||||
continue
|
||||
if current and len(current) + len(para) + 2 > max_chars:
|
||||
chunks.append(current.strip())
|
||||
current = para
|
||||
else:
|
||||
current = f"{current}\n\n{para}" if current else para
|
||||
if current.strip():
|
||||
chunks.append(current.strip())
|
||||
# Guard against very large paragraphs that exceed max_chars
|
||||
final: List[str] = []
|
||||
for chunk in chunks:
|
||||
while len(chunk) > max_chars:
|
||||
# Find last space within limit
|
||||
split_at = chunk.rfind(" ", 0, max_chars)
|
||||
if split_at == -1:
|
||||
split_at = max_chars
|
||||
final.append(chunk[:split_at].strip())
|
||||
chunk = chunk[split_at:].strip()
|
||||
if chunk:
|
||||
final.append(chunk)
|
||||
return final
|
||||
|
||||
|
||||
def _extract_text_from_pdf(data: bytes) -> str:
|
||||
"""Extract text from a PDF using pdfplumber or PyPDF2."""
|
||||
# Try pdfplumber first
|
||||
try:
|
||||
import pdfplumber # type: ignore[import-untyped]
|
||||
|
||||
with pdfplumber.open(io.BytesIO(data)) as pdf:
|
||||
pages = [p.extract_text() or "" for p in pdf.pages]
|
||||
return "\n\n".join(pages)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Fall back to PyPDF2
|
||||
try:
|
||||
from PyPDF2 import PdfReader # type: ignore[import-untyped]
|
||||
|
||||
reader = PdfReader(io.BytesIO(data))
|
||||
pages = [p.extract_text() or "" for p in reader.pages]
|
||||
return "\n\n".join(pages)
|
||||
except ImportError:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=(
|
||||
"PDF parsing requires pdfplumber or PyPDF2. "
|
||||
"Install one with: pip install pdfplumber"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _extract_text_from_docx(data: bytes) -> str:
|
||||
"""Extract text from a .docx file using python-docx."""
|
||||
try:
|
||||
from docx import Document # type: ignore[import-untyped]
|
||||
|
||||
doc = Document(io.BytesIO(data))
|
||||
return "\n\n".join(p.text for p in doc.paragraphs if p.text.strip())
|
||||
except ImportError:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=(
|
||||
"DOCX parsing requires python-docx. "
|
||||
"Install with: pip install python-docx"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _get_store() -> KnowledgeStore:
|
||||
"""Return a KnowledgeStore pointing at the default knowledge DB."""
|
||||
db_path = DEFAULT_CONFIG_DIR / "knowledge.db"
|
||||
return KnowledgeStore(db_path=db_path)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class PasteRequest(BaseModel):
|
||||
title: str = ""
|
||||
content: str
|
||||
|
||||
|
||||
class IngestResponse(BaseModel):
|
||||
chunks_added: int
|
||||
source: str = "upload"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Routes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/ingest", response_model=IngestResponse)
|
||||
async def ingest_paste(body: PasteRequest) -> IngestResponse:
|
||||
"""Ingest pasted text into the knowledge store."""
|
||||
text = body.content.strip()
|
||||
if not text:
|
||||
raise HTTPException(status_code=400, detail="Content is empty")
|
||||
|
||||
store = _get_store()
|
||||
doc_id = str(uuid.uuid4())
|
||||
chunks = _chunk_text(text)
|
||||
|
||||
for idx, chunk in enumerate(chunks):
|
||||
store.store(
|
||||
chunk,
|
||||
source="upload",
|
||||
doc_type="paste",
|
||||
doc_id=doc_id,
|
||||
title=body.title or "Pasted text",
|
||||
chunk_index=idx,
|
||||
)
|
||||
|
||||
logger.info("Ingested %d chunks from pasted text (doc_id=%s)", len(chunks), doc_id)
|
||||
return IngestResponse(chunks_added=len(chunks))
|
||||
|
||||
|
||||
@router.post("/ingest/files", response_model=IngestResponse)
|
||||
async def ingest_files(
|
||||
files: List[UploadFile] = File(...),
|
||||
title: Optional[str] = Form(None),
|
||||
) -> IngestResponse:
|
||||
"""Ingest uploaded files into the knowledge store."""
|
||||
store = _get_store()
|
||||
total_chunks = 0
|
||||
|
||||
for upload in files:
|
||||
filename = upload.filename or "untitled"
|
||||
ext = ""
|
||||
if "." in filename:
|
||||
ext = "." + filename.rsplit(".", 1)[-1].lower()
|
||||
|
||||
if ext not in _ALLOWED_EXTENSIONS:
|
||||
allowed = ", ".join(sorted(_ALLOWED_EXTENSIONS))
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
f"Unsupported file type: {ext}. "
|
||||
f"Allowed: {allowed}"
|
||||
),
|
||||
)
|
||||
|
||||
data = await upload.read()
|
||||
|
||||
# Parse content based on extension
|
||||
if ext in (".txt", ".md", ".csv"):
|
||||
try:
|
||||
text = data.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
text = data.decode("latin-1")
|
||||
elif ext == ".pdf":
|
||||
text = _extract_text_from_pdf(data)
|
||||
elif ext == ".docx":
|
||||
text = _extract_text_from_docx(data)
|
||||
else:
|
||||
continue
|
||||
|
||||
text = text.strip()
|
||||
if not text:
|
||||
continue
|
||||
|
||||
doc_id = str(uuid.uuid4())
|
||||
doc_title = title or filename
|
||||
chunks = _chunk_text(text)
|
||||
|
||||
for idx, chunk in enumerate(chunks):
|
||||
store.store(
|
||||
chunk,
|
||||
source="upload",
|
||||
doc_type=ext.lstrip("."),
|
||||
doc_id=doc_id,
|
||||
title=doc_title,
|
||||
chunk_index=idx,
|
||||
)
|
||||
|
||||
total_chunks += len(chunks)
|
||||
logger.info(
|
||||
"Ingested %d chunks from file %s (doc_id=%s)",
|
||||
len(chunks),
|
||||
filename,
|
||||
doc_id,
|
||||
)
|
||||
|
||||
return IngestResponse(chunks_added=total_chunks)
|
||||
Reference in New Issue
Block a user