Personal AI, On Personal Devices
Python
feat: add document import via paste text or file upload
Jon Saad-Falconbc0153141b1c
diff --git a/frontend/src/pages/DataSourcesPage.tsx b/frontend/src/pages/DataSourcesPage.tsxindex 1033cc2f..be729b74 100644--- a/frontend/src/pages/DataSourcesPage.tsx+++ b/frontend/src/pages/DataSourcesPage.tsx@@ -10,6 +10,7 @@ import {sendblueHealth,} from '../lib/api';import type { ChannelBinding, ManagedAgent } from '../lib/api';+import { getBase } from '../lib/api';import { Database, MessageSquare, Loader2 } from 'lucide-react';import { SOURCE_CATALOG } from '../types/connectors';import type { ConnectRequest } from '../types/connectors';@@ -87,6 +88,191 @@ function InlineConnectForm({);}+// ---------------------------------------------------------------------------+// Upload / Paste form+// ---------------------------------------------------------------------------++const ACCEPTED_EXTENSIONS = '.txt,.md,.pdf,.docx,.csv';++function UploadForm({ onDone }: { onDone?: () => void }) {+ const [tab, setTab] = useState<'paste' | 'upload'>('paste');+ const [title, setTitle] = useState('');+ const [content, setContent] = useState('');+ const [files, setFiles] = useState<File[]>([]);+ const [busy, setBusy] = useState(false);+ const [result, setResult] = useState('');+ const [error, setError] = useState('');++ const handlePaste = async () => {+ if (!content.trim()) return;+ setBusy(true);+ setError('');+ setResult('');+ try {+ const res = await fetch(`${getBase()}/v1/connectors/upload/ingest`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({ title: title.trim(), content }),+ });+ if (!res.ok) {+ const err = await res.json().catch(() => ({ detail: res.statusText }));+ throw new Error(err.detail || `Upload failed: ${res.status}`);+ }+ const data = await res.json();+ setResult(`Added ${data.chunks_added} chunk${data.chunks_added !== 1 ? 's' : ''} to knowledge base`);+ setTitle('');+ setContent('');+ onDone?.();+ } catch (err: any) {+ setError(err.message || 'Upload failed');+ } finally {+ setBusy(false);+ }+ };++ const handleUpload = async () => {+ if (files.length === 0) return;+ setBusy(true);+ setError('');+ setResult('');+ try {+ const formData = new FormData();+ for (const f of files) formData.append('files', f);+ if (title.trim()) formData.append('title', title.trim());++ const res = await fetch(`${getBase()}/v1/connectors/upload/ingest/files`, {+ method: 'POST',+ body: formData,+ });+ if (!res.ok) {+ const err = await res.json().catch(() => ({ detail: res.statusText }));+ throw new Error(err.detail || `Upload failed: ${res.status}`);+ }+ const data = await res.json();+ setResult(`Added ${data.chunks_added} chunk${data.chunks_added !== 1 ? 's' : ''} from ${files.length} file${files.length !== 1 ? 's' : ''}`);+ setFiles([]);+ setTitle('');+ onDone?.();+ } catch (err: any) {+ setError(err.message || 'Upload failed');+ } finally {+ setBusy(false);+ }+ };++ const tabStyle = (active: boolean): React.CSSProperties => ({+ flex: 1, padding: '6px 0', textAlign: 'center',+ fontSize: 12, fontWeight: 600, cursor: 'pointer',+ background: active ? '#7c3aed' : 'transparent',+ color: active ? 'white' : 'var(--color-text-secondary)',+ border: 'none', borderRadius: 4,+ });++ const inputStyle: React.CSSProperties = {+ width: '100%', padding: '7px 10px',+ background: 'var(--color-bg)',+ border: '1px solid var(--color-border)',+ borderRadius: 4, color: 'var(--color-text)',+ fontSize: 12, marginBottom: 6,+ boxSizing: 'border-box' as const,+ };++ return (+ <div>+ {/* Tab bar */}+ <div style={{ display: 'flex', gap: 4, marginBottom: 10,+ background: 'var(--color-bg)', borderRadius: 6, padding: 2 }}>+ <button style={tabStyle(tab === 'paste')} onClick={() => setTab('paste')}>+ Paste Text+ </button>+ <button style={tabStyle(tab === 'upload')} onClick={() => setTab('upload')}>+ Upload Files+ </button>+ </div>++ {/* Title input (shared) */}+ <input+ value={title}+ onChange={(e) => setTitle(e.target.value)}+ placeholder="Title (optional)"+ style={inputStyle}+ />++ {tab === 'paste' && (+ <>+ <textarea+ value={content}+ onChange={(e) => setContent(e.target.value)}+ placeholder="Paste your text here..."+ rows={6}+ style={{+ ...inputStyle,+ resize: 'vertical',+ fontFamily: 'inherit',+ minHeight: 100,+ }}+ />+ <button+ onClick={handlePaste}+ disabled={busy || !content.trim()}+ style={{+ width: '100%', padding: 8,+ background: busy || !content.trim() ? '#444' : '#7c3aed',+ color: 'white', border: 'none',+ borderRadius: 6, fontSize: 12, cursor: 'pointer',+ }}+ >+ {busy ? 'Adding...' : 'Add to Knowledge Base'}+ </button>+ </>+ )}++ {tab === 'upload' && (+ <>+ <input+ type="file"+ multiple+ accept={ACCEPTED_EXTENSIONS}+ onChange={(e) => {+ const selected = Array.from(e.target.files || []);+ setFiles(selected);+ }}+ style={{ ...inputStyle, padding: 6 }}+ />+ {files.length > 0 && (+ <div style={{ fontSize: 11, color: 'var(--color-text-secondary)', marginBottom: 6 }}>+ {files.map((f) => f.name).join(', ')}+ </div>+ )}+ <button+ onClick={handleUpload}+ disabled={busy || files.length === 0}+ style={{+ width: '100%', padding: 8,+ background: busy || files.length === 0 ? '#444' : '#7c3aed',+ color: 'white', border: 'none',+ borderRadius: 6, fontSize: 12, cursor: 'pointer',+ }}+ >+ {busy ? 'Uploading...' : 'Upload & Index'}+ </button>+ </>+ )}++ {result && (+ <div style={{ fontSize: 12, color: '#4ade80', marginTop: 8 }}>+ {result}+ </div>+ )}+ {error && (+ <div style={{ fontSize: 12, color: '#ef4444', marginTop: 8 }}>+ {error}+ </div>+ )}+ </div>+ );+}+// ---------------------------------------------------------------------------// Icon map// ---------------------------------------------------------------------------@@ -96,7 +282,7 @@ const iconMap: Record<string, string> = {imessage: '\uD83D\uDCAC', gdrive: '\uD83D\uDCC1', notion: '\uD83D\uDCC4',obsidian: '\uD83D\uDCC1', granola: '\uD83C\uDF99\uFE0F', gcalendar: '\uD83D\uDCC5',gcontacts: '\uD83D\uDCC7', outlook: '\u2709\uFE0F', apple_notes: '\uD83C\uDF4E',- dropbox: '\uD83D\uDCE6', whatsapp: '\uD83D\uDCF1',+ dropbox: '\uD83D\uDCE6', whatsapp: '\uD83D\uDCF1', upload: '\uD83D\uDCC2',};// ---------------------------------------------------------------------------@@ -383,7 +569,12 @@ function DataSourcesSection() {};const connected = connectors.filter((c) => c.connected);- const notConnected = connectors.filter((c) => !c.connected);+ const notConnectedBase = connectors.filter((c) => !c.connected);+ // Always show the upload card in the not-connected list (it has no backend connector)+ const uploadEntry = { connector_id: 'upload', display_name: 'Upload / Paste', connected: false, chunks: 0 };+ const notConnected = notConnectedBase.some((c) => c.connector_id === 'upload')+ ? notConnectedBase+ : [...notConnectedBase, uploadEntry];return (<div>@@ -531,7 +722,16 @@ function DataSourcesSection() {</span></div>- {isExpanded && meta?.steps && (+ {isExpanded && c.connector_id === 'upload' && (+ <div style={{ borderTop: '1px solid var(--color-border)', padding: 12 }}>+ <div style={{ fontSize: 12, color: 'var(--color-text-secondary)', marginBottom: 10 }}>+ Paste text or upload files (.txt, .md, .pdf, .docx, .csv) to add them to your knowledge base.+ </div>+ <UploadForm onDone={loadConnectors} />+ </div>+ )}++ {isExpanded && c.connector_id !== 'upload' && meta?.steps && (<div style={{ borderTop: '1px solid var(--color-border)', padding: 12 }}>{meta.steps.map((step, i) => (<divdiff --git a/frontend/src/types/connectors.ts b/frontend/src/types/connectors.tsindex d74c5046..48af7daa 100644--- a/frontend/src/types/connectors.ts+++ b/frontend/src/types/connectors.ts@@ -8,7 +8,7 @@ export interface ConnectorMeta {connector_id: string;display_name: string;auth_type: 'oauth' | 'local' | 'bridge' | 'filesystem';- category: 'communication' | 'documents' | 'pim';+ category: 'communication' | 'documents' | 'pim' | 'other';icon: string;color: string;description: string;@@ -53,7 +53,24 @@ export type WizardStep = "pick" | "connect" | "ingest" | "ready";// Backward-compatible aliasexport type SourceCard = ConnectorMeta;+export type ConnectorCategory = ConnectorMeta['category'];+export const SOURCE_CATALOG: ConnectorMeta[] = [+ // ── Upload / Paste ─────────────────────────────────────────────────+ {+ connector_id: 'upload',+ display_name: 'Upload / Paste',+ auth_type: 'filesystem',+ category: 'other',+ icon: 'FileUp',+ color: 'text-blue-400',+ description: 'Paste text or upload documents',+ unitLabel: 'documents',+ steps: [+ { label: 'Paste text or upload files (.txt, .md, .pdf, .docx, .csv) to add them to your knowledge base.' },+ ],+ inputFields: [],+ },// ── Communication ──────────────────────────────────────────────────{connector_id: 'gmail_imap',diff --git a/src/openjarvis/server/app.py b/src/openjarvis/server/app.pyindex 549e589c..5868b204 100644--- a/src/openjarvis/server/app.py+++ b/src/openjarvis/server/app.py@@ -15,6 +15,7 @@ from openjarvis.server.comparison import comparison_routerfrom openjarvis.server.connectors_router import create_connectors_routerfrom openjarvis.server.dashboard import dashboard_routerfrom openjarvis.server.routes import router+from openjarvis.server.upload_router import router as upload_routerlogger = logging.getLogger(__name__)@@ -222,6 +223,7 @@ def create_app(app.include_router(dashboard_router)app.include_router(comparison_router)app.include_router(create_connectors_router())+ app.include_router(upload_router)include_all_routes(app)# Restore SendBlue channel bindings from database on startupdiff --git a/src/openjarvis/server/upload_router.py b/src/openjarvis/server/upload_router.pynew file mode 100644index 00000000..24f02863--- /dev/null+++ b/src/openjarvis/server/upload_router.py@@ -0,0 +1,221 @@+"""Upload / Paste router for ingesting documents into the knowledge store."""++from __future__ import annotations++import io+import logging+import uuid+from typing import List, Optional++from fastapi import APIRouter, File, Form, HTTPException, UploadFile+from pydantic import BaseModel++from openjarvis.connectors.store import KnowledgeStore+from openjarvis.core.config import DEFAULT_CONFIG_DIR++logger = logging.getLogger(__name__)++router = APIRouter(prefix="/v1/connectors/upload", tags=["upload"])++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------++_ALLOWED_EXTENSIONS = {".txt", ".md", ".csv", ".pdf", ".docx"}+++def _chunk_text(text: str, max_chars: int = 1000) -> List[str]:+ """Split *text* into ~max_chars pieces at paragraph boundaries."""+ paragraphs = text.split("\n\n")+ chunks: List[str] = []+ current = ""+ for para in paragraphs:+ para = para.strip()+ if not para:+ continue+ if current and len(current) + len(para) + 2 > max_chars:+ chunks.append(current.strip())+ current = para+ else:+ current = f"{current}\n\n{para}" if current else para+ if current.strip():+ chunks.append(current.strip())+ # Guard against very large paragraphs that exceed max_chars+ final: List[str] = []+ for chunk in chunks:+ while len(chunk) > max_chars:+ # Find last space within limit+ split_at = chunk.rfind(" ", 0, max_chars)+ if split_at == -1:+ split_at = max_chars+ final.append(chunk[:split_at].strip())+ chunk = chunk[split_at:].strip()+ if chunk:+ final.append(chunk)+ return final+++def _extract_text_from_pdf(data: bytes) -> str:+ """Extract text from a PDF using pdfplumber or PyPDF2."""+ # Try pdfplumber first+ try:+ import pdfplumber # type: ignore[import-untyped]++ with pdfplumber.open(io.BytesIO(data)) as pdf:+ pages = [p.extract_text() or "" for p in pdf.pages]+ return "\n\n".join(pages)+ except ImportError:+ pass++ # Fall back to PyPDF2+ try:+ from PyPDF2 import PdfReader # type: ignore[import-untyped]++ reader = PdfReader(io.BytesIO(data))+ pages = [p.extract_text() or "" for p in reader.pages]+ return "\n\n".join(pages)+ except ImportError:+ raise HTTPException(+ status_code=500,+ detail=(+ "PDF parsing requires pdfplumber or PyPDF2. "+ "Install one with: pip install pdfplumber"+ ),+ )+++def _extract_text_from_docx(data: bytes) -> str:+ """Extract text from a .docx file using python-docx."""+ try:+ from docx import Document # type: ignore[import-untyped]++ doc = Document(io.BytesIO(data))+ return "\n\n".join(p.text for p in doc.paragraphs if p.text.strip())+ except ImportError:+ raise HTTPException(+ status_code=500,+ detail=(+ "DOCX parsing requires python-docx. "+ "Install with: pip install python-docx"+ ),+ )+++def _get_store() -> KnowledgeStore:+ """Return a KnowledgeStore pointing at the default knowledge DB."""+ db_path = DEFAULT_CONFIG_DIR / "knowledge.db"+ return KnowledgeStore(db_path=db_path)+++# ---------------------------------------------------------------------------+# Schemas+# ---------------------------------------------------------------------------+++class PasteRequest(BaseModel):+ title: str = ""+ content: str+++class IngestResponse(BaseModel):+ chunks_added: int+ source: str = "upload"+++# ---------------------------------------------------------------------------+# Routes+# ---------------------------------------------------------------------------++[email protected]("/ingest", response_model=IngestResponse)+async def ingest_paste(body: PasteRequest) -> IngestResponse:+ """Ingest pasted text into the knowledge store."""+ text = body.content.strip()+ if not text:+ raise HTTPException(status_code=400, detail="Content is empty")++ store = _get_store()+ doc_id = str(uuid.uuid4())+ chunks = _chunk_text(text)++ for idx, chunk in enumerate(chunks):+ store.store(+ chunk,+ source="upload",+ doc_type="paste",+ doc_id=doc_id,+ title=body.title or "Pasted text",+ chunk_index=idx,+ )++ logger.info("Ingested %d chunks from pasted text (doc_id=%s)", len(chunks), doc_id)+ return IngestResponse(chunks_added=len(chunks))++[email protected]("/ingest/files", response_model=IngestResponse)+async def ingest_files(+ files: List[UploadFile] = File(...),+ title: Optional[str] = Form(None),+) -> IngestResponse:+ """Ingest uploaded files into the knowledge store."""+ store = _get_store()+ total_chunks = 0++ for upload in files:+ filename = upload.filename or "untitled"+ ext = ""+ if "." in filename:+ ext = "." + filename.rsplit(".", 1)[-1].lower()++ if ext not in _ALLOWED_EXTENSIONS:+ allowed = ", ".join(sorted(_ALLOWED_EXTENSIONS))+ raise HTTPException(+ status_code=400,+ detail=(+ f"Unsupported file type: {ext}. "+ f"Allowed: {allowed}"+ ),+ )++ data = await upload.read()++ # Parse content based on extension+ if ext in (".txt", ".md", ".csv"):+ try:+ text = data.decode("utf-8")+ except UnicodeDecodeError:+ text = data.decode("latin-1")+ elif ext == ".pdf":+ text = _extract_text_from_pdf(data)+ elif ext == ".docx":+ text = _extract_text_from_docx(data)+ else:+ continue++ text = text.strip()+ if not text:+ continue++ doc_id = str(uuid.uuid4())+ doc_title = title or filename+ chunks = _chunk_text(text)++ for idx, chunk in enumerate(chunks):+ store.store(+ chunk,+ source="upload",+ doc_type=ext.lstrip("."),+ doc_id=doc_id,+ title=doc_title,+ chunk_index=idx,+ )++ total_chunks += len(chunks)+ logger.info(+ "Ingested %d chunks from file %s (doc_id=%s)",+ len(chunks),+ filename,+ doc_id,+ )++ return IngestResponse(chunks_added=total_chunks)