Skip to content

Personal AI, On Personal Devices

Python

feat: add document import via paste text or file upload

Jon Saad-Falconbc0153141b1c
diff --git a/frontend/src/pages/DataSourcesPage.tsx b/frontend/src/pages/DataSourcesPage.tsx
index 1033cc2f..be729b74 100644
--- a/frontend/src/pages/DataSourcesPage.tsx
+++ b/frontend/src/pages/DataSourcesPage.tsx
@@ -10,6 +10,7 @@ import {
sendblueHealth,
} from '../lib/api';
import type { ChannelBinding, ManagedAgent } from '../lib/api';
+import { getBase } from '../lib/api';
import { Database, MessageSquare, Loader2 } from 'lucide-react';
import { SOURCE_CATALOG } from '../types/connectors';
import type { ConnectRequest } from '../types/connectors';
@@ -87,6 +88,191 @@ function InlineConnectForm({
);
}
+// ---------------------------------------------------------------------------
+// Upload / Paste form
+// ---------------------------------------------------------------------------
+
+const ACCEPTED_EXTENSIONS = '.txt,.md,.pdf,.docx,.csv';
+
+function UploadForm({ onDone }: { onDone?: () => void }) {
+ const [tab, setTab] = useState<'paste' | 'upload'>('paste');
+ const [title, setTitle] = useState('');
+ const [content, setContent] = useState('');
+ const [files, setFiles] = useState<File[]>([]);
+ const [busy, setBusy] = useState(false);
+ const [result, setResult] = useState('');
+ const [error, setError] = useState('');
+
+ const handlePaste = async () => {
+ if (!content.trim()) return;
+ setBusy(true);
+ setError('');
+ setResult('');
+ try {
+ const res = await fetch(`${getBase()}/v1/connectors/upload/ingest`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ title: title.trim(), content }),
+ });
+ if (!res.ok) {
+ const err = await res.json().catch(() => ({ detail: res.statusText }));
+ throw new Error(err.detail || `Upload failed: ${res.status}`);
+ }
+ const data = await res.json();
+ setResult(`Added ${data.chunks_added} chunk${data.chunks_added !== 1 ? 's' : ''} to knowledge base`);
+ setTitle('');
+ setContent('');
+ onDone?.();
+ } catch (err: any) {
+ setError(err.message || 'Upload failed');
+ } finally {
+ setBusy(false);
+ }
+ };
+
+ const handleUpload = async () => {
+ if (files.length === 0) return;
+ setBusy(true);
+ setError('');
+ setResult('');
+ try {
+ const formData = new FormData();
+ for (const f of files) formData.append('files', f);
+ if (title.trim()) formData.append('title', title.trim());
+
+ const res = await fetch(`${getBase()}/v1/connectors/upload/ingest/files`, {
+ method: 'POST',
+ body: formData,
+ });
+ if (!res.ok) {
+ const err = await res.json().catch(() => ({ detail: res.statusText }));
+ throw new Error(err.detail || `Upload failed: ${res.status}`);
+ }
+ const data = await res.json();
+ setResult(`Added ${data.chunks_added} chunk${data.chunks_added !== 1 ? 's' : ''} from ${files.length} file${files.length !== 1 ? 's' : ''}`);
+ setFiles([]);
+ setTitle('');
+ onDone?.();
+ } catch (err: any) {
+ setError(err.message || 'Upload failed');
+ } finally {
+ setBusy(false);
+ }
+ };
+
+ const tabStyle = (active: boolean): React.CSSProperties => ({
+ flex: 1, padding: '6px 0', textAlign: 'center',
+ fontSize: 12, fontWeight: 600, cursor: 'pointer',
+ background: active ? '#7c3aed' : 'transparent',
+ color: active ? 'white' : 'var(--color-text-secondary)',
+ border: 'none', borderRadius: 4,
+ });
+
+ const inputStyle: React.CSSProperties = {
+ width: '100%', padding: '7px 10px',
+ background: 'var(--color-bg)',
+ border: '1px solid var(--color-border)',
+ borderRadius: 4, color: 'var(--color-text)',
+ fontSize: 12, marginBottom: 6,
+ boxSizing: 'border-box' as const,
+ };
+
+ return (
+ <div>
+ {/* Tab bar */}
+ <div style={{ display: 'flex', gap: 4, marginBottom: 10,
+ background: 'var(--color-bg)', borderRadius: 6, padding: 2 }}>
+ <button style={tabStyle(tab === 'paste')} onClick={() => setTab('paste')}>
+ Paste Text
+ </button>
+ <button style={tabStyle(tab === 'upload')} onClick={() => setTab('upload')}>
+ Upload Files
+ </button>
+ </div>
+
+ {/* Title input (shared) */}
+ <input
+ value={title}
+ onChange={(e) => setTitle(e.target.value)}
+ placeholder="Title (optional)"
+ style={inputStyle}
+ />
+
+ {tab === 'paste' && (
+ <>
+ <textarea
+ value={content}
+ onChange={(e) => setContent(e.target.value)}
+ placeholder="Paste your text here..."
+ rows={6}
+ style={{
+ ...inputStyle,
+ resize: 'vertical',
+ fontFamily: 'inherit',
+ minHeight: 100,
+ }}
+ />
+ <button
+ onClick={handlePaste}
+ disabled={busy || !content.trim()}
+ style={{
+ width: '100%', padding: 8,
+ background: busy || !content.trim() ? '#444' : '#7c3aed',
+ color: 'white', border: 'none',
+ borderRadius: 6, fontSize: 12, cursor: 'pointer',
+ }}
+ >
+ {busy ? 'Adding...' : 'Add to Knowledge Base'}
+ </button>
+ </>
+ )}
+
+ {tab === 'upload' && (
+ <>
+ <input
+ type="file"
+ multiple
+ accept={ACCEPTED_EXTENSIONS}
+ onChange={(e) => {
+ const selected = Array.from(e.target.files || []);
+ setFiles(selected);
+ }}
+ style={{ ...inputStyle, padding: 6 }}
+ />
+ {files.length > 0 && (
+ <div style={{ fontSize: 11, color: 'var(--color-text-secondary)', marginBottom: 6 }}>
+ {files.map((f) => f.name).join(', ')}
+ </div>
+ )}
+ <button
+ onClick={handleUpload}
+ disabled={busy || files.length === 0}
+ style={{
+ width: '100%', padding: 8,
+ background: busy || files.length === 0 ? '#444' : '#7c3aed',
+ color: 'white', border: 'none',
+ borderRadius: 6, fontSize: 12, cursor: 'pointer',
+ }}
+ >
+ {busy ? 'Uploading...' : 'Upload & Index'}
+ </button>
+ </>
+ )}
+
+ {result && (
+ <div style={{ fontSize: 12, color: '#4ade80', marginTop: 8 }}>
+ {result}
+ </div>
+ )}
+ {error && (
+ <div style={{ fontSize: 12, color: '#ef4444', marginTop: 8 }}>
+ {error}
+ </div>
+ )}
+ </div>
+ );
+}
+
// ---------------------------------------------------------------------------
// Icon map
// ---------------------------------------------------------------------------
@@ -96,7 +282,7 @@ const iconMap: Record<string, string> = {
imessage: '\uD83D\uDCAC', gdrive: '\uD83D\uDCC1', notion: '\uD83D\uDCC4',
obsidian: '\uD83D\uDCC1', granola: '\uD83C\uDF99\uFE0F', gcalendar: '\uD83D\uDCC5',
gcontacts: '\uD83D\uDCC7', outlook: '\u2709\uFE0F', apple_notes: '\uD83C\uDF4E',
- dropbox: '\uD83D\uDCE6', whatsapp: '\uD83D\uDCF1',
+ dropbox: '\uD83D\uDCE6', whatsapp: '\uD83D\uDCF1', upload: '\uD83D\uDCC2',
};
// ---------------------------------------------------------------------------
@@ -383,7 +569,12 @@ function DataSourcesSection() {
};
const connected = connectors.filter((c) => c.connected);
- const notConnected = connectors.filter((c) => !c.connected);
+ const notConnectedBase = connectors.filter((c) => !c.connected);
+ // Always show the upload card in the not-connected list (it has no backend connector)
+ const uploadEntry = { connector_id: 'upload', display_name: 'Upload / Paste', connected: false, chunks: 0 };
+ const notConnected = notConnectedBase.some((c) => c.connector_id === 'upload')
+ ? notConnectedBase
+ : [...notConnectedBase, uploadEntry];
return (
<div>
@@ -531,7 +722,16 @@ function DataSourcesSection() {
</span>
</div>
- {isExpanded && meta?.steps && (
+ {isExpanded && c.connector_id === 'upload' && (
+ <div style={{ borderTop: '1px solid var(--color-border)', padding: 12 }}>
+ <div style={{ fontSize: 12, color: 'var(--color-text-secondary)', marginBottom: 10 }}>
+ Paste text or upload files (.txt, .md, .pdf, .docx, .csv) to add them to your knowledge base.
+ </div>
+ <UploadForm onDone={loadConnectors} />
+ </div>
+ )}
+
+ {isExpanded && c.connector_id !== 'upload' && meta?.steps && (
<div style={{ borderTop: '1px solid var(--color-border)', padding: 12 }}>
{meta.steps.map((step, i) => (
<div
diff --git a/frontend/src/types/connectors.ts b/frontend/src/types/connectors.ts
index d74c5046..48af7daa 100644
--- a/frontend/src/types/connectors.ts
+++ b/frontend/src/types/connectors.ts
@@ -8,7 +8,7 @@ export interface ConnectorMeta {
connector_id: string;
display_name: string;
auth_type: 'oauth' | 'local' | 'bridge' | 'filesystem';
- category: 'communication' | 'documents' | 'pim';
+ category: 'communication' | 'documents' | 'pim' | 'other';
icon: string;
color: string;
description: string;
@@ -53,7 +53,24 @@ export type WizardStep = "pick" | "connect" | "ingest" | "ready";
// Backward-compatible alias
export type SourceCard = ConnectorMeta;
+export type ConnectorCategory = ConnectorMeta['category'];
+
export const SOURCE_CATALOG: ConnectorMeta[] = [
+ // ── Upload / Paste ─────────────────────────────────────────────────
+ {
+ connector_id: 'upload',
+ display_name: 'Upload / Paste',
+ auth_type: 'filesystem',
+ category: 'other',
+ icon: 'FileUp',
+ color: 'text-blue-400',
+ description: 'Paste text or upload documents',
+ unitLabel: 'documents',
+ steps: [
+ { label: 'Paste text or upload files (.txt, .md, .pdf, .docx, .csv) to add them to your knowledge base.' },
+ ],
+ inputFields: [],
+ },
// ── Communication ──────────────────────────────────────────────────
{
connector_id: 'gmail_imap',
diff --git a/src/openjarvis/server/app.py b/src/openjarvis/server/app.py
index 549e589c..5868b204 100644
--- a/src/openjarvis/server/app.py
+++ b/src/openjarvis/server/app.py
@@ -15,6 +15,7 @@ from openjarvis.server.comparison import comparison_router
from openjarvis.server.connectors_router import create_connectors_router
from openjarvis.server.dashboard import dashboard_router
from openjarvis.server.routes import router
+from openjarvis.server.upload_router import router as upload_router
logger = logging.getLogger(__name__)
@@ -222,6 +223,7 @@ def create_app(
app.include_router(dashboard_router)
app.include_router(comparison_router)
app.include_router(create_connectors_router())
+ app.include_router(upload_router)
include_all_routes(app)
# Restore SendBlue channel bindings from database on startup
diff --git a/src/openjarvis/server/upload_router.py b/src/openjarvis/server/upload_router.py
new file mode 100644
index 00000000..24f02863
--- /dev/null
+++ b/src/openjarvis/server/upload_router.py
@@ -0,0 +1,221 @@
+"""Upload / Paste router for ingesting documents into the knowledge store."""
+
+from __future__ import annotations
+
+import io
+import logging
+import uuid
+from typing import List, Optional
+
+from fastapi import APIRouter, File, Form, HTTPException, UploadFile
+from pydantic import BaseModel
+
+from openjarvis.connectors.store import KnowledgeStore
+from openjarvis.core.config import DEFAULT_CONFIG_DIR
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/v1/connectors/upload", tags=["upload"])
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+_ALLOWED_EXTENSIONS = {".txt", ".md", ".csv", ".pdf", ".docx"}
+
+
+def _chunk_text(text: str, max_chars: int = 1000) -> List[str]:
+ """Split *text* into ~max_chars pieces at paragraph boundaries."""
+ paragraphs = text.split("\n\n")
+ chunks: List[str] = []
+ current = ""
+ for para in paragraphs:
+ para = para.strip()
+ if not para:
+ continue
+ if current and len(current) + len(para) + 2 > max_chars:
+ chunks.append(current.strip())
+ current = para
+ else:
+ current = f"{current}\n\n{para}" if current else para
+ if current.strip():
+ chunks.append(current.strip())
+ # Guard against very large paragraphs that exceed max_chars
+ final: List[str] = []
+ for chunk in chunks:
+ while len(chunk) > max_chars:
+ # Find last space within limit
+ split_at = chunk.rfind(" ", 0, max_chars)
+ if split_at == -1:
+ split_at = max_chars
+ final.append(chunk[:split_at].strip())
+ chunk = chunk[split_at:].strip()
+ if chunk:
+ final.append(chunk)
+ return final
+
+
+def _extract_text_from_pdf(data: bytes) -> str:
+ """Extract text from a PDF using pdfplumber or PyPDF2."""
+ # Try pdfplumber first
+ try:
+ import pdfplumber # type: ignore[import-untyped]
+
+ with pdfplumber.open(io.BytesIO(data)) as pdf:
+ pages = [p.extract_text() or "" for p in pdf.pages]
+ return "\n\n".join(pages)
+ except ImportError:
+ pass
+
+ # Fall back to PyPDF2
+ try:
+ from PyPDF2 import PdfReader # type: ignore[import-untyped]
+
+ reader = PdfReader(io.BytesIO(data))
+ pages = [p.extract_text() or "" for p in reader.pages]
+ return "\n\n".join(pages)
+ except ImportError:
+ raise HTTPException(
+ status_code=500,
+ detail=(
+ "PDF parsing requires pdfplumber or PyPDF2. "
+ "Install one with: pip install pdfplumber"
+ ),
+ )
+
+
+def _extract_text_from_docx(data: bytes) -> str:
+ """Extract text from a .docx file using python-docx."""
+ try:
+ from docx import Document # type: ignore[import-untyped]
+
+ doc = Document(io.BytesIO(data))
+ return "\n\n".join(p.text for p in doc.paragraphs if p.text.strip())
+ except ImportError:
+ raise HTTPException(
+ status_code=500,
+ detail=(
+ "DOCX parsing requires python-docx. "
+ "Install with: pip install python-docx"
+ ),
+ )
+
+
+def _get_store() -> KnowledgeStore:
+ """Return a KnowledgeStore pointing at the default knowledge DB."""
+ db_path = DEFAULT_CONFIG_DIR / "knowledge.db"
+ return KnowledgeStore(db_path=db_path)
+
+
+# ---------------------------------------------------------------------------
+# Schemas
+# ---------------------------------------------------------------------------
+
+
+class PasteRequest(BaseModel):
+ title: str = ""
+ content: str
+
+
+class IngestResponse(BaseModel):
+ chunks_added: int
+ source: str = "upload"
+
+
+# ---------------------------------------------------------------------------
+# Routes
+# ---------------------------------------------------------------------------
+
+
[email protected]("/ingest", response_model=IngestResponse)
+async def ingest_paste(body: PasteRequest) -> IngestResponse:
+ """Ingest pasted text into the knowledge store."""
+ text = body.content.strip()
+ if not text:
+ raise HTTPException(status_code=400, detail="Content is empty")
+
+ store = _get_store()
+ doc_id = str(uuid.uuid4())
+ chunks = _chunk_text(text)
+
+ for idx, chunk in enumerate(chunks):
+ store.store(
+ chunk,
+ source="upload",
+ doc_type="paste",
+ doc_id=doc_id,
+ title=body.title or "Pasted text",
+ chunk_index=idx,
+ )
+
+ logger.info("Ingested %d chunks from pasted text (doc_id=%s)", len(chunks), doc_id)
+ return IngestResponse(chunks_added=len(chunks))
+
+
[email protected]("/ingest/files", response_model=IngestResponse)
+async def ingest_files(
+ files: List[UploadFile] = File(...),
+ title: Optional[str] = Form(None),
+) -> IngestResponse:
+ """Ingest uploaded files into the knowledge store."""
+ store = _get_store()
+ total_chunks = 0
+
+ for upload in files:
+ filename = upload.filename or "untitled"
+ ext = ""
+ if "." in filename:
+ ext = "." + filename.rsplit(".", 1)[-1].lower()
+
+ if ext not in _ALLOWED_EXTENSIONS:
+ allowed = ", ".join(sorted(_ALLOWED_EXTENSIONS))
+ raise HTTPException(
+ status_code=400,
+ detail=(
+ f"Unsupported file type: {ext}. "
+ f"Allowed: {allowed}"
+ ),
+ )
+
+ data = await upload.read()
+
+ # Parse content based on extension
+ if ext in (".txt", ".md", ".csv"):
+ try:
+ text = data.decode("utf-8")
+ except UnicodeDecodeError:
+ text = data.decode("latin-1")
+ elif ext == ".pdf":
+ text = _extract_text_from_pdf(data)
+ elif ext == ".docx":
+ text = _extract_text_from_docx(data)
+ else:
+ continue
+
+ text = text.strip()
+ if not text:
+ continue
+
+ doc_id = str(uuid.uuid4())
+ doc_title = title or filename
+ chunks = _chunk_text(text)
+
+ for idx, chunk in enumerate(chunks):
+ store.store(
+ chunk,
+ source="upload",
+ doc_type=ext.lstrip("."),
+ doc_id=doc_id,
+ title=doc_title,
+ chunk_index=idx,
+ )
+
+ total_chunks += len(chunks)
+ logger.info(
+ "Ingested %d chunks from file %s (doc_id=%s)",
+ len(chunks),
+ filename,
+ doc_id,
+ )
+
+ return IngestResponse(chunks_added=total_chunks)