RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(connector): add Zotero as a data source (#19742)
Sourav Rajvi5e0bcd50c406
diff --git a/common/constants.py b/common/constants.pyindex 46eededed2..6080e687b9 100644--- a/common/constants.py+++ b/common/constants.py@@ -171,6 +171,7 @@ class FileSource(StrEnum):BITBUCKET = "bitbucket"AZURE_DEVOPS = "azure_devops"ZENDESK = "zendesk"+ ZOTERO = "zotero"SEAFILE = "seafile"MYSQL = "mysql"POSTGRESQL = "postgresql"diff --git a/common/data_source/__init__.py b/common/data_source/__init__.pyindex ec79f4d2a7..21d8a21238 100644--- a/common/data_source/__init__.py+++ b/common/data_source/__init__.py@@ -70,6 +70,7 @@ from .teams_connector import TeamsConnectorfrom .webdav_connector import WebDAVConnectorfrom .xquik_connector import XquikConnectorfrom .zendesk_connector import ZendeskConnector+from .zotero_connector import ZoteroConnectorCONNECTOR_BY_SOURCE: dict[str, type] = {FileSource.S3: BlobStorageConnector,@@ -97,6 +98,7 @@ CONNECTOR_BY_SOURCE: dict[str, type] = {FileSource.GITHUB: GithubConnector,FileSource.IMAP: ImapConnector,FileSource.ZENDESK: ZendeskConnector,+ FileSource.ZOTERO: ZoteroConnector,FileSource.GITLAB: GitlabConnector,FileSource.BITBUCKET: BitbucketConnector,FileSource.AZURE_DEVOPS: AzureDevOpsConnector,@@ -173,5 +175,6 @@ __all__ = ["WebDAVConnector","XquikConnector","ZendeskConnector",+ "ZoteroConnector","build_connector_for_source",]diff --git a/common/data_source/config.py b/common/data_source/config.pyindex 2198a4f360..31a07143ea 100644--- a/common/data_source/config.py+++ b/common/data_source/config.py@@ -72,6 +72,7 @@ class DocumentSource(str, Enum):IMAP = "imap"BITBUCKET = "bitbucket"ZENDESK = "zendesk"+ ZOTERO = "zotero"SEAFILE = "seafile"MYSQL = "mysql"POSTGRESQL = "postgresql"diff --git a/common/data_source/zotero_connector.py b/common/data_source/zotero_connector.pynew file mode 100644index 0000000000..8ed265ea0b--- /dev/null+++ b/common/data_source/zotero_connector.py@@ -0,0 +1,357 @@+"""Zotero library connector for syncing PDF attachments."""++from __future__ import annotations++import io+import logging+import time+import zipfile+from collections.abc import Generator+from datetime import datetime, timezone+from typing import Any+from urllib.parse import urljoin, urlparse++from common.data_source.config import BLOB_STORAGE_SIZE_THRESHOLD, INDEX_BATCH_SIZE, DocumentSource+from common.data_source.exceptions import (+ ConnectorMissingCredentialError,+ ConnectorValidationError,+ CredentialExpiredError,+ InsufficientPermissionsError,+)+from common.data_source.interfaces import LoadConnector, PollConnector, SecondsSinceUnixEpoch, SlimConnectorWithPermSync+from common.data_source.models import Document, GenerateDocumentsOutput, GenerateSlimDocumentOutput, SlimDocument+from common.data_source.utils import rl_requests+from common.ssrf_guard import assert_url_is_safe, pin_dns++logger = logging.getLogger(__name__)++ZOTERO_API_BASE = "https://api.zotero.org"+DEFAULT_WEBDAV_URL = ""+STORAGE_MODE_ZOTERO = "zotero_storage"+STORAGE_MODE_WEBDAV = "webdav"+PAGE_SIZE = 100+ZOTERO_API_KEY_HOSTS = frozenset({"api.zotero.org"})+MAX_FILE_REDIRECTS = 10+_READ_CHUNK_BYTES = 64 * 1024+MAX_DOWNLOAD_ATTEMPTS = 3+DOWNLOAD_RETRY_BACKOFF_SEC = 0.5+++def _read_response_capped(response, max_bytes: int) -> bytes:+ """Read a response body up to max_bytes without loading unbounded data."""+ chunks: list[bytes] = []+ total = 0+ try:+ for chunk in response.iter_content(chunk_size=_READ_CHUNK_BYTES):+ if not chunk:+ continue+ total += len(chunk)+ if total > max_bytes:+ response.close()+ return b""+ chunks.append(chunk)+ finally:+ response.close()+ return b"".join(chunks)+++class ZoteroConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):+ def __init__(+ self,+ zotero_user_id: str | None,+ storage_mode: str = STORAGE_MODE_ZOTERO,+ webdav_url: str = DEFAULT_WEBDAV_URL,+ batch_size: int = INDEX_BATCH_SIZE,+ ) -> None:+ self.user_id = (zotero_user_id or "").strip()+ self.storage_mode = (storage_mode or STORAGE_MODE_ZOTERO).strip()+ self.webdav_url = (webdav_url or DEFAULT_WEBDAV_URL).rstrip("/")+ self.batch_size = batch_size+ self.api_key: str | None = None+ self.webdav_username: str | None = None+ self.webdav_password: str | None = None+ self.size_threshold = BLOB_STORAGE_SIZE_THRESHOLD++ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "ZoteroConnector":+ credentials = config.get("credentials") or {}+ user_id = (config.get("zotero_user_id") or credentials.get("zotero_user_id") or "").strip()+ connector = cls(+ zotero_user_id=user_id,+ storage_mode=config.get("storage_mode", STORAGE_MODE_ZOTERO),+ webdav_url=config.get("webdav_url", DEFAULT_WEBDAV_URL),+ batch_size=int(config.get("batch_size") or INDEX_BATCH_SIZE),+ )+ connector.load_credentials(credentials)+ return connector++ def load_credentials(self, credentials: dict[str, Any]) -> None:+ api_key = (credentials.get("zotero_api_key") or "").strip()+ if not api_key:+ raise ConnectorMissingCredentialError("Zotero API key is required")+ self.api_key = api_key+ self.webdav_username = (credentials.get("webdav_username") or "").strip() or None+ self.webdav_password = credentials.get("webdav_password")++ def validate_local_settings(self) -> None:+ if not self.user_id:+ raise ConnectorMissingCredentialError("Zotero user ID is required")+ if not self.api_key:+ raise ConnectorMissingCredentialError("Zotero API key is required")+ if self.storage_mode not in {STORAGE_MODE_ZOTERO, STORAGE_MODE_WEBDAV}:+ raise ConnectorValidationError("storage_mode must be 'zotero_storage' or 'webdav'")+ if self.storage_mode != STORAGE_MODE_WEBDAV:+ return+ if not self.webdav_url:+ raise ConnectorValidationError("webdav_url is required when storage_mode is webdav")+ if urlparse(self.webdav_url).scheme.lower() != "https":+ raise ConnectorValidationError("WebDAV URL must use HTTPS")+ probe_url = urljoin(self.webdav_url + "/", "zotero/")+ try:+ assert_url_is_safe(probe_url, allowed_schemes=frozenset({"https"}))+ except ValueError as exc:+ raise ConnectorValidationError(f"WebDAV URL is not allowed: {exc}") from exc+ if not self.webdav_username:+ raise ConnectorMissingCredentialError("WebDAV username is required when storage_mode is webdav")+ if not self.webdav_password:+ raise ConnectorMissingCredentialError("WebDAV password is required when storage_mode is webdav")++ def validate_connector_settings(self) -> None:+ self.validate_local_settings()+ self._list_attachment_items(start=0)++ def load_from_state(self) -> GenerateDocumentsOutput:+ yield from self._yield_documents()++ def poll_source(self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch) -> GenerateDocumentsOutput:+ start_dt = datetime.fromtimestamp(start, tz=timezone.utc)+ end_dt = datetime.fromtimestamp(end, tz=timezone.utc)+ yield from self._yield_documents(start_dt=start_dt, end_dt=end_dt)++ def retrieve_all_slim_docs_perm_sync(self, callback: Any = None) -> GenerateSlimDocumentOutput:+ del callback+ batch: list[SlimDocument] = []+ for item in self._iter_pdf_attachments():+ batch.append(SlimDocument(id=self._document_id(item["key"])))+ if len(batch) >= self.batch_size:+ yield batch+ batch = []+ if batch:+ yield batch++ def _yield_documents(+ self,+ start_dt: datetime | None = None,+ end_dt: datetime | None = None,+ ) -> GenerateDocumentsOutput:+ documents: list[Document] = []+ for item in self._iter_pdf_attachments():+ modified = self._parse_time(item["data"].get("dateModified"))+ if start_dt and modified <= start_dt:+ continue+ if end_dt and modified > end_dt:+ continue+ blob, filename = self._download_pdf_with_retry(item)+ if not blob:+ continue+ documents.append(self._build_document(item, blob, filename, modified))+ if len(documents) >= self.batch_size:+ yield documents+ documents = []+ if documents:+ yield documents++ def _build_document(self, item: dict[str, Any], blob: bytes, filename: str, modified: datetime) -> Document:+ data = item.get("data") or {}+ title = (data.get("title") or filename or item.get("key", "")).strip()+ extension = "pdf"+ if "." in filename:+ extension = filename.rsplit(".", 1)[-1].lower()+ return Document(+ id=self._document_id(item["key"]),+ source=DocumentSource.ZOTERO.value,+ semantic_identifier=title,+ extension=extension,+ blob=blob,+ doc_updated_at=modified,+ size_bytes=len(blob),+ metadata={+ "source": "zotero",+ "zotero_item_key": item["key"],+ "zotero_parent": data.get("parentItem"),+ "filename": filename,+ },+ )++ def _document_id(self, attachment_key: str) -> str:+ return f"zotero:{self.user_id}:{attachment_key}"++ def _iter_pdf_attachments(self) -> Generator[dict[str, Any], None, None]:+ start = 0+ while True:+ items = self._list_attachment_items(start=start)+ if not items:+ break+ for item in items:+ data = item.get("data") or {}+ if data.get("itemType") != "attachment":+ continue+ content_type = (data.get("contentType") or "").lower()+ filename = (data.get("filename") or "").lower()+ if "pdf" in content_type or filename.endswith(".pdf"):+ yield item+ start += len(items)+ if len(items) < PAGE_SIZE:+ break++ def _list_attachment_items(self, start: int) -> list[dict[str, Any]]:+ url = f"{ZOTERO_API_BASE}/users/{self.user_id}/items"+ params = {"itemType": "attachment", "format": "json", "limit": PAGE_SIZE, "start": start}+ response = rl_requests.get(+ url,+ headers=self._headers(),+ params=params,+ timeout=60,+ allow_redirects=False,+ )+ if 300 <= response.status_code < 400:+ response.close()+ raise ConnectorValidationError("Unexpected redirect from Zotero API")+ if response.status_code in {401, 403}:+ raise CredentialExpiredError("Zotero API key is invalid or expired")+ if response.status_code == 404:+ raise InsufficientPermissionsError("Zotero library was not found")+ try:+ response.raise_for_status()+ payload = response.json()+ finally:+ response.close()+ if not isinstance(payload, list):+ raise ConnectorValidationError("Unexpected Zotero API response")+ return payload++ def _download_pdf_with_retry(self, item: dict[str, Any]) -> tuple[bytes, str]:+ attachment_key = item.get("key", "")+ filename = ""+ for attempt in range(1, MAX_DOWNLOAD_ATTEMPTS + 1):+ blob, filename = self._download_pdf(item)+ if blob:+ return blob, filename+ if attempt < MAX_DOWNLOAD_ATTEMPTS:+ time.sleep(DOWNLOAD_RETRY_BACKOFF_SEC * attempt)+ logger.warning(+ "Skipping Zotero attachment %s after %d failed download attempts",+ attachment_key,+ MAX_DOWNLOAD_ATTEMPTS,+ )+ return b"", filename++ def _download_pdf(self, item: dict[str, Any]) -> tuple[bytes, str]:+ data = item.get("data") or {}+ filename = (data.get("filename") or f"{data.get('title', item['key'])}.pdf").strip()+ if self.storage_mode == STORAGE_MODE_WEBDAV:+ return self._download_via_webdav(item["key"], filename)+ return self._download_via_zotero_api(item["key"], filename)++ def _download_via_zotero_api(self, attachment_key: str, filename: str) -> tuple[bytes, str]:+ url = f"{ZOTERO_API_BASE}/users/{self.user_id}/items/{attachment_key}/file"+ blob = self._get_zotero_file(url)+ if blob is None:+ return b"", filename+ if len(blob) > self.size_threshold:+ logger.warning("Skipping oversized Zotero attachment %s", attachment_key)+ return b"", filename+ return blob, filename++ def _get_zotero_file(self, start_url: str) -> bytes | None:+ current = start_url+ for _ in range(MAX_FILE_REDIRECTS):+ parsed = urlparse(current)+ headers = {"Zotero-API-Version": "3"}+ if (parsed.hostname or "").lower() in ZOTERO_API_KEY_HOSTS:+ headers["Zotero-API-Key"] = self.api_key or ""+ try:+ host, ip = assert_url_is_safe(current, allowed_schemes=frozenset({"https"}))+ except ValueError as exc:+ logger.warning("Blocked Zotero file URL %s: %s", current, exc)+ return None+ with pin_dns(host, ip):+ response = rl_requests.get(current, headers=headers, timeout=120, allow_redirects=False)+ if 300 <= response.status_code < 400:+ location = response.headers.get("Location")+ response.close()+ if not location:+ logger.warning("Zotero file redirect missing Location header")+ return None+ current = urljoin(current, location)+ continue+ if response.status_code >= 400:+ logger.warning("Failed to download Zotero attachment: HTTP %s", response.status_code)+ response.close()+ return None+ blob = _read_response_capped(response, self.size_threshold + 1)+ if len(blob) > self.size_threshold:+ logger.warning("Skipping oversized Zotero attachment download")+ return None+ return blob+ logger.warning("Stopped after too many Zotero file redirects")+ return None++ def _download_via_webdav(self, attachment_key: str, filename: str) -> tuple[bytes, str]:+ self.validate_local_settings()+ zip_url = urljoin(self.webdav_url + "/", f"zotero/{attachment_key}.zip")+ try:+ host, ip = assert_url_is_safe(zip_url, allowed_schemes=frozenset({"https"}))+ except ValueError as exc:+ logger.warning("Blocked Zotero WebDAV URL %s: %s", zip_url, exc)+ return b"", filename+ with pin_dns(host, ip):+ response = rl_requests.get(+ zip_url,+ auth=(self.webdav_username or "", self.webdav_password or ""),+ timeout=120,+ allow_redirects=False,+ stream=True,+ )+ if response.status_code >= 400:+ logger.warning("Failed to download Zotero WebDAV archive %s: HTTP %s", attachment_key, response.status_code)+ response.close()+ return b"", filename+ zip_bytes = _read_response_capped(response, self.size_threshold + 1)+ if len(zip_bytes) > self.size_threshold:+ logger.warning("Skipping oversized Zotero WebDAV archive %s", attachment_key)+ return b"", filename+ return self._extract_pdf_from_zip(zip_bytes, filename)++ def _extract_pdf_from_zip(self, zip_bytes: bytes, fallback_name: str) -> tuple[bytes, str]:+ try:+ archive_ctx = zipfile.ZipFile(io.BytesIO(zip_bytes))+ except zipfile.BadZipFile:+ logger.warning("Malformed Zotero WebDAV archive for %s", fallback_name)+ return b"", fallback_name+ with archive_ctx as archive:+ for name in archive.namelist():+ if name.endswith("/") or not name.lower().endswith(".pdf"):+ continue+ with archive.open(name) as handle:+ blob = handle.read(self.size_threshold + 1)+ if len(blob) > self.size_threshold:+ logger.warning("Skipping oversized PDF in Zotero WebDAV archive %s", name)+ continue+ if blob:+ return blob, name.split("/")[-1]+ logger.warning("No PDF found in Zotero WebDAV archive for %s", fallback_name)+ return b"", fallback_name++ def _headers(self) -> dict[str, str]:+ return {"Zotero-API-Key": self.api_key or "", "Zotero-API-Version": "3"}++ @staticmethod+ def _parse_time(value: str | None) -> datetime:+ if not value:+ return datetime.now(timezone.utc)+ normalized = value.replace("Z", "+00:00")+ try:+ return datetime.fromisoformat(normalized).astimezone(timezone.utc)+ except ValueError:+ return datetime.now(timezone.utc)diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.goindex 0dbeaba817..74952a85a4 100644--- a/internal/syncer/connector/builtin.go+++ b/internal/syncer/connector/builtin.go@@ -67,6 +67,7 @@ func RegisterBuiltIns(registry *Registry) {registerBuiltIn(registry, "webdav", NewWebDAVConnector)registerBuiltIn(registry, "seafile", NewSeaFileConnector)registerBuiltIn(registry, "bigquery", NewBigQueryConnector)+ registerBuiltIn(registry, "zotero", NewZoteroConnector)}func registerBuiltIn[T Connector](registry *Registry, source string, factory func(map[string]any) (T, error)) {diff --git a/internal/syncer/connector/zotero.go b/internal/syncer/connector/zotero.gonew file mode 100644index 0000000000..2332b7a27b--- /dev/null+++ b/internal/syncer/connector/zotero.go@@ -0,0 +1,755 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "archive/zip"+ "bytes"+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "log/slog"+ "net/http"+ "net/url"+ "path/filepath"+ "sort"+ "strconv"+ "strings"+ "time"+)++const (+ zoteroAPIBaseURL = "https://api.zotero.org"+ defaultZoteroBatchSize = 4+ zoteroRequestTimeout = 120 * time.Second+ zoteroDefaultPageSize = 100+ zoteroMaxAttachmentBytes = 100 * 1024 * 1024+ zoteroStorageModeZotero = "zotero_storage"+ zoteroStorageModeWebDAV = "webdav"+ zoteroMaxFileRedirects = 10+ zoteroMaxDownloadAttemptsPerAttachment = 3+)++// ZoteroConnector syncs PDF attachments from a Zotero library.+type ZoteroConnector struct {+ userID string+ apiKey string+ storageMode string+ webdavURL string+ webdavUser string+ webdavPass string+ batchSize int+ httpClient *http.Client+ listItems func(ctx context.Context, start int) ([]zoteroAPIItem, int, error)+ downloadPDF func(ctx context.Context, attachment zoteroAPIItem) ([]byte, string, error)+}++type zoteroAPIItem struct {+ Key string `json:"key"`+ Data zoteroItemData `json:"data"`+}++type zoteroItemData struct {+ ItemType string `json:"itemType"`+ Title string `json:"title"`+ Filename string `json:"filename"`+ ContentType string `json:"contentType"`+ LinkMode string `json:"linkMode"`+ DateModified string `json:"dateModified"`+ ParentItem string `json:"parentItem"`+}++type zoteroPDFRecord struct {+ attachment zoteroAPIItem+ updatedAt time.Time+ title string+}++// NewZoteroConnector creates a Zotero connector from stored connector config.+func NewZoteroConnector(config map[string]any) (*ZoteroConnector, error) {+ credentials := configAnyMap(config["credentials"])+ storageMode := strings.TrimSpace(firstNonEmpty(stringConfig(config["storage_mode"]), zoteroStorageModeZotero))+ if storageMode != zoteroStorageModeZotero && storageMode != zoteroStorageModeWebDAV {+ return nil, &ConnectorValidationError{Message: "storage_mode must be 'zotero_storage' or 'webdav'"}+ }+ webdavURL := strings.TrimRight(strings.TrimSpace(stringConfig(config["webdav_url"])), "/")+ connector := &ZoteroConnector{+ userID: strings.TrimSpace(firstNonEmpty(stringConfig(config["zotero_user_id"]), stringConfig(credentials["zotero_user_id"]))),+ apiKey: strings.TrimSpace(stringConfig(credentials["zotero_api_key"])),+ storageMode: storageMode,+ webdavURL: webdavURL,+ webdavUser: strings.TrimSpace(stringConfig(credentials["webdav_username"])),+ webdavPass: stringConfig(credentials["webdav_password"]),+ batchSize: configInt(config["batch_size"], defaultZoteroBatchSize),+ httpClient: &http.Client{+ Timeout: zoteroRequestTimeout,+ CheckRedirect: func(req *http.Request, via []*http.Request) error {+ return http.ErrUseLastResponse+ },+ },+ }+ connector.listItems = connector.defaultListAttachmentItems+ connector.downloadPDF = connector.defaultDownloadPDF+ return connector, nil+}++// Validate validates Zotero settings and credentials.+func (c *ZoteroConnector) Validate(ctx context.Context) error {+ if err := c.validateStatic(); err != nil {+ return err+ }+ if err := c.validateWebDAVAccess(ctx); err != nil {+ return err+ }+ if _, _, err := c.listItems(ctx, 0); err != nil {+ return classifyZoteroError(err)+ }+ return nil+}++// ValidateConnectorSetting validates an unsaved Zotero config.+func (c *ZoteroConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)+ defer cancel()+ connector, err := NewZoteroConnector(request)+ if err != nil {+ return err+ }+ connector.httpClient = c.httpClient+ return connector.Validate(ctx)+}++// OpenSync opens one Zotero sync session.+func (c *ZoteroConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {+ if err := c.validateStatic(); err != nil {+ return nil, err+ }+ if err := c.validateWebDAVAccess(ctx); err != nil {+ return nil, err+ }+ records, err := c.collectPDFRecords(ctx, request)+ if err != nil {+ return nil, classifyZoteroError(err)+ }+ sort.Slice(records, func(i, j int) bool {+ return c.sourceID(records[i].attachment.Key) < c.sourceID(records[j].attachment.Key)+ })+ session := &zoteroSyncSession{+ connector: c,+ records: records,+ batchSize: c.batchSize,+ downloadAttempts: make(map[string]int),+ }+ if err := session.applyResume(request.Resume); err != nil {+ return nil, err+ }+ return session, nil+}++// OpenPrune opens one complete Zotero prune snapshot session.+func (c *ZoteroConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {+ if err := c.validateStatic(); err != nil {+ return nil, err+ }+ if err := c.validateWebDAVAccess(ctx); err != nil {+ return nil, err+ }+ records, err := c.collectPDFRecords(ctx, SyncRequest{FromBeginning: true})+ if err != nil {+ return nil, classifyZoteroError(err)+ }+ documents := make([]SlimDocument, 0, len(records))+ for _, record := range records {+ documents = append(documents, SlimDocument{SourceID: c.sourceID(record.attachment.Key)})+ }+ sort.Slice(documents, func(i, j int) bool { return documents[i].SourceID < documents[j].SourceID })+ return &zoteroPruneSession{documents: documents, batchSize: c.batchSize}, nil+}++func (c *ZoteroConnector) validateStatic() error {+ if c == nil {+ return &ConnectorValidationError{Message: "Zotero connector is nil"}+ }+ if c.userID == "" {+ return &ConnectorMissingCredentialError{Message: "Zotero user ID is required"}+ }+ if c.apiKey == "" {+ return &ConnectorMissingCredentialError{Message: "Zotero API key is required"}+ }+ if c.storageMode == zoteroStorageModeWebDAV {+ if c.webdavURL == "" {+ return &ConnectorValidationError{Message: "webdav_url is required when storage_mode is webdav"}+ }+ parsed, err := url.Parse(c.webdavURL)+ if err != nil || parsed.Scheme != "https" {+ return &ConnectorValidationError{Message: "WebDAV URL must use HTTPS"}+ }+ if c.webdavUser == "" {+ return &ConnectorMissingCredentialError{Message: "WebDAV username is required when storage_mode is webdav"}+ }+ if c.webdavPass == "" {+ return &ConnectorMissingCredentialError{Message: "WebDAV password is required when storage_mode is webdav"}+ }+ }+ if c.batchSize <= 0 {+ return &ConnectorValidationError{Message: "batch_size must be a positive integer"}+ }+ return nil+}++func (c *ZoteroConnector) validateWebDAVAccess(ctx context.Context) error {+ if c.storageMode != zoteroStorageModeWebDAV {+ return nil+ }+ probeURL := strings.TrimRight(c.webdavURL, "/") + "/zotero/"+ resp, err := connectorRequest(ctx, connectorRequestOptions{+ Method: "PROPFIND",+ RawURL: probeURL,+ Body: []byte(webdavPropfindBody),+ Headers: map[string]string{+ "Depth": "0",+ "Content-Type": "application/xml",+ "Authorization": "Basic " + basicAuthHeader(c.webdavUser, c.webdavPass),+ },+ Timeout: zoteroRequestTimeout,+ Base: c.httpClient,+ })+ if err != nil {+ return &ConnectorValidationError{Message: fmt.Sprintf("WebDAV probe failed: %v", connectorUnsafeErr(err))}+ }+ defer resp.Body.Close()+ _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))+ switch resp.StatusCode {+ case http.StatusMultiStatus, http.StatusOK:+ return nil+ case http.StatusUnauthorized, http.StatusForbidden:+ return &ConnectorMissingCredentialError{Message: "WebDAV credentials appear invalid or expired"}+ case http.StatusNotFound:+ return &ConnectorValidationError{Message: "WebDAV zotero collection was not found"}+ default:+ return &ConnectorValidationError{Message: fmt.Sprintf("WebDAV probe returned HTTP %d", resp.StatusCode)}+ }+}++func (c *ZoteroConnector) collectPDFRecords(ctx context.Context, request SyncRequest) ([]zoteroPDFRecord, error) {+ records := make([]zoteroPDFRecord, 0)+ start := 0+ for {+ items, total, err := c.listItems(ctx, start)+ if err != nil {+ return nil, err+ }+ for _, item := range items {+ if !zoteroIsPDFAttachment(item) {+ continue+ }+ updatedAt, err := zoteroParseTime(item.Data.DateModified)+ if err != nil {+ slog.Warn(+ "zotero: using fallback timestamp for attachment with invalid dateModified",+ "item_key", item.Key,+ "error", err,+ )+ updatedAt = time.Unix(0, 0).UTC()+ }+ if !request.FromBeginning && request.WindowStart != nil {+ if !updatedAt.After(*request.WindowStart) {+ continue+ }+ }+ if !request.WindowEnd.IsZero() && updatedAt.After(request.WindowEnd) {+ continue+ }+ records = append(records, zoteroPDFRecord{+ attachment: item,+ updatedAt: updatedAt,+ title: zoteroAttachmentTitle(item),+ })+ }+ start += len(items)+ if len(items) == 0 || start >= total {+ break+ }+ }+ return records, nil+}++func (c *ZoteroConnector) buildDocument(record zoteroPDFRecord, blob []byte, filename string) SourceDocument {+ ext := strings.ToLower(filepath.Ext(filename))+ if ext == "" {+ ext = ".pdf"+ }+ return SourceDocument{+ SourceID: c.sourceID(record.attachment.Key),+ SemanticIdentifier: record.title,+ Extension: strings.TrimPrefix(ext, "."),+ Blob: blob,+ UpdatedAt: record.updatedAt,+ SizeBytes: int64(len(blob)),+ Fingerprint: contentFingerprint(blob),+ Metadata: map[string]any{+ "source": "zotero",+ "zotero_item_key": record.attachment.Key,+ "zotero_parent": record.attachment.Data.ParentItem,+ "filename": filename,+ },+ }+}++func (c *ZoteroConnector) sourceID(attachmentKey string) string {+ return "zotero:" + c.userID + ":" + attachmentKey+}++func (c *ZoteroConnector) defaultListAttachmentItems(ctx context.Context, start int) ([]zoteroAPIItem, int, error) {+ endpoint := fmt.Sprintf("%s/users/%s/items", zoteroAPIBaseURL, c.userID)+ query := fmt.Sprintf("?itemType=attachment&format=json&limit=%d&start=%d", zoteroDefaultPageSize, start)+ body, headers, err := c.doZoteroRequest(ctx, endpoint+query, http.MethodGet, nil)+ if err != nil {+ return nil, 0, err+ }+ var items []zoteroAPIItem+ if err := json.Unmarshal(body, &items); err != nil {+ return nil, 0, err+ }+ total := start + len(items)+ if headers != nil {+ if raw := strings.TrimSpace(headers.Get("Total-Results")); raw != "" {+ if parsed, err := strconv.Atoi(raw); err == nil && parsed > total {+ total = parsed+ }+ }+ }+ return items, total, nil+}++func (c *ZoteroConnector) doZoteroRequest(ctx context.Context, url string, method string, payload []byte) ([]byte, http.Header, error) {+ requestCtx, cancel := context.WithTimeout(ctx, zoteroRequestTimeout)+ defer cancel()+ var body io.Reader+ if len(payload) > 0 {+ body = bytes.NewReader(payload)+ }+ req, err := http.NewRequestWithContext(requestCtx, method, url, body)+ if err != nil {+ return nil, nil, err+ }+ if hostAllowsZoteroAPIKey(req.URL.Hostname()) {+ req.Header.Set("Zotero-API-Key", c.apiKey)+ }+ req.Header.Set("Zotero-API-Version", "3")+ if len(payload) > 0 {+ req.Header.Set("Content-Type", "application/json")+ }+ resp, err := c.httpClient.Do(req)+ if err != nil {+ return nil, nil, err+ }+ defer resp.Body.Close()+ if resp.StatusCode >= 300 && resp.StatusCode < 400 {+ io.Copy(io.Discard, resp.Body)+ return nil, resp.Header.Clone(), &ConnectorValidationError{Message: "Unexpected redirect from Zotero API"}+ }+ data, err := io.ReadAll(io.LimitReader(resp.Body, zoteroMaxAttachmentBytes+1024))+ if err != nil {+ return nil, nil, err+ }+ if resp.StatusCode >= 400 {+ return nil, resp.Header.Clone(), &zoteroHTTPError{Status: resp.StatusCode, Body: string(data), URL: url}+ }+ return data, resp.Header.Clone(), nil+}++func (c *ZoteroConnector) defaultDownloadPDF(ctx context.Context, attachment zoteroAPIItem) ([]byte, string, error) {+ filename := strings.TrimSpace(attachment.Data.Filename)+ if filename == "" {+ filename = attachment.Data.Title + ".pdf"+ }+ switch c.storageMode {+ case zoteroStorageModeWebDAV:+ return c.downloadPDFViaWebDAV(ctx, attachment.Key, filename)+ default:+ return c.downloadPDFViaZoteroAPI(ctx, attachment.Key, filename)+ }+}++func (c *ZoteroConnector) downloadPDFViaZoteroAPI(ctx context.Context, attachmentKey, filename string) ([]byte, string, error) {+ fileURL := fmt.Sprintf("%s/users/%s/items/%s/file", zoteroAPIBaseURL, c.userID, attachmentKey)+ data, err := c.downloadAuthedURL(ctx, fileURL)+ if err != nil {+ return nil, "", err+ }+ return data, filename, nil+}++func (c *ZoteroConnector) downloadAuthedURL(ctx context.Context, rawURL string) ([]byte, error) {+ currentURL := rawURL+ for redirects := 0; redirects < zoteroMaxFileRedirects; redirects++ {+ data, err := c.getPinnedURL(ctx, currentURL, func(req *http.Request) {+ req.Header.Set("Zotero-API-Version", "3")+ if hostAllowsZoteroAPIKey(req.URL.Hostname()) {+ req.Header.Set("Zotero-API-Key", c.apiKey)+ }+ })+ if err != nil {+ var redirect redirectError+ if errors.As(err, &redirect) {+ currentURL = redirect.Location+ continue+ }+ return nil, err+ }+ return data, nil+ }+ return nil, fmt.Errorf("stopped after too many Zotero file redirects")+}++type redirectError struct {+ Location string+}++func (e redirectError) Error() string {+ return "HTTP redirect to " + e.Location+}++func hostAllowsZoteroAPIKey(hostname string) bool {+ return strings.EqualFold(strings.TrimSuffix(hostname, "."), "api.zotero.org")+}++func (c *ZoteroConnector) getPinnedURL(ctx context.Context, rawURL string, authorize func(*http.Request)) ([]byte, error) {+ parsed, err := url.Parse(rawURL)+ if err != nil {+ return nil, err+ }+ if !strings.EqualFold(parsed.Scheme, "https") {+ return nil, fmt.Errorf("Zotero download URL must use HTTPS")+ }+ hostname, pinIP, err := assertRestAPIURLSafe(ctx, rawURL)+ if err != nil {+ return nil, err+ }+ requestCtx, cancel := context.WithTimeout(ctx, zoteroRequestTimeout)+ defer cancel()+ req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, rawURL, nil)+ if err != nil {+ return nil, err+ }+ if authorize != nil {+ authorize(req)+ }+ client := c.httpClient+ if client == nil {+ client = http.DefaultClient+ }+ httpClient := *client+ httpClient.Transport = newConnectorPinnedTransport(client, hostname, pinIP, zoteroRequestTimeout)+ httpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {+ return http.ErrUseLastResponse+ }+ resp, err := httpClient.Do(req)+ if err != nil {+ return nil, err+ }+ defer resp.Body.Close()+ if resp.StatusCode >= 300 && resp.StatusCode < 400 {+ location := resp.Header.Get("Location")+ if location == "" {+ return nil, fmt.Errorf("Zotero file redirect missing Location header")+ }+ next, err := resp.Request.URL.Parse(location)+ if err != nil {+ return nil, err+ }+ return nil, redirectError{Location: next.String()}+ }+ if resp.StatusCode >= 400 {+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))+ return nil, &zoteroHTTPError{Status: resp.StatusCode, Body: string(body), URL: rawURL}+ }+ data, err := io.ReadAll(io.LimitReader(resp.Body, zoteroMaxAttachmentBytes+1))+ if err != nil {+ return nil, err+ }+ if int64(len(data)) > zoteroMaxAttachmentBytes {+ return nil, fmt.Errorf("Zotero attachment exceeds maximum size")+ }+ return data, nil+}++func (c *ZoteroConnector) downloadPDFViaWebDAV(ctx context.Context, attachmentKey, filename string) ([]byte, string, error) {+ zipURL := strings.TrimRight(c.webdavURL, "/") + "/zotero/" + attachmentKey + ".zip"+ data, err := c.getPinnedURL(ctx, zipURL, func(req *http.Request) {+ req.SetBasicAuth(c.webdavUser, c.webdavPass)+ })+ if err != nil {+ return nil, "", err+ }+ return extractPDFFromZip(data, filename)+}++func extractPDFFromZip(zipBytes []byte, fallbackName string) ([]byte, string, error) {+ reader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))+ if err != nil {+ return nil, "", nil+ }+ var pdfName string+ var pdfData []byte+ for _, file := range reader.File {+ if file.FileInfo().IsDir() {+ continue+ }+ name := filepath.Base(file.Name)+ if !strings.HasSuffix(strings.ToLower(name), ".pdf") {+ continue+ }+ rc, err := file.Open()+ if err != nil {+ return nil, "", err+ }+ data, err := io.ReadAll(io.LimitReader(rc, zoteroMaxAttachmentBytes+1))+ rc.Close()+ if err != nil {+ return nil, "", err+ }+ if int64(len(data)) > zoteroMaxAttachmentBytes {+ continue+ }+ if len(data) == 0 {+ continue+ }+ pdfName = name+ pdfData = data+ break+ }+ if len(pdfData) == 0 {+ return nil, "", fmt.Errorf("no PDF found in Zotero WebDAV archive for %s", fallbackName)+ }+ return pdfData, pdfName, nil+}++func zoteroIsPDFAttachment(item zoteroAPIItem) bool {+ if item.Data.ItemType != "attachment" {+ return false+ }+ contentType := strings.ToLower(strings.TrimSpace(item.Data.ContentType))+ if strings.Contains(contentType, "pdf") {+ return true+ }+ filename := strings.ToLower(item.Data.Filename)+ return strings.HasSuffix(filename, ".pdf")+}++func zoteroAttachmentTitle(item zoteroAPIItem) string {+ title := strings.TrimSpace(item.Data.Title)+ if title != "" {+ return title+ }+ if name := strings.TrimSpace(item.Data.Filename); name != "" {+ return strings.TrimSuffix(name, filepath.Ext(name))+ }+ return "Zotero attachment " + item.Key+}++func zoteroParseTime(value string) (time.Time, error) {+ value = strings.TrimSpace(value)+ if value == "" {+ return time.Time{}, errors.New("missing timestamp")+ }+ layouts := []string{time.RFC3339Nano, time.RFC3339, "2006-01-02T15:04:05Z"}+ for _, layout := range layouts {+ if parsed, err := time.Parse(layout, value); err == nil {+ return parsed.UTC(), nil+ }+ }+ return time.Time{}, fmt.Errorf("unsupported Zotero timestamp %q", value)+}++type zoteroHTTPError struct {+ Status int+ Body string+ URL string+}++func (e *zoteroHTTPError) Error() string {+ return fmt.Sprintf("Zotero API returned HTTP %d for %s: %s", e.Status, e.URL, strings.TrimSpace(e.Body))+}++func classifyZoteroError(err error) error {+ if err == nil {+ return nil+ }+ var httpErr *zoteroHTTPError+ if errors.As(err, &httpErr) {+ switch httpErr.Status {+ case http.StatusUnauthorized, http.StatusForbidden:+ return &ConnectorMissingCredentialError{Message: "Zotero API key is invalid or expired"}+ case http.StatusNotFound:+ return &ConnectorValidationError{Message: "Zotero library or attachment was not found"}+ default:+ return &ConnectorValidationError{Message: httpErr.Error()}+ }+ }+ return err+}++type zoteroSyncSession struct {+ connector *ZoteroConnector+ records []zoteroPDFRecord+ batchSize int+ index int+ downloadFailures int+ downloadAttempts map[string]int+}++func (s *zoteroSyncSession) handleDownloadFailure(+ record zoteroPDFRecord,+ reason string,+ documentsLen int,+ attrs ...any,+) (breakBatch bool, advanceIndex bool) {+ s.downloadFailures+++ key := record.attachment.Key+ s.downloadAttempts[key]+++ args := []any{+ "item_key", key,+ "source_id", s.connector.sourceID(key),+ "attempt", s.downloadAttempts[key],+ "max_attempts", zoteroMaxDownloadAttemptsPerAttachment,+ }+ args = append(args, attrs...)+ slog.Warn("zotero: "+reason, args...)++ if s.downloadAttempts[key] >= zoteroMaxDownloadAttemptsPerAttachment {+ slog.Warn(+ "zotero: permanently skipping attachment after repeated download failures",+ "item_key", key,+ "source_id", s.connector.sourceID(key),+ "attempts", s.downloadAttempts[key],+ )+ return false, true+ }+ if documentsLen > 0 {+ return true, false+ }+ return false, false+}++func (s *zoteroSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {+ documents := make([]SourceDocument, 0, s.batchSize)+ for len(documents) < s.batchSize && s.index < len(s.records) {+ record := s.records[s.index]+ blob, filename, err := s.connector.downloadPDF(ctx, record.attachment)+ if err != nil {+ if ctx.Err() != nil {+ return SyncBatch{}, ctx.Err()+ }+ }+ if len(blob) == 0 && ctx.Err() != nil {+ return SyncBatch{}, ctx.Err()+ }+ if err != nil || len(blob) == 0 {+ breakBatch, advanceIndex := s.handleDownloadFailure(+ record,+ "attachment download failed",+ len(documents),+ "error", err,+ )+ if advanceIndex {+ s.index+++ continue+ }+ if breakBatch {+ break+ }+ continue+ }+ if int64(len(blob)) > zoteroMaxAttachmentBytes {+ breakBatch, advanceIndex := s.handleDownloadFailure(+ record,+ "skipping oversized attachment",+ len(documents),+ "bytes", len(blob),+ )+ if advanceIndex {+ s.index+++ continue+ }+ if breakBatch {+ break+ }+ continue+ }+ documents = append(documents, s.connector.buildDocument(record, blob, filename))+ s.index+++ }+ if len(documents) == 0 {+ if s.index >= len(s.records) {+ if s.downloadFailures > 0 {+ slog.Warn("zotero: sync completed with attachment download failures", "count", s.downloadFailures)+ }+ return SyncBatch{}, io.EOF+ }+ return SyncBatch{}, fmt.Errorf("zotero: stalled downloading attachment %q", s.records[s.index].attachment.Key)+ }+ return SyncBatch{Documents: documents, Checkpoint: zoteroSyncCheckpoint(documents[len(documents)-1])}, nil+}++func (s *zoteroSyncSession) Close() error { return nil }++func (s *zoteroSyncSession) applyResume(checkpoint *SyncCheckpoint) error {+ if checkpoint == nil {+ return nil+ }+ sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)+ if sourceID == "" {+ return fmt.Errorf("zotero sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)+ }+ for index, record := range s.records {+ if s.connector.sourceID(record.attachment.Key) == sourceID {+ s.index = index + 1+ return nil+ }+ }+ return fmt.Errorf("zotero resume anchor %q was not found in the current listing: %w", sourceID, ErrSyncResumeInvalid)+}++func zoteroSyncCheckpoint(doc SourceDocument) *SyncCheckpoint {+ updatedAt := doc.UpdatedAt+ return &SyncCheckpoint{Cursor: doc.SourceID, SourceID: doc.SourceID, UpdatedAt: &updatedAt}+}++type zoteroPruneSession struct {+ documents []SlimDocument+ batchSize int+ index int+}++func (s *zoteroPruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {+ if s.index >= len(s.documents) {+ return PruneBatch{}, io.EOF+ }+ end := s.index + s.batchSize+ if end > len(s.documents) {+ end = len(s.documents)+ }+ batch := PruneBatch{Documents: s.documents[s.index:end]}+ s.index = end+ return batch, nil+}++func (s *zoteroPruneSession) Close() error { return nil }diff --git a/internal/syncer/connector/zotero_test.go b/internal/syncer/connector/zotero_test.gonew file mode 100644index 0000000000..0174cf520e--- /dev/null+++ b/internal/syncer/connector/zotero_test.go@@ -0,0 +1,264 @@+package connector++import (+ "context"+ "encoding/json"+ "errors"+ "io"+ "net/http"+ "net/http/httptest"+ "testing"+ "time"+)++func TestZoteroConnectorOpenSyncDownloadsPDF(t *testing.T) {+ server := newZoteroTestServer(t)+ connector, err := NewZoteroConnector(map[string]any{+ "zotero_user_id": "12345678",+ "storage_mode": zoteroStorageModeZotero,+ "batch_size": 1,+ "credentials": map[string]any{+ "zotero_api_key": "test-key",+ },+ })+ if err != nil {+ t.Fatalf("NewZoteroConnector: %v", err)+ }+ connector.httpClient = server.Client()+ downloads := 0+ connector.listItems = func(ctx context.Context, start int) ([]zoteroAPIItem, int, error) {+ return server.listItems(start)+ }+ connector.downloadPDF = func(ctx context.Context, attachment zoteroAPIItem) ([]byte, string, error) {+ downloads+++ return server.downloadPDF(attachment.Key)+ }++ session, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true, WindowEnd: mustTime(t, "2026-02-01T00:00:00Z")})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ if downloads != 0 {+ t.Fatalf("OpenSync downloaded %d attachments before NextBatch", downloads)+ }+ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(first.Documents) != 1 {+ t.Fatalf("batch len = %d", len(first.Documents))+ }+ doc := first.Documents[0]+ if doc.SourceID != "zotero:12345678:ATTACH1" {+ t.Fatalf("source id = %s", doc.SourceID)+ }+ if string(doc.Blob) != "%PDF-1.4 test" {+ t.Fatalf("blob = %q", string(doc.Blob))+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("expected EOF, got %v", err)+ }+}++func TestZoteroParseTime(t *testing.T) {+ parsed, err := zoteroParseTime("2026-01-02T12:34:56Z")+ if err != nil {+ t.Fatalf("zoteroParseTime: %v", err)+ }+ if parsed.UTC().Format(time.RFC3339) != "2026-01-02T12:34:56Z" {+ t.Fatalf("parsed = %s", parsed.UTC().Format(time.RFC3339))+ }+}++func TestZoteroNextBatchRetriesFailedDownloadAfterPartialBatch(t *testing.T) {+ connector, err := NewZoteroConnector(map[string]any{+ "zotero_user_id": "12345678",+ "storage_mode": zoteroStorageModeZotero,+ "batch_size": 10,+ "credentials": map[string]any{+ "zotero_api_key": "test-key",+ },+ })+ if err != nil {+ t.Fatalf("NewZoteroConnector: %v", err)+ }+ attempts := map[string]int{}+ connector.downloadPDF = func(ctx context.Context, attachment zoteroAPIItem) ([]byte, string, error) {+ attempts[attachment.Key]+++ switch attachment.Key {+ case "ATTACH1":+ return []byte("%PDF"), "one.pdf", nil+ case "ATTACH2":+ return nil, "", errors.New("transient failure")+ default:+ return nil, "", errors.New("unexpected attachment")+ }+ }+ session := &zoteroSyncSession{+ connector: connector,+ records: []zoteroPDFRecord{+ {attachment: zoteroAPIItem{Key: "ATTACH1"}, title: "One"},+ {attachment: zoteroAPIItem{Key: "ATTACH2"}, title: "Two"},+ },+ batchSize: 10,+ downloadAttempts: make(map[string]int),+ }++ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("first NextBatch: %v", err)+ }+ if len(first.Documents) != 1 || first.Documents[0].SourceID != "zotero:12345678:ATTACH1" {+ t.Fatalf("unexpected first batch: %+v", first.Documents)+ }+ if session.index != 1 {+ t.Fatalf("index = %d, want 1 to retry failed attachment", session.index)+ }++ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("second NextBatch: %v", err)+ }+ if attempts["ATTACH2"] != 3 {+ t.Fatalf("ATTACH2 attempts = %d, want 3 (1 in partial batch + 2 before permanent skip)", attempts["ATTACH2"])+ }+}++func TestZoteroNextBatchRetriesImmediatelyBeforeSkip(t *testing.T) {+ connector, err := NewZoteroConnector(map[string]any{+ "zotero_user_id": "12345678",+ "storage_mode": zoteroStorageModeZotero,+ "batch_size": 1,+ "credentials": map[string]any{+ "zotero_api_key": "test-key",+ },+ })+ if err != nil {+ t.Fatalf("NewZoteroConnector: %v", err)+ }+ attempts := 0+ connector.downloadPDF = func(ctx context.Context, attachment zoteroAPIItem) ([]byte, string, error) {+ attempts+++ if attempts < 3 {+ return nil, "", errors.New("transient failure")+ }+ return []byte("%PDF"), "paper.pdf", nil+ }+ session := &zoteroSyncSession{+ connector: connector,+ records: []zoteroPDFRecord{+ {attachment: zoteroAPIItem{Key: "ATTACH1"}, title: "One"},+ },+ batchSize: 1,+ downloadAttempts: make(map[string]int),+ }++ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("batch len = %d, want 1 after in-batch retries", len(batch.Documents))+ }+ if attempts != 3 {+ t.Fatalf("attempts = %d, want 3 immediate retries before success", attempts)+ }+}++func TestZoteroConnectorOpenSyncIncrementalWindow(t *testing.T) {+ server := newZoteroTestServer(t)+ connector, err := NewZoteroConnector(map[string]any{+ "zotero_user_id": "12345678",+ "batch_size": 10,+ "credentials": map[string]any{+ "zotero_api_key": "test-key",+ },+ })+ if err != nil {+ t.Fatalf("NewZoteroConnector: %v", err)+ }+ connector.httpClient = server.Client()+ connector.listItems = func(ctx context.Context, start int) ([]zoteroAPIItem, int, error) {+ return server.listItems(start)+ }+ connector.downloadPDF = func(ctx context.Context, attachment zoteroAPIItem) ([]byte, string, error) {+ return server.downloadPDF(attachment.Key)+ }++ start := mustTime(t, "2025-12-01T00:00:00Z")+ session, err := connector.OpenSync(t.Context(), SyncRequest{+ WindowStart: &start,+ WindowEnd: mustTime(t, "2026-01-15T00:00:00Z"),+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "zotero:12345678:ATTACH1" {+ t.Fatalf("unexpected batch: %+v", batch.Documents)+ }+}++type zoteroTestServer struct {+ *httptest.Server+ items []zoteroAPIItem+}++func newZoteroTestServer(t *testing.T) *zoteroTestServer {+ state := &zoteroTestServer{+ items: []zoteroAPIItem{+ {+ Key: "ATTACH1",+ Data: zoteroItemData{+ ItemType: "attachment",+ Title: "Paper One",+ Filename: "paper-one.pdf",+ ContentType: "application/pdf",+ DateModified: "2026-01-02T00:00:00Z",+ ParentItem: "ITEM1",+ },+ },+ {+ Key: "ATTACH2",+ Data: zoteroItemData{+ ItemType: "attachment",+ Title: "Old Paper",+ Filename: "old.pdf",+ ContentType: "application/pdf",+ DateModified: "2024-01-02T00:00:00Z",+ ParentItem: "ITEM2",+ },+ },+ },+ }+ state.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case r.URL.Path == "/users/12345678/items" && r.Method == http.MethodGet:+ w.Header().Set("Total-Results", "2")+ _ = json.NewEncoder(w).Encode(state.items)+ case r.URL.Path == "/users/12345678/items/ATTACH1/file":+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte("%PDF-1.4 test"))+ default:+ http.NotFound(w, r)+ }+ }))+ t.Cleanup(state.Close)+ return state+}++func (s *zoteroTestServer) listItems(start int) ([]zoteroAPIItem, int, error) {+ if start >= len(s.items) {+ return nil, len(s.items), nil+ }+ return s.items[start:], len(s.items), nil+}++func (s *zoteroTestServer) downloadPDF(key string) ([]byte, string, error) {+ if key == "ATTACH1" {+ return []byte("%PDF-1.4 test"), "paper-one.pdf", nil+ }+ return nil, "", errors.New("missing attachment")+}diff --git a/rag/svr/sync_data_source.py b/rag/svr/sync_data_source.pyindex f8b0f9d80c..387d382b5d 100644--- a/rag/svr/sync_data_source.py+++ b/rag/svr/sync_data_source.py@@ -59,6 +59,7 @@ from common.data_source import (AsanaConnector,ImapConnector,ZendeskConnector,+ ZoteroConnector,SeaFileConnector,RDBMSConnector,BigQueryConnector,@@ -2125,6 +2126,55 @@ class AzureDevOps(SyncBase):return wrapper()+class Zotero(SyncBase):+ SOURCE_NAME: str = FileSource.ZOTERO++ def _zotero_batch_size(self) -> int:+ raw_batch_size = self.conf.get("batch_size", INDEX_BATCH_SIZE)+ try:+ batch_size = int(raw_batch_size)+ except (TypeError, ValueError):+ batch_size = INDEX_BATCH_SIZE+ if batch_size <= 0:+ batch_size = INDEX_BATCH_SIZE+ return batch_size++ async def _prepare_connector(self, task: dict):+ conf = self.conf+ user_id = (conf.get("zotero_user_id") or conf["credentials"].get("zotero_user_id") or "").strip()+ self.connector = ZoteroConnector(+ zotero_user_id=user_id,+ storage_mode=conf.get("storage_mode", "zotero_storage"),+ webdav_url=conf.get("webdav_url"),+ batch_size=self._zotero_batch_size(),+ )+ self.connector.load_credentials(conf["credentials"])+ self.connector.validate_local_settings()+ self.log_connection(+ "Zotero",+ f"user_id={user_id} storage={conf.get('storage_mode', 'zotero_storage')}",+ task,+ )++ async def _initialize_for_prune(self, task: dict):+ await self._prepare_connector(task)++ async def _generate(self, task: dict):+ await self._prepare_connector(task)++ poll_start = task.get("poll_range_start")+ if task["reindex"] == "1" or poll_start is None:+ document_generator = self.connector.load_from_state()+ else:+ end_ts = datetime.now(timezone.utc).timestamp()+ document_generator = self.connector.poll_source(+ poll_start.timestamp(),+ end_ts,+ )++ return iter_in_worker_thread(document_generator)++class SeaFile(SyncBase):SOURCE_NAME: str = FileSource.SEAFILE@@ -2437,6 +2487,7 @@ func_factory = {FileSource.ASANA: Asana,FileSource.IMAP: IMAP,FileSource.ZENDESK: Zendesk,+ FileSource.ZOTERO: Zotero,FileSource.GITHUB: Github,FileSource.GITLAB: Gitlab,FileSource.BITBUCKET: Bitbucket,diff --git a/test/unit_test/data_source/test_zotero_connector_unit.py b/test/unit_test/data_source/test_zotero_connector_unit.pynew file mode 100644index 0000000000..be3c9b391e--- /dev/null+++ b/test/unit_test/data_source/test_zotero_connector_unit.py@@ -0,0 +1,202 @@+from contextlib import nullcontext as _nullcontext+from unittest.mock import MagicMock, patch++import pytest++from common.data_source import zotero_connector as zotero_mod+from common.data_source.exceptions import ConnectorMissingCredentialError+from common.data_source.zotero_connector import ZoteroConnector+++def _file_response(content: bytes) -> MagicMock:+ response = MagicMock()+ response.status_code = 200+ response.iter_content = lambda chunk_size=None: iter([content])+ response.close = MagicMock()+ return response+++def _attachment_item(key: str, modified: str) -> dict:+ return {+ "key": key,+ "data": {+ "itemType": "attachment",+ "title": "Sample Paper",+ "filename": "sample.pdf",+ "contentType": "application/pdf",+ "dateModified": modified,+ "parentItem": "PARENT1",+ },+ }+++def test_zotero_connector_builds_pdf_document(monkeypatch):+ monkeypatch.setattr(+ zotero_mod,+ "assert_url_is_safe",+ lambda url, allowed_schemes=None: ("api.zotero.org", "1.2.3.4"),+ )+ monkeypatch.setattr(+ zotero_mod,+ "pin_dns",+ lambda host, ip: _nullcontext(),+ )+ connector = ZoteroConnector(zotero_user_id="12345678", batch_size=2)+ connector.load_credentials({"zotero_api_key": "secret"})++ items = [_attachment_item("ATTACH1", "2026-01-02T00:00:00Z")]++ with patch("common.data_source.zotero_connector.rl_requests.get") as mock_get:+ list_response = MagicMock()+ list_response.status_code = 200+ list_response.json.return_value = items+ mock_get.side_effect = [list_response, _file_response(b"%PDF-1.4 test")]++ batches = list(connector.load_from_state())++ assert len(batches) == 1+ assert len(batches[0]) == 1+ document = batches[0][0]+ assert document.id == "zotero:12345678:ATTACH1"+ assert document.extension == "pdf"+ assert document.blob == b"%PDF-1.4 test"+++def test_zotero_connector_poll_filters_by_modified_time(monkeypatch):+ monkeypatch.setattr(+ zotero_mod,+ "assert_url_is_safe",+ lambda url, allowed_schemes=None: ("api.zotero.org", "1.2.3.4"),+ )+ monkeypatch.setattr(+ zotero_mod,+ "pin_dns",+ lambda host, ip: _nullcontext(),+ )+ connector = ZoteroConnector(zotero_user_id="12345678", batch_size=10)+ connector.load_credentials({"zotero_api_key": "secret"})++ items = [+ _attachment_item("OLD", "2024-01-01T00:00:00Z"),+ _attachment_item("NEW", "2026-01-02T00:00:00Z"),+ ]++ with patch("common.data_source.zotero_connector.rl_requests.get") as mock_get:+ list_response = MagicMock()+ list_response.status_code = 200+ list_response.json.return_value = items+ mock_get.side_effect = [list_response, _file_response(b"%PDF")]++ batches = list(+ connector.poll_source(+ start=1735689600, # 2025-01-01+ end=1893456000, # 2030-01-01+ )+ )++ assert len(batches) == 1+ assert len(batches[0]) == 1+ assert batches[0][0].id.endswith(":NEW")+++def test_zotero_connector_requires_api_key():+ connector = ZoteroConnector(zotero_user_id="12345678")+ with pytest.raises(ConnectorMissingCredentialError):+ connector.load_credentials({})+++def test_zotero_connector_requires_user_id():+ connector = ZoteroConnector(zotero_user_id=None)+ connector.load_credentials({"zotero_api_key": "secret"})+ with pytest.raises(ConnectorMissingCredentialError, match="user ID"):+ connector.validate_local_settings()+++def test_zotero_rejects_blocked_webdav_url(monkeypatch):+ def _blocked(url, allowed_schemes=None):+ if url.endswith("/zotero/"):+ raise ValueError("blocked host")+ return ("example.com", "1.2.3.4")++ monkeypatch.setattr(zotero_mod, "assert_url_is_safe", _blocked)+ connector = ZoteroConnector(+ zotero_user_id="12345678",+ storage_mode="webdav",+ webdav_url="https://example.com/dav",+ )+ connector.load_credentials({"zotero_api_key": "secret", "webdav_username": "dav", "webdav_password": "pw"})+ with pytest.raises(Exception, match="not allowed"):+ connector.validate_local_settings()+++def test_zotero_rejects_http_webdav_url():+ connector = ZoteroConnector(+ zotero_user_id="12345678",+ storage_mode="webdav",+ webdav_url="http://example.com",+ )+ connector.load_credentials({"zotero_api_key": "secret", "webdav_username": "dav", "webdav_password": "pw"})+ with pytest.raises(Exception, match="HTTPS"):+ connector.validate_local_settings()+++def test_zotero_download_retries_before_skip(monkeypatch):+ monkeypatch.setattr(+ zotero_mod,+ "assert_url_is_safe",+ lambda url, allowed_schemes=None: ("api.zotero.org", "1.2.3.4"),+ )+ monkeypatch.setattr(+ zotero_mod,+ "pin_dns",+ lambda host, ip: _nullcontext(),+ )+ monkeypatch.setattr(zotero_mod, "DOWNLOAD_RETRY_BACKOFF_SEC", 0)+ connector = ZoteroConnector(zotero_user_id="12345678", batch_size=10)+ connector.load_credentials({"zotero_api_key": "secret"})+ attempts = {"count": 0}++ def fake_download(_item):+ attempts["count"] += 1+ if attempts["count"] < 3:+ return b"", "paper.pdf"+ return b"%PDF", "paper.pdf"++ monkeypatch.setattr(connector, "_download_pdf", fake_download)+ items = [_attachment_item("ATTACH1", "2026-01-02T00:00:00Z")]+ with patch.object(connector, "_iter_pdf_attachments", return_value=iter(items)):+ batches = list(connector.load_from_state())+ assert len(batches) == 1+ assert attempts["count"] == 3+++def test_zotero_yields_batches_incrementally(monkeypatch):+ monkeypatch.setattr(+ zotero_mod,+ "assert_url_is_safe",+ lambda url, allowed_schemes=None: ("api.zotero.org", "1.2.3.4"),+ )+ monkeypatch.setattr(+ zotero_mod,+ "pin_dns",+ lambda host, ip: _nullcontext(),+ )+ connector = ZoteroConnector(zotero_user_id="12345678", batch_size=1)+ connector.load_credentials({"zotero_api_key": "secret"})+ items = [+ _attachment_item("A1", "2026-01-02T00:00:00Z"),+ _attachment_item("A2", "2026-01-03T00:00:00Z"),+ ]+ with patch("common.data_source.zotero_connector.rl_requests.get") as mock_get:+ list_response = MagicMock()+ list_response.status_code = 200+ list_response.json.return_value = items+ mock_get.side_effect = [+ list_response,+ _file_response(b"%PDF"),+ _file_response(b"%PDF"),+ ]+ batches = list(connector.load_from_state())+ assert len(batches) == 2+ assert batches[0][0].id.endswith(":A1")+ assert batches[1][0].id.endswith(":A2")diff --git a/test/unit_test/feishu_wiki_test_support.py b/test/unit_test/feishu_wiki_test_support.pyindex f79381b344..e66d926973 100644--- a/test/unit_test/feishu_wiki_test_support.py+++ b/test/unit_test/feishu_wiki_test_support.py@@ -33,6 +33,7 @@ _UNRELATED_PROVIDERS = {"webdav_connector": "WebDAVConnector","xquik_connector": "XquikConnector","zendesk_connector": "ZendeskConnector",+ "zotero_connector": "ZoteroConnector","azure_devops.connector": "AzureDevOpsConnector","bitbucket.connector": "BitbucketConnector","github.connector": "GithubConnector",diff --git a/web/src/locales/en.ts b/web/src/locales/en.tsindex b08430686e..2091ed9a65 100644--- a/web/src/locales/en.ts+++ b/web/src/locales/en.ts@@ -1651,6 +1651,33 @@ Example: Virtual Hosted Style`,'Email address that has access to the Drive content being synced',zendeskDescription:'Connect your Zendesk to sync tickets, articles, and other content.',+ zoteroDescription:+ 'Connect your Zotero library to sync PDF attachments from papers and references.',+ dataSourceFieldZoteroUserId: 'Zotero user ID',+ dataSourceFieldZoteroApiKey: 'Zotero API key',+ dataSourceFieldZoteroStorageMode: 'Attachment storage',+ dataSourceOptionZoteroCloudStorage: 'Zotero cloud',+ dataSourceOptionZoteroWebdav: 'WebDAV',+ dataSourceFieldZoteroWebdavPassword: 'WebDAV password',+ zoteroUserIdTip:+ 'Your numeric Zotero user ID from https://www.zotero.org/settings/keys.',+ zoteroApiKeyTip:+ 'Create a personal API key with library access at https://www.zotero.org/settings/keys.',+ zoteroStorageModeTip:+ 'Choose Zotero cloud to download files via the Web API, or WebDAV if your attachments are stored on your own WebDAV server.',+ zoteroWebdavUrlTip:+ 'HTTPS URL of your WebDAV server. Zotero cloud storage does not provide a hosted WebDAV endpoint.',+ zoteroWebdavUrlRequired:+ 'WebDAV server URL is required when attachment storage is WebDAV.',+ dataSourceFieldZoteroWebdavUsername: 'WebDAV username',+ zoteroWebdavUsernameTip:+ 'Username for WebDAV Basic authentication. This is often different from your Zotero user ID.',+ zoteroWebdavUsernameRequired:+ 'WebDAV username is required when attachment storage is WebDAV.',+ zoteroWebdavPasswordTip:+ 'WebDAV password from Zotero storage settings (not your Zotero account password).',+ zoteroWebdavPasswordRequired:+ 'WebDAV password is required when attachment storage is WebDAV.',google_driveMyDriveEmailsTip:'Comma-separated emails whose "My Drive" contents should be indexed (include the primary admin).',google_driveSharedFoldersTip:diff --git a/web/src/pages/user-setting/data-source/constant/index.tsx b/web/src/pages/user-setting/data-source/constant/index.tsxindex 4cc971efff..30685f0ab8 100644--- a/web/src/pages/user-setting/data-source/constant/index.tsx+++ b/web/src/pages/user-setting/data-source/constant/index.tsx@@ -59,6 +59,7 @@ export enum DataSourceKey {DISCORD = 'discord',XQUIK = 'xquik',ZENDESK = 'zendesk',+ ZOTERO = 'zotero',WEBDAV = 'webdav',AIRTABLE = 'airtable',ASANA = 'asana',@@ -152,6 +153,9 @@ export const DataSourceFeatureVisibilityMap: Partial<[DataSourceKey.ZENDESK]: {syncDeletedFiles: true,},+ [DataSourceKey.ZOTERO]: {+ syncDeletedFiles: true,+ },[DataSourceKey.SEAFILE]: {syncDeletedFiles: true,},@@ -375,6 +379,11 @@ export const generateDataSourceInfo = (t: TFunction) => {description: t(`setting.${DataSourceKey.ZENDESK}Description`),icon: <SvgIcon name={'data-source/zendesk'} width={38} />,},+ [DataSourceKey.ZOTERO]: {+ name: 'Zotero',+ description: t(`setting.${DataSourceKey.ZOTERO}Description`),+ icon: <BookOpen className="text-text-primary" size={22} />,+ },[DataSourceKey.SEAFILE]: {name: 'SeaFile',description: t(`setting.${DataSourceKey.SEAFILE}Description`),@@ -1487,6 +1496,93 @@ const generateDataSourceFormFields = (t: TFunction) => ({],},],+ [DataSourceKey.ZOTERO]: [+ {+ label: t('setting.dataSourceFieldZoteroUserId'),+ name: 'config.zotero_user_id',+ type: FormFieldType.Text,+ required: true,+ placeholder: '12345678',+ tooltip: t('setting.zoteroUserIdTip'),+ },+ {+ label: t('setting.dataSourceFieldZoteroApiKey'),+ name: 'config.credentials.zotero_api_key',+ type: FormFieldType.Password,+ required: true,+ tooltip: t('setting.zoteroApiKeyTip'),+ },+ {+ label: t('setting.dataSourceFieldZoteroStorageMode'),+ name: 'config.storage_mode',+ type: FormFieldType.Segmented,+ required: true,+ options: [+ {+ label: t('setting.dataSourceOptionZoteroCloudStorage'),+ value: 'zotero_storage',+ },+ {+ label: t('setting.dataSourceOptionZoteroWebdav'),+ value: 'webdav',+ },+ ],+ tooltip: t('setting.zoteroStorageModeTip'),+ },+ {+ label: t('setting.dataSourceFieldWebdavServerUrl'),+ name: 'config.webdav_url',+ type: FormFieldType.Text,+ required: false,+ placeholder: 'https://webdav.example.com',+ tooltip: t('setting.zoteroWebdavUrlTip'),+ shouldRender: (formValues: any) =>+ formValues?.config?.storage_mode === 'webdav',+ customValidate: (val: string, formValues: any) => {+ if (formValues?.config?.storage_mode === 'webdav' && !val?.trim()) {+ return t('setting.zoteroWebdavUrlRequired');+ }+ return true;+ },+ },+ {+ label: t('setting.dataSourceFieldZoteroWebdavUsername'),+ name: 'config.credentials.webdav_username',+ type: FormFieldType.Text,+ required: false,+ tooltip: t('setting.zoteroWebdavUsernameTip'),+ shouldRender: (formValues: any) =>+ formValues?.config?.storage_mode === 'webdav',+ customValidate: (val: string, formValues: any) => {+ if (formValues?.config?.storage_mode === 'webdav' && !val?.trim()) {+ return t('setting.zoteroWebdavUsernameRequired');+ }+ return true;+ },+ },+ {+ label: t('setting.dataSourceFieldZoteroWebdavPassword'),+ name: 'config.credentials.webdav_password',+ type: FormFieldType.Password,+ required: false,+ tooltip: t('setting.zoteroWebdavPasswordTip'),+ shouldRender: (formValues: any) =>+ formValues?.config?.storage_mode === 'webdav',+ customValidate: (val: string, formValues: any) => {+ if (formValues?.config?.storage_mode === 'webdav' && !val?.trim()) {+ return t('setting.zoteroWebdavPasswordRequired');+ }+ return true;+ },+ },+ {+ label: t('setting.dataSourceFieldBatchSize'),+ name: 'config.batch_size',+ type: FormFieldType.Number,+ required: false,+ placeholder: '4',+ },+ ],[DataSourceKey.SEAFILE]: seafileConstant(t),[DataSourceKey.MYSQL]: [{@@ -2507,6 +2603,21 @@ export const DataSourceFormDefaultValues = {},},},+ [DataSourceKey.ZOTERO]: {+ name: '',+ source: DataSourceKey.ZOTERO,+ config: {+ zotero_user_id: '',+ storage_mode: 'zotero_storage',+ webdav_url: '',+ batch_size: 4,+ credentials: {+ zotero_api_key: '',+ webdav_username: '',+ webdav_password: '',+ },+ },+ },[DataSourceKey.SEAFILE]: {name: '',source: DataSourceKey.SEAFILE,