Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: COS file overwrite not reflected after syncing connectors (#2366)

Rico Furtado661805a3c37f
diff --git a/enhancements/connectors/azure_blob/connector.py b/enhancements/connectors/azure_blob/connector.py
index f7509553b..a1543faf2 100644
--- a/enhancements/connectors/azure_blob/connector.py
+++ b/enhancements/connectors/azure_blob/connector.py
@@ -25,7 +25,13 @@ from config.settings import (
is_azure_blob_enabled,
is_dev_azure_blob_enabled,
)
-from connectors.base import BaseConnector, ConnectorDocument, DocumentACL
+from connectors.base import (
+ CONTENT_ETAG_METADATA_KEY,
+ BaseConnector,
+ ConnectorDocument,
+ DocumentACL,
+ normalize_etag,
+)
from utils.logging_config import get_logger
from .auth import account_name_from_config, create_blob_service_client
@@ -251,6 +257,9 @@ class AzureBlobConnector(BaseConnector):
"key": blob.name,
"size": getattr(blob, "size", 0),
"modified_time": last_modified.isoformat() if last_modified else None,
+ # Change detection compares this against the tag
+ # stored at ingest; it moves on every overwrite.
+ "etag": normalize_etag(getattr(blob, "etag", None)),
}
)
if max_files and len(files) >= max_files:
@@ -269,16 +278,19 @@ class AzureBlobConnector(BaseConnector):
props = getattr(downloader, "properties", None)
content_type = ""
last_modified = None
+ etag = None
size = len(content)
if props is not None:
settings = getattr(props, "content_settings", None)
content_type = getattr(settings, "content_type", "") or ""
last_modified = getattr(props, "last_modified", None)
+ etag = getattr(props, "etag", None)
size = getattr(props, "size", None) or len(content)
return {
"content": content,
"content_type": content_type,
"last_modified": last_modified,
+ "etag": etag,
"size": size,
}
@@ -352,6 +364,9 @@ class AzureBlobConnector(BaseConnector):
"azure_container": container_name,
"azure_blob": blob_name,
"size": size,
+ # Persisted at ingest so a later sync can tell an overwritten
+ # blob from an untouched one without trusting clocks.
+ CONTENT_ETAG_METADATA_KEY: normalize_etag(result.get("etag")),
},
)
diff --git a/enhancements/connectors/ibm_cos/connector.py b/enhancements/connectors/ibm_cos/connector.py
index 6aef47cc4..ae8e32c5b 100644
--- a/enhancements/connectors/ibm_cos/connector.py
+++ b/enhancements/connectors/ibm_cos/connector.py
@@ -11,7 +11,13 @@ from posixpath import basename
from typing import Any
from config.settings import IBM_AUTH_ENABLED, is_dev_ibm_cos_enabled
-from connectors.base import BaseConnector, ConnectorDocument, DocumentACL
+from connectors.base import (
+ CONTENT_ETAG_METADATA_KEY,
+ BaseConnector,
+ ConnectorDocument,
+ DocumentACL,
+ normalize_etag,
+)
from utils.logging_config import get_logger
from .auth import create_ibm_cos_client, create_ibm_cos_resource
@@ -249,6 +255,9 @@ class IBMCOSConnector(BaseConnector):
"modified_time": obj.last_modified.isoformat()
if obj.last_modified
else None,
+ # Change detection compares this against the tag
+ # stored at ingest; it moves on every overwrite.
+ "etag": normalize_etag(getattr(obj, "e_tag", None)),
}
)
if max_files and len(files) >= max_files:
@@ -274,6 +283,7 @@ class IBMCOSConnector(BaseConnector):
"modified_time": obj["LastModified"].isoformat()
if obj.get("LastModified")
else None,
+ "etag": normalize_etag(obj.get("ETag")),
}
)
if max_files and len(files) >= max_files:
@@ -342,6 +352,9 @@ class IBMCOSConnector(BaseConnector):
"ibm_cos_bucket": bucket_name,
"ibm_cos_key": key,
"size": size,
+ # Persisted at ingest so a later sync can tell an overwritten
+ # object from an untouched one without trusting clocks.
+ CONTENT_ETAG_METADATA_KEY: normalize_etag(response.get("ETag")),
},
)
diff --git a/frontend/app/api/mutations/useSyncConnector.ts b/frontend/app/api/mutations/useSyncConnector.ts
index 3017d692c..61799dce0 100644
--- a/frontend/app/api/mutations/useSyncConnector.ts
+++ b/frontend/app/api/mutations/useSyncConnector.ts
@@ -29,12 +29,19 @@ export interface SyncPreviewResponse {
* connection was unauthenticated). UI should reflect that deletions
* cannot be predicted in that case. */
orphans_available: boolean;
+ /** Files whose source copy changed and that the sync will re-ingest. */
+ updates: OrphanFile[];
+ /** False when this connector can't tell in advance — it decides per file
+ * while ingesting. Not the same as "nothing will be updated". */
+ updates_available: boolean;
}
export interface SyncAllPreviewResponse {
orphans_by_type: Record<string, OrphanFile[]>;
synced_count_by_type: Record<string, number>;
orphans_available_by_type: Record<string, boolean>;
+ updates_by_type: Record<string, OrphanFile[]>;
+ updates_available_by_type: Record<string, boolean>;
}
// Sync all cloud connectors
diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsx
index 268b86f69..92c4309ec 100644
--- a/frontend/app/knowledge/page.tsx
+++ b/frontend/app/knowledge/page.tsx
@@ -1247,6 +1247,8 @@ function SearchPage() {
isSyncAll
orphansByType={syncPreview?.orphans_by_type}
orphansAvailableByType={syncPreview?.orphans_available_by_type}
+ updatesByType={syncPreview?.updates_by_type}
+ updatesAvailableByType={syncPreview?.updates_available_by_type}
syncedCountByType={syncPreview?.synced_count_by_type}
/>
</>
diff --git a/frontend/components/knowledge-actions-dropdown.tsx b/frontend/components/knowledge-actions-dropdown.tsx
index f92e30569..91ea6a36a 100644
--- a/frontend/components/knowledge-actions-dropdown.tsx
+++ b/frontend/components/knowledge-actions-dropdown.tsx
@@ -252,6 +252,20 @@ export const KnowledgeActionsDropdown = ({
isSyncing={syncConnectorMutation.isPending}
connectorType={connectorType}
orphans={syncPreview?.orphans}
+ orphansAvailableByType={
+ connectorType && syncPreview
+ ? { [connectorType]: syncPreview.orphans_available }
+ : undefined
+ }
+ updates={syncPreview?.updates}
+ // Every connector in CLOUD_CONNECTOR_TYPES re-reads each file during
+ // sync and decides then, so this is false and the dialog reports
+ // "re-checked" rather than a count it cannot know in advance.
+ updatesAvailableByType={
+ connectorType && syncPreview
+ ? { [connectorType]: syncPreview.updates_available }
+ : undefined
+ }
syncedCount={syncPreview?.synced_count}
/>
</>
diff --git a/frontend/components/knowledge-search-bar.tsx b/frontend/components/knowledge-search-bar.tsx
index bb74acc39..ea945a45d 100644
--- a/frontend/components/knowledge-search-bar.tsx
+++ b/frontend/components/knowledge-search-bar.tsx
@@ -216,6 +216,8 @@ export const KnowledgeSearchBar = () => {
isSyncAll
orphansByType={syncPreview?.orphans_by_type}
orphansAvailableByType={syncPreview?.orphans_available_by_type}
+ updatesByType={syncPreview?.updates_by_type}
+ updatesAvailableByType={syncPreview?.updates_available_by_type}
syncedCountByType={syncPreview?.synced_count_by_type}
/>
</form>
diff --git a/frontend/components/sync-confirm-dialog-data.test.ts b/frontend/components/sync-confirm-dialog-data.test.ts
new file mode 100644
index 000000000..bcca61c9e
--- /dev/null
+++ b/frontend/components/sync-confirm-dialog-data.test.ts
@@ -0,0 +1,142 @@
+/**
+ * What the sync confirmation dialog tells the user.
+ *
+ * The dialog used to derive "N files will be updated" from the *total synced
+ * count*, which claimed every indexed file would be updated no matter how many
+ * had actually changed. Updates now come from the preview's own list.
+ *
+ * The case worth pinning is the third state: a connector that re-reads every
+ * file and decides during ingest can't predict updates at all. That must read
+ * as "re-checked", never as a confident count and never as "nothing changed".
+ */
+
+import { describe, expect, it } from "vitest";
+import { summarizeSyncPreview } from "./sync-confirm-dialog-data";
+
+const file = (id: string, name: string) => ({
+ document_id: id,
+ filename: name,
+});
+
+describe("summarizeSyncPreview — single connector", () => {
+ it("reports the changed files, not the synced total", () => {
+ const summary = summarizeSyncPreview({
+ connectorType: "ibm_cos",
+ updates: [file("c::b", "b.pdf")],
+ updatesAvailableByType: { ibm_cos: true },
+ syncedCount: 40,
+ });
+
+ expect(summary.totalUpdates).toBe(1);
+ expect(summary.updatesByType).toEqual({
+ ibm_cos: [file("c::b", "b.pdf")],
+ });
+ // The other 39 synced files are untouched and must not be counted.
+ expect(summary.totalRechecked).toBe(0);
+ });
+
+ it("reports nothing when a predicting connector has no changes", () => {
+ const summary = summarizeSyncPreview({
+ connectorType: "ibm_cos",
+ updates: [],
+ updatesAvailableByType: { ibm_cos: true },
+ syncedCount: 40,
+ });
+
+ expect(summary.totalUpdates).toBe(0);
+ expect(summary.totalRechecked).toBe(0);
+ });
+
+ it("falls back to re-checked when the connector can't predict updates", () => {
+ const summary = summarizeSyncPreview({
+ connectorType: "google_drive",
+ updates: [],
+ updatesAvailableByType: { google_drive: false },
+ syncedCount: 7,
+ });
+
+ expect(summary.totalUpdates).toBe(0);
+ expect(summary.totalRechecked).toBe(7);
+ expect(summary.recheckedByType).toEqual({ google_drive: 7 });
+ });
+
+ it("keeps deletions separate from updates", () => {
+ const summary = summarizeSyncPreview({
+ connectorType: "ibm_cos",
+ orphans: [file("c::gone", "gone.pdf")],
+ orphansAvailableByType: { ibm_cos: true },
+ updates: [file("c::b", "b.pdf")],
+ updatesAvailableByType: { ibm_cos: true },
+ syncedCount: 2,
+ });
+
+ expect(summary.totalOrphans).toBe(1);
+ expect(summary.totalUpdates).toBe(1);
+ expect(summary.unavailableConnectors).toEqual([]);
+ });
+});
+
+describe("summarizeSyncPreview — sync all", () => {
+ it("groups updates by connector and drops empty groups", () => {
+ const summary = summarizeSyncPreview({
+ isSyncAll: true,
+ updatesByType: {
+ ibm_cos: [file("c::b", "b.pdf"), file("c::c", "c.pdf")],
+ aws_s3: [],
+ },
+ updatesAvailableByType: { ibm_cos: true, aws_s3: true },
+ syncedCountByType: { ibm_cos: 10, aws_s3: 4 },
+ });
+
+ expect(summary.totalUpdates).toBe(2);
+ expect(Object.keys(summary.updatesByType)).toEqual(["ibm_cos"]);
+ });
+
+ it("mixes predicting and non-predicting connectors without conflating them", () => {
+ const summary = summarizeSyncPreview({
+ isSyncAll: true,
+ updatesByType: { ibm_cos: [file("c::b", "b.pdf")], google_drive: [] },
+ updatesAvailableByType: { ibm_cos: true, google_drive: false },
+ syncedCountByType: { ibm_cos: 10, google_drive: 5 },
+ });
+
+ expect(summary.totalUpdates).toBe(1);
+ expect(summary.recheckedByType).toEqual({ google_drive: 5 });
+ // ibm_cos predicted its updates, so its synced total is not re-checked too.
+ expect(summary.totalRechecked).toBe(5);
+ });
+
+ it("surfaces connectors whose deletion check couldn't complete", () => {
+ const summary = summarizeSyncPreview({
+ isSyncAll: true,
+ orphansByType: { ibm_cos: [] },
+ orphansAvailableByType: { ibm_cos: false, aws_s3: true },
+ syncedCountByType: { ibm_cos: 3 },
+ });
+
+ expect(summary.unavailableConnectors).toEqual(["ibm_cos"]);
+ expect(summary.totalOrphans).toBe(0);
+ });
+
+ it("treats a missing availability flag as unpredictable, not as no work", () => {
+ // A call site that forgets updatesAvailableByType must not make the dialog
+ // claim there is nothing to do: it understates to zero and the user skips a
+ // sync that had pending changes. Unknown fails towards "re-checked".
+ const summary = summarizeSyncPreview({
+ connectorType: "google_drive",
+ syncedCount: 7,
+ });
+
+ expect(summary.totalUpdates).toBe(0);
+ expect(summary.totalRechecked).toBe(7);
+ });
+
+ it("is empty for an empty preview", () => {
+ const summary = summarizeSyncPreview({ isSyncAll: true });
+
+ expect(summary.totalOrphans).toBe(0);
+ expect(summary.totalUpdates).toBe(0);
+ expect(summary.totalRechecked).toBe(0);
+ expect(summary.unavailableConnectors).toEqual([]);
+ });
+});
diff --git a/frontend/components/sync-confirm-dialog-data.ts b/frontend/components/sync-confirm-dialog-data.ts
new file mode 100644
index 000000000..f980f559f
--- /dev/null
+++ b/frontend/components/sync-confirm-dialog-data.ts
@@ -0,0 +1,108 @@
+import type { OrphanFile } from "@/app/api/mutations/useSyncConnector";
+
+/** What a sync will do, shaped for the confirmation dialog.
+ *
+ * Kept out of the .tsx so the part that can actually be wrong — deciding what
+ * the user is told — is plain logic with its own tests.
+ */
+export interface SyncPreviewSummary {
+ /** Deletions grouped by connector_type (single-mode collapses to one entry). */
+ orphansByType: Record<string, OrphanFile[]>;
+ /** Known updates grouped by connector_type. */
+ updatesByType: Record<string, OrphanFile[]>;
+ /** Synced totals for connectors that can't predict updates, by connector_type. */
+ recheckedByType: Record<string, number>;
+ /** Connectors whose orphan detection couldn't complete. */
+ unavailableConnectors: string[];
+ totalOrphans: number;
+ totalUpdates: number;
+ totalRechecked: number;
+}
+
+export interface SyncPreviewInput {
+ orphans?: OrphanFile[];
+ orphansByType?: Record<string, OrphanFile[]>;
+ orphansAvailableByType?: Record<string, boolean>;
+ updates?: OrphanFile[];
+ updatesByType?: Record<string, OrphanFile[]>;
+ updatesAvailableByType?: Record<string, boolean>;
+ syncedCount?: number;
+ syncedCountByType?: Record<string, number>;
+ connectorType?: string;
+ isSyncAll?: boolean;
+}
+
+const sumLengths = (lists: OrphanFile[][]): number =>
+ lists.reduce((sum, list) => sum + list.length, 0);
+
+/** Fold the single-connector and sync-all response shapes into one structure.
+ *
+ * The distinction that matters here is between a connector reporting *no*
+ * updates and one that cannot predict them: connectors on `replace_always`
+ * change detection re-read every file and decide during ingest, so their synced
+ * total is surfaced as "re-checked" instead of being presented as a count of
+ * files that will change.
+ */
+export const summarizeSyncPreview = (
+ props: SyncPreviewInput,
+): SyncPreviewSummary => {
+ const {
+ orphans,
+ orphansByType,
+ orphansAvailableByType,
+ updates,
+ updatesByType,
+ updatesAvailableByType,
+ syncedCount,
+ syncedCountByType,
+ connectorType,
+ isSyncAll,
+ } = props;
+
+ const byConnector = (
+ grouped: Record<string, OrphanFile[]> | undefined,
+ single: OrphanFile[] | undefined,
+ ): Record<string, OrphanFile[]> =>
+ isSyncAll
+ ? Object.fromEntries(
+ Object.entries(grouped ?? {}).filter(([, list]) => list.length > 0),
+ )
+ : single && single.length > 0 && connectorType
+ ? { [connectorType]: single }
+ : {};
+
+ const normalizedOrphans = byConnector(orphansByType, orphans);
+ const normalizedUpdates = byConnector(updatesByType, updates);
+
+ const recheckedByType: Record<string, number> = {};
+ const syncedTotals: Record<string, number> = isSyncAll
+ ? (syncedCountByType ?? {})
+ : connectorType && syncedCount
+ ? { [connectorType]: syncedCount }
+ : {};
+ // `!== true` rather than `=== false`: a connector that did not report its
+ // availability is one whose updates we cannot predict, which is the same
+ // situation as an explicit false. Requiring the explicit false made a caller
+ // that forgot to pass the field render as "nothing to change" — telling the
+ // user there is no work when files are in fact pending. Unknown must fail
+ // towards "re-checked", which is true in every case.
+ for (const [type, count] of Object.entries(syncedTotals)) {
+ if (count > 0 && (updatesAvailableByType ?? {})[type] !== true) {
+ recheckedByType[type] = count;
+ }
+ }
+
+ const unavailableConnectors = Object.entries(orphansAvailableByType ?? {})
+ .filter(([, available]) => !available)
+ .map(([type]) => type);
+
+ return {
+ orphansByType: normalizedOrphans,
+ updatesByType: normalizedUpdates,
+ recheckedByType,
+ unavailableConnectors,
+ totalOrphans: sumLengths(Object.values(normalizedOrphans)),
+ totalUpdates: sumLengths(Object.values(normalizedUpdates)),
+ totalRechecked: Object.values(recheckedByType).reduce((s, n) => s + n, 0),
+ };
+};
diff --git a/frontend/components/sync-confirm-dialog.test.tsx b/frontend/components/sync-confirm-dialog.test.tsx
new file mode 100644
index 000000000..90abd5d74
--- /dev/null
+++ b/frontend/components/sync-confirm-dialog.test.tsx
@@ -0,0 +1,170 @@
+import { describe, expect, it, vi } from "vitest";
+import { renderWithProviders, screen } from "@/test-utils/render";
+import { SyncConfirmDialog } from "./sync-confirm-dialog";
+
+/**
+ * What the sync confirmation dialog tells the user before they commit.
+ *
+ * The update count used to come from the *total synced count*, so a sync of 40
+ * COS files with one modified announced "40 files will be updated" and then
+ * updated one. These pin the three distinct states the dialog now has to keep
+ * apart, because conflating any two of them misleads the user about what the
+ * button is going to do:
+ *
+ * - files that will change (named, from the preview)
+ * - files that can only be re-checked (the connector cannot predict)
+ * - nothing to do
+ *
+ * The dialog is presentational, but it's driven by radix primitives that need
+ * a portal, so it renders through the shared harness rather than being asserted
+ * on summarizeSyncPreview's return value alone (that lives in
+ * sync-confirm-dialog-data.test.ts).
+ */
+
+const noop = () => {};
+
+const file = (id: string, name: string) => ({
+ document_id: id,
+ filename: name,
+});
+
+function renderDialog(props: Partial<Parameters<typeof SyncConfirmDialog>[0]>) {
+ return renderWithProviders(
+ <SyncConfirmDialog
+ open
+ onOpenChange={noop}
+ onConfirm={vi.fn()}
+ {...props}
+ />,
+ );
+}
+
+describe("SyncConfirmDialog — updates", () => {
+ it("names the files that changed instead of counting everything synced", async () => {
+ renderDialog({
+ connectorType: "ibm_cos",
+ updates: [file("c::b", "b.pdf")],
+ updatesAvailableByType: { ibm_cos: true },
+ syncedCount: 40,
+ });
+
+ expect(await screen.findByText("1 file will be updated")).toBeVisible();
+ expect(screen.getByText("b.pdf")).toBeVisible();
+ // The 39 untouched files must not be presented as updates.
+ expect(screen.queryByText("40 files will be updated")).toBeNull();
+ });
+
+ it("groups changed files by connector when syncing all", async () => {
+ renderDialog({
+ isSyncAll: true,
+ updatesByType: {
+ ibm_cos: [file("c::b", "b.pdf")],
+ aws_s3: [file("s::c", "c.pdf")],
+ },
+ updatesAvailableByType: { ibm_cos: true, aws_s3: true },
+ syncedCountByType: { ibm_cos: 5, aws_s3: 5 },
+ });
+
+ expect(await screen.findByText("2 files will be updated")).toBeVisible();
+ expect(screen.getByText("b.pdf")).toBeVisible();
+ expect(screen.getByText("c.pdf")).toBeVisible();
+ });
+});
+
+describe("SyncConfirmDialog — re-checks", () => {
+ it("says re-checked, not updated, when the connector can't predict", async () => {
+ renderDialog({
+ connectorType: "google_drive",
+ updates: [],
+ updatesAvailableByType: { google_drive: false },
+ syncedCount: 7,
+ });
+
+ expect(await screen.findByText("7 files will be re-checked")).toBeVisible();
+ expect(
+ screen.getByText(
+ "Sync re-reads these files and updates the ones whose content changed.",
+ ),
+ ).toBeVisible();
+ expect(screen.queryByText("7 files will be updated")).toBeNull();
+ });
+
+ it("breaks the re-check count down per connector when several apply", async () => {
+ renderDialog({
+ isSyncAll: true,
+ updatesByType: { google_drive: [], sharepoint: [] },
+ updatesAvailableByType: { google_drive: false, sharepoint: false },
+ syncedCountByType: { google_drive: 4, sharepoint: 2 },
+ });
+
+ expect(await screen.findByText("6 files will be re-checked")).toBeVisible();
+ expect(screen.getByText("4")).toBeVisible();
+ expect(screen.getByText("2")).toBeVisible();
+ });
+});
+
+describe("SyncConfirmDialog — per-file sync call shape", () => {
+ it("reports re-checks for a replace_always connector, not 'up to date'", async () => {
+ // The shape knowledge-actions-dropdown passes: a single-connector preview
+ // for google_drive / onedrive / sharepoint, which cannot predict updates.
+ // Dropping updatesAvailableByType here made the dialog say nothing would
+ // change while files were pending — worse than the overstated count it
+ // replaced, because the user skips the sync entirely.
+ renderDialog({
+ connectorType: "google_drive",
+ orphans: [],
+ orphansAvailableByType: { google_drive: true },
+ updates: [],
+ updatesAvailableByType: { google_drive: false },
+ syncedCount: 7,
+ });
+
+ expect(await screen.findByText("7 files will be re-checked")).toBeVisible();
+ expect(screen.queryByText("Nothing to change")).toBeNull();
+ });
+});
+
+describe("SyncConfirmDialog — nothing to do", () => {
+ it("says so rather than showing a zero count", async () => {
+ renderDialog({
+ connectorType: "ibm_cos",
+ orphans: [],
+ updates: [],
+ updatesAvailableByType: { ibm_cos: true },
+ syncedCount: 12,
+ });
+
+ expect(await screen.findByText("Nothing to change")).toBeVisible();
+ expect(
+ screen.getByText(
+ "Sync reconciles files already in Knowledge; it does not add new ones from the source.",
+ ),
+ ).toBeVisible();
+ expect(screen.queryByText("0 files will be updated")).toBeNull();
+ // The preview only diffs already-indexed files against the source; it never
+ // enumerates new remote files, so the empty state must not imply it did.
+ expect(screen.queryByText(/were added/i)).toBeNull();
+ expect(screen.queryByText(/everything is already up to date/i)).toBeNull();
+ });
+});
+
+describe("SyncConfirmDialog — deletions stay dominant", () => {
+ it("shows deletions alongside updates and keeps the destructive CTA", async () => {
+ renderDialog({
+ connectorType: "ibm_cos",
+ orphans: [file("c::gone", "gone.pdf")],
+ orphansAvailableByType: { ibm_cos: true },
+ updates: [file("c::b", "b.pdf")],
+ updatesAvailableByType: { ibm_cos: true },
+ syncedCount: 2,
+ });
+
+ expect(await screen.findByText("1 file will be deleted")).toBeVisible();
+ expect(screen.getByText("1 file will be updated")).toBeVisible();
+ expect(screen.getByText("gone.pdf")).toBeVisible();
+ expect(screen.getByText("b.pdf")).toBeVisible();
+ expect(
+ screen.getByRole("button", { name: /delete & sync/i }),
+ ).toBeVisible();
+ });
+});
diff --git a/frontend/components/sync-confirm-dialog.tsx b/frontend/components/sync-confirm-dialog.tsx
index 5e887706f..dc55badad 100644
--- a/frontend/components/sync-confirm-dialog.tsx
+++ b/frontend/components/sync-confirm-dialog.tsx
@@ -4,6 +4,7 @@ import { AlertTriangle, Check, Loader2, RefreshCw, Trash2 } from "lucide-react";
import type React from "react";
import type { OrphanFile } from "@/app/api/mutations/useSyncConnector";
import { getConnectorLabel } from "@/lib/connectors/registry";
+import { summarizeSyncPreview } from "./sync-confirm-dialog-data";
import { Alert, AlertDescription, AlertTitle } from "./ui/alert";
import { Badge } from "./ui/badge";
import { Button } from "./ui/button";
@@ -44,9 +45,17 @@ interface SyncConfirmDialogProps {
/** Per-connector availability flag — false means orphan detection couldn't
* complete safely (e.g. unauthenticated connection). */
orphansAvailableByType?: Record<string, boolean>;
- /** Single-connector mode: total files that will be updated. */
+ /** Single-connector mode: files whose source copy changed and will be re-ingested. */
+ updates?: OrphanFile[];
+ /** Sync-all mode: changed files grouped by connector_type. */
+ updatesByType?: Record<string, OrphanFile[]>;
+ /** Per-connector flag — false means the connector can't predict updates (it
+ * decides per file while ingesting), NOT that nothing will be updated. Those
+ * connectors are reported as "re-checked" using their synced count. */
+ updatesAvailableByType?: Record<string, boolean>;
+ /** Single-connector mode: total files currently synced for this connector. */
syncedCount?: number;
- /** Sync-all mode: per-connector update counts. */
+ /** Sync-all mode: per-connector synced totals. */
syncedCountByType?: Record<string, number>;
/** Single-connector mode: connector type for the title (e.g. "sharepoint"). */
connectorType?: string;
@@ -54,79 +63,6 @@ interface SyncConfirmDialogProps {
isSyncAll?: boolean;
}
-interface NormalizedData {
- /** Deletions grouped by connector_type (single-mode collapses to one entry). */
- orphansByType: Record<string, OrphanFile[]>;
- /** Update counts grouped by connector_type. */
- updatesByType: Record<string, number>;
- /** Connectors whose orphan detection couldn't complete. */
- unavailableConnectors: string[];
- totalOrphans: number;
- totalUpdates: number;
-}
-
-const normalize = (
- props: Pick<
- SyncConfirmDialogProps,
- | "orphans"
- | "orphansByType"
- | "orphansAvailableByType"
- | "syncedCount"
- | "syncedCountByType"
- | "connectorType"
- | "isSyncAll"
- >,
-): NormalizedData => {
- const {
- orphans,
- orphansByType,
- orphansAvailableByType,
- syncedCount,
- syncedCountByType,
- connectorType,
- isSyncAll,
- } = props;
-
- const normalizedOrphans: Record<string, OrphanFile[]> = isSyncAll
- ? Object.fromEntries(
- Object.entries(orphansByType ?? {}).filter(
- ([, list]) => list.length > 0,
- ),
- )
- : orphans && orphans.length > 0 && connectorType
- ? { [connectorType]: orphans }
- : {};
-
- const normalizedUpdates: Record<string, number> = isSyncAll
- ? Object.fromEntries(
- Object.entries(syncedCountByType ?? {}).filter(([, n]) => n > 0),
- )
- : syncedCount && syncedCount > 0 && connectorType
- ? { [connectorType]: syncedCount }
- : {};
-
- const unavailableConnectors = Object.entries(orphansAvailableByType ?? {})
- .filter(([, available]) => !available)
- .map(([type]) => type);
-
- const totalOrphans = Object.values(normalizedOrphans).reduce(
- (sum, list) => sum + list.length,
- 0,
- );
- const totalUpdates = Object.values(normalizedUpdates).reduce(
- (sum, n) => sum + n,
- 0,
- );
-
- return {
- orphansByType: normalizedOrphans,
- updatesByType: normalizedUpdates,
- unavailableConnectors,
- totalOrphans,
- totalUpdates,
- };
-};
-
const OrphanList = ({ list }: { list: OrphanFile[] }) => (
<ul className="space-y-1 text-sm">
{list.map((o) => (
@@ -219,7 +155,7 @@ const UpdatesAlert = ({
totalUpdates,
isSyncAll,
}: {
- updatesByType: Record<string, number>;
+ updatesByType: Record<string, OrphanFile[]>;
totalUpdates: number;
isSyncAll: boolean;
}) => {
@@ -229,9 +165,63 @@ const UpdatesAlert = ({
<Alert>
<RefreshCw className="size-5" />
<AlertTitle>{pluralize(totalUpdates, "file")} will be updated</AlertTitle>
- {isSyncAll && entries.length > 1 ? (
- <AlertDescription className="col-start-2 block min-w-0">
- <ul className="mt-1 space-y-0.5">
+ <AlertDescription className="col-start-2 block min-w-0">
+ <p>These files changed at the source since they were last ingested.</p>
+ <ScrollArea className="mt-2 max-h-60 w-full">
+ {isSyncAll && entries.length > 1 ? (
+ <div className="space-y-3 pr-2">
+ {entries.map(([type, list], index) => (
+ <div key={type}>
+ <div className="text-xs font-semibold uppercase tracking-wide mb-1">
+ {formatConnectorLabel(type)} ({list.length})
+ </div>
+ <OrphanList list={list} />
+ {index < entries.length - 1 ? (
+ <Separator className="mt-3" />
+ ) : null}
+ </div>
+ ))}
+ </div>
+ ) : (
+ <div className="pr-2">
+ <OrphanList list={entries[0]?.[1] ?? []} />
+ </div>
+ )}
+ </ScrollArea>
+ {totalUpdates > SCROLL_HINT_THRESHOLD ? (
+ <p className="mt-2 text-xs italic opacity-80">
+ Scroll to review all {totalUpdates} files.
+ </p>
+ ) : null}
+ </AlertDescription>
+ </Alert>
+ );
+};
+
+/** Connectors that re-read every file during sync and update the ones whose
+ * content actually changed. The count can't be known before the sync runs, so
+ * say that rather than showing a number that looks like a prediction. */
+const RecheckAlert = ({
+ recheckedByType,
+ totalRechecked,
+}: {
+ recheckedByType: Record<string, number>;
+ totalRechecked: number;
+}) => {
+ const entries = Object.entries(recheckedByType);
+
+ return (
+ <Alert>
+ <RefreshCw className="size-5" />
+ <AlertTitle>
+ {pluralize(totalRechecked, "file")} will be re-checked
+ </AlertTitle>
+ <AlertDescription className="col-start-2 block min-w-0 [text-wrap:pretty]">
+ <p>
+ Sync re-reads these files and updates the ones whose content changed.
+ </p>
+ {entries.length > 1 ? (
+ <ul className="mt-2 space-y-0.5">
{entries.map(([type, count]) => (
<li key={type} className="flex justify-between gap-4">
<span>{formatConnectorLabel(type)}</span>
@@ -239,8 +229,8 @@ const UpdatesAlert = ({
</li>
))}
</ul>
- </AlertDescription>
- ) : null}
+ ) : null}
+ </AlertDescription>
</Alert>
);
};
@@ -254,6 +244,9 @@ export const SyncConfirmDialog = ({
orphans,
orphansByType,
orphansAvailableByType,
+ updates,
+ updatesByType,
+ updatesAvailableByType,
syncedCount,
syncedCountByType,
connectorType,
@@ -264,10 +257,13 @@ export const SyncConfirmDialog = ({
onOpenChange(false);
};
- const data = normalize({
+ const data = summarizeSyncPreview({
orphans,
orphansByType,
orphansAvailableByType,
+ updates,
+ updatesByType,
+ updatesAvailableByType,
syncedCount,
syncedCountByType,
connectorType,
@@ -276,15 +272,18 @@ export const SyncConfirmDialog = ({
const {
orphansByType: normOrphans,
- updatesByType,
+ updatesByType: normUpdates,
+ recheckedByType,
unavailableConnectors,
totalOrphans,
totalUpdates,
+ totalRechecked,
} = data;
const hasDeletes = totalOrphans > 0;
const hasUnavailable = unavailableConnectors.length > 0;
const hasUpdates = totalUpdates > 0;
+ const hasRechecks = totalRechecked > 0;
const busy = isLoading || isSyncing;
const title = isSyncAll ? "Sync all connectors" : "Confirm sync";
@@ -296,8 +295,13 @@ export const SyncConfirmDialog = ({
description = `Sync will remove ${pluralize(totalOrphans, "file")}.`;
} else if (hasUnavailable) {
description = "Some connectors couldn't be checked for deletions.";
- } else {
+ } else if (hasUpdates) {
description = `Sync will update ${pluralize(totalUpdates, "file")}.`;
+ } else if (hasRechecks) {
+ description = `Sync will re-check ${pluralize(totalRechecked, "file")}.`;
+ } else {
+ // Scoped to indexed files: the preview never looked for new remote ones.
+ description = "No indexed files need updating or removing.";
}
// CTA variant + copy follows the most-severe state present.
@@ -353,14 +357,40 @@ export const SyncConfirmDialog = ({
renames as a distinct category. Insert between Unavailable
and Updates when the preview endpoint returns rename data. */}
- {/* Updates — informational, count only */}
+ {/* Updates — informational: the files that actually changed */}
{hasUpdates ? (
<UpdatesAlert
- updatesByType={updatesByType}
+ updatesByType={normUpdates}
totalUpdates={totalUpdates}
isSyncAll={isSyncAll}
/>
) : null}
+
+ {/* Re-checks — connectors that can't predict what will change */}
+ {hasRechecks ? (
+ <RecheckAlert
+ recheckedByType={recheckedByType}
+ totalRechecked={totalRechecked}
+ />
+ ) : null}
+
+ {!hasDeletes && !hasUnavailable && !hasUpdates && !hasRechecks ? (
+ <Alert>
+ <Check className="size-5" />
+ <AlertTitle>Nothing to change</AlertTitle>
+ <AlertDescription className="col-start-2 block min-w-0 [text-wrap:pretty]">
+ {/* Only claim what the preview checked. It diffs the files
+ already in Knowledge against the source and never
+ enumerates new remote files — and sync would not ingest
+ them anyway (see bucket_changed_file_ids), so promising
+ "nothing was added" is doubly unverifiable. */}
+ <p>
+ Sync reconciles files already in Knowledge; it does not add
+ new ones from the source.
+ </p>
+ </AlertDescription>
+ </Alert>
+ ) : null}
</div>
)}
diff --git a/src/api/connectors.py b/src/api/connectors.py
index 812540daf..287b5bd3c 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -1,6 +1,6 @@
import copy
from datetime import UTC, datetime
-from typing import Any
+from typing import Any, NamedTuple
from fastapi import Depends, Request
from fastapi.responses import JSONResponse, PlainTextResponse
@@ -8,6 +8,7 @@ from pydantic import BaseModel
from sqlalchemy.ext.asyncio import AsyncSession
from config.settings import get_index_name, is_workspace_oauth_overrides_enabled
+from connectors.base import normalize_etag
from connectors.sharepoint.utils import is_valid_sharepoint_url
from dependencies import (
get_connector_service,
@@ -88,9 +89,10 @@ def _connector_sync_should_replace(connector_type: str) -> bool:
def _connector_uses_timestamp_change_detection(connector_type: str) -> bool:
- """True when sync should re-ingest only files whose remote modified_time is
- newer than the stored one (see bucket_changed_file_ids), instead of
- replacing every indexed file.
+ """True when sync should re-ingest only the files that actually differ at
+ source — by entity tag, or by a newer remote modified_time where there are
+ no tags to compare (see bucket_changed_file_ids) — instead of replacing
+ every indexed file.
Declared per connector via ``BaseConnector.CHANGE_DETECTION`` — the bucket
connectors set ``"timestamp"``; the default is ``"replace_always"``.
@@ -275,21 +277,35 @@ async def get_synced_id_to_filename_map(
return {}
-async def get_synced_id_to_modified_time_map(
+class SyncedFileState(NamedTuple):
+ """What the index knows about an already-ingested connector file.
+
+ ``modified_time_ms`` is the stored source timestamp; ``content_etag`` is the
+ object's stored entity tag. Either can be None for a file ingested before
+ that signal was recorded, or by a path that didn't persist it.
+ """
+
+ modified_time_ms: float | None = None
+ content_etag: str | None = None
+
+
+async def get_synced_file_state_map(
connector_type: str,
user_id: str,
session_manager,
jwt_token: str | None = None,
-) -> dict[str, float | None]:
- """Map each ingested connector source id → its stored ``modified_time`` (epoch ms).
+) -> dict[str, SyncedFileState]:
+ """Map each ingested connector source id → the change-detection state stored for it.
- Powers change detection for bucket connectors: callers compare a blob's remote
- ``modified_time`` against the stored value to decide whether a re-ingest is needed.
+ Powers change detection for bucket connectors: callers compare a blob's
+ remote entity tag and ``modified_time`` against the stored ones to decide
+ whether a re-ingest is needed.
A key being **present** means the source id is already ingested under this
- ``connector_type``. A value of **None** means it was ingested but no
- ``modified_time`` was persisted (pre-change-detection docs, or an ingest path that
- didn't enrich it) — callers treat that as *unchanged* to avoid a mass re-ingest.
+ ``connector_type``. Its fields may still be None — ingested before change
+ detection recorded that signal, or by a path that didn't enrich it — and
+ callers treat a file with no comparable signal at all as *unchanged*, to
+ avoid a mass re-ingest.
Keys cover both ingest layouts (mirrors ``get_synced_file_ids_for_connector``):
the non-Langflow path stores the connector id in ``connector_file_id`` (where
@@ -299,35 +315,71 @@ async def get_synced_id_to_modified_time_map(
try:
opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
- query_body: dict[str, Any] = {
- "size": 0,
- "query": {"term": {"connector_type": connector_type}},
- "aggs": {
- "by_connector_file_id": {
- "terms": {"field": "connector_file_id", "size": 10000},
- "aggs": {"latest_modified": {"max": {"field": "modified_time"}}},
- },
- "by_document_id": {
- "terms": {"field": "document_id", "size": 10000},
- "aggs": {"latest_modified": {"max": {"field": "modified_time"}}},
+ def _sub_aggs(etag_field: str) -> dict[str, Any]:
+ return {
+ "latest_modified": {"max": {"field": "modified_time"}},
+ # All chunks of one file carry the same etag, so one bucket is enough.
+ "etag": {"terms": {"field": etag_field, "size": 1}},
+ }
+
+ def _build_body(id_field: str, etag_field: str) -> dict[str, Any]:
+ return {
+ "size": 0,
+ "query": {"term": {"connector_type": connector_type}},
+ "aggs": {
+ "by_connector_file_id": {
+ "terms": {"field": id_field, "size": 10000},
+ "aggs": _sub_aggs(etag_field),
+ },
+ "by_document_id": {
+ "terms": {"field": "document_id", "size": 10000},
+ "aggs": _sub_aggs(etag_field),
+ },
},
- },
- }
+ }
- try:
- result = await opensearch_client.search(index=get_index_name(), body=query_body)
- except Exception as agg_err:
- if not _is_unmapped_keyword_agg_error(agg_err):
- raise
- # See get_synced_file_ids_for_connector: some indices predate the
- # explicit keyword mapping for connector_file_id.
- query_body["aggs"]["by_connector_file_id"]["terms"]["field"] = (
- "connector_file_id.keyword"
- )
- result = await opensearch_client.search(index=get_index_name(), body=query_body)
+ # See get_synced_file_ids_for_connector: some indices predate the explicit
+ # keyword mapping for these fields, so they were dynamically mapped as
+ # analyzed text with a `.keyword` multi-field and must be aggregated via
+ # that sub-field instead.
+ #
+ # A field may only be switched once a failure has PROVEN the drift: a
+ # terms agg on an analyzed text field raises, but one on a field that
+ # does not exist returns no buckets and no error. Switching speculatively
+ # therefore empties that aggregation silently.
+ #
+ # The two fields drift independently — connector_file_id predates the
+ # explicit mapping by far longer than content_etag, and an index can have
+ # either one stale — so each candidate switches only what the previous
+ # failure proved and leaves the other in the form that resolves. Retrying
+ # both together would silently drop every etag on a legacy index (whose
+ # content_etag, being newer, is correctly mapped and has no `.keyword`
+ # sub-field) or, worse, drop every connector source id from the map,
+ # leaving change detection to report every file as unchanged.
+ field_candidates = (
+ ("connector_file_id", "content_etag"),
+ ("connector_file_id.keyword", "content_etag"),
+ ("connector_file_id", "content_etag.keyword"),
+ ("connector_file_id.keyword", "content_etag.keyword"),
+ )
+
+ async def _search_through_mapping_drift() -> dict[str, Any]:
+ agg_err: Exception | None = None
+ for id_field, etag_field in field_candidates:
+ try:
+ return await opensearch_client.search(
+ index=get_index_name(), body=_build_body(id_field, etag_field)
+ )
+ except Exception as err:
+ if not _is_unmapped_keyword_agg_error(err):
+ raise
+ agg_err = err
+ raise agg_err # field_candidates is never empty
+
+ result = await _search_through_mapping_drift()
aggs = result.get("aggregations", {})
- mapping: dict[str, float | None] = {}
+ mapping: dict[str, SyncedFileState] = {}
# document_id first; connector_file_id overlays it (connector_file_id wins).
# The content-hash document_ids from the non-Langflow path are harmless noise —
# they never match an enumerated connector source id.
@@ -336,11 +388,15 @@ async def get_synced_id_to_modified_time_map(
key = bucket.get("key")
if not key:
continue
- mapping[key] = bucket.get("latest_modified", {}).get("value")
+ etag_buckets = bucket.get("etag", {}).get("buckets", [])
+ mapping[key] = SyncedFileState(
+ modified_time_ms=bucket.get("latest_modified", {}).get("value"),
+ content_etag=normalize_etag(etag_buckets[0]["key"]) if etag_buckets else None,
+ )
return mapping
except Exception as e:
logger.error(
- "Failed to build id→modified_time map",
+ "Failed to build connector file state map",
connector_type=connector_type,
error=str(e),
)
@@ -371,14 +427,15 @@ def _parse_iso_to_epoch_ms(value: str | None) -> float | None:
def remote_is_newer_than_synced(
file_id: str,
remote_modified_time: str | None,
- synced_modified_map: dict[str, float | None],
+ synced_state_map: dict[str, SyncedFileState],
) -> bool:
"""True only when a stored ``modified_time`` exists and the remote is strictly newer.
Missing/unparseable timestamps (remote or stored) → False, so we never re-ingest
on ambiguity (backfill-safe).
"""
- stored_ms = synced_modified_map.get(file_id)
+ stored = synced_state_map.get(file_id)
+ stored_ms = stored.modified_time_ms if stored else None
if stored_ms is None:
return False
remote_ms = _parse_iso_to_epoch_ms(remote_modified_time)
@@ -391,22 +448,60 @@ def classify_remote_file_change(
file_id: str,
remote_modified_time: str | None,
is_ingested: bool,
- synced_modified_map: dict[str, float | None],
+ synced_state_map: dict[str, SyncedFileState],
+ remote_etag: str | None = None,
) -> str:
"""Classify a remote blob as ``"new"`` / ``"changed"`` / ``"unchanged"``.
- ``new`` — not yet ingested.
- - ``changed`` — ingested and the remote version is strictly newer than stored.
- - ``unchanged`` — ingested and same-age/older, or the change can't be proven
- (no stored token, e.g. backfill).
+ - ``changed`` — ingested, and either the entity tags differ or (with no
+ tags to compare) the remote version is strictly newer.
+ - ``unchanged`` — ingested and provably identical, same-age/older, or the
+ change can't be proven at all (no stored signal, e.g. a
+ file ingested before change detection existed).
+
+ The entity tag is checked first and is decisive in both directions. Object
+ stores change it on every overwrite, so it catches a modification whose
+ timestamp we can't compare — and equal tags mean identical bytes, so a
+ timestamp that moved without the content changing (a copy, a re-upload of
+ the same file) no longer triggers a pointless re-ingest.
"""
if not is_ingested:
return "new"
- if remote_is_newer_than_synced(file_id, remote_modified_time, synced_modified_map):
+ stored = synced_state_map.get(file_id)
+ stored_etag = stored.content_etag if stored else None
+ remote_etag = normalize_etag(remote_etag)
+ if stored_etag and remote_etag:
+ return "unchanged" if stored_etag == remote_etag else "changed"
+ if remote_is_newer_than_synced(file_id, remote_modified_time, synced_state_map):
return "changed"
return "unchanged"
+def has_comparable_change_signal(
+ file_id: str,
+ remote_modified_time: str | None,
+ synced_state_map: dict[str, SyncedFileState],
+ remote_etag: str | None = None,
+) -> bool:
+ """Whether a modification to this file could be detected at all.
+
+ False means no stored etag matching a remote one AND no stored timestamp to
+ compare — so the file is reported "unchanged" no matter what happened to it
+ at the source. Callers log the count: silently classifying an unknowable
+ file as up to date is exactly how a stale copy goes unnoticed.
+ """
+ stored = synced_state_map.get(file_id)
+ if stored is None:
+ return False
+ if stored.content_etag and normalize_etag(remote_etag):
+ return True
+ return (
+ stored.modified_time_ms is not None
+ and _parse_iso_to_epoch_ms(remote_modified_time) is not None
+ )
+
+
async def bucket_changed_file_ids(
connector,
connector_type: str,
@@ -415,20 +510,21 @@ async def bucket_changed_file_ids(
jwt_token: str | None,
existing_file_ids: list[str],
) -> list[str]:
- """Return the already-ingested bucket file ids whose remote copy is newer.
+ """Return the already-ingested bucket file ids whose remote copy has changed.
Updates-only change detection for the Sync path: list the connector's blobs
- once and keep the ids that are (a) already ingested and (b) strictly newer at
- source than the stored ``modified_time``. New blobs that aren't ingested yet
- are intentionally ignored — Sync reconciles existing files; new files are
- added via the connector's "Add from" panel. Listing exceptions propagate to
- the caller (same as the bucket_filter sync path).
+ once and keep the ids that are (a) already ingested and (b) differ at source
+ — by entity tag where both sides have one, otherwise by a strictly newer
+ ``modified_time``. New blobs that aren't ingested yet are intentionally
+ ignored — Sync reconciles existing files; new files are added via the
+ connector's "Add from" panel. Listing exceptions propagate to the caller
+ (same as the bucket_filter sync path).
"""
existing_set = set(existing_file_ids)
if not existing_set:
return []
- modified_map = await get_synced_id_to_modified_time_map(
+ state_map = await get_synced_file_state_map(
connector_type=connector_type,
user_id=user_id,
session_manager=session_manager,
@@ -436,6 +532,7 @@ async def bucket_changed_file_ids(
)
changed_ids: list[str] = []
+ undetectable: list[str] = []
page_token = None
while True:
result = await connector.list_files(page_token=page_token)
@@ -443,55 +540,71 @@ async def bucket_changed_file_ids(
fid = f.get("id")
if not fid or fid not in existing_set:
continue
+ remote_modified_time = f.get("modified_time")
+ remote_etag = f.get("etag")
if (
- classify_remote_file_change(fid, f.get("modified_time"), True, modified_map)
+ classify_remote_file_change(fid, remote_modified_time, True, state_map, remote_etag)
== "changed"
):
changed_ids.append(fid)
+ elif not has_comparable_change_signal(
+ fid, remote_modified_time, state_map, remote_etag
+ ):
+ undetectable.append(fid)
page_token = result.get("next_page_token")
if not page_token:
break
+ if undetectable:
+ # These report "unchanged" whatever happened to them at the source —
+ # nothing stored to compare against. Say so, rather than letting a stale
+ # copy look up to date. Re-ingesting them once (via the connector's
+ # "Add from" panel, overwriting) records a signal and fixes it for good.
+ logger.warning(
+ "Files have no stored change-detection signal and cannot be checked for updates",
+ connector_type=connector_type,
+ count=len(undetectable),
+ sample=undetectable[:5],
+ )
+
return changed_ids
-async def compute_orphans_for_connector_type(
+async def list_remote_files_for_connector_type(
connector_type: str,
user_id: str,
connector_service,
- session_manager,
- jwt_token: str | None,
existing_file_ids: list[str],
- id_to_filename: dict[str, str] | None = None,
-) -> list[dict[str, str]] | None:
- """Compute orphan documents (ingested but no longer present at the source)
- for this connector_type without deleting them.
+) -> dict[str, dict[str, Any]] | None:
+ """Enumerate what currently exists at the source, as {file_id: record}.
- Returns a list of {"document_id", "filename"} dicts. Returns None when strict
- gating aborts the pass (unauthenticated connection or listing exception) so
- callers can distinguish "no orphans" from "could not determine safely".
- """
- if not existing_file_ids:
- return []
+ One listing pass shared by everything that needs to diff the index against
+ the source — orphan detection and sync preview both read it, so a preview
+ doesn't enumerate the whole bucket twice.
+ Returns None (not {}) when strict gating aborts the pass: no active
+ connection, an unauthenticated one, or a listing that raised. Callers must
+ treat that as "could not determine safely" rather than "nothing there" —
+ the difference between leaving the index alone and deleting all of it.
+ """
connections = await connector_service.connection_manager.list_connections(
user_id=user_id, connector_type=connector_type
)
active = [c for c in connections if c.is_active]
if not active:
logger.info(
- "Skipping orphan compute — no active connections",
+ "Skipping remote listing — no active connections",
connector_type=connector_type,
)
return None
- remote_ids: set = set()
+ remote_files: dict[str, dict[str, Any]] = {}
for conn in active:
try:
connector = await connector_service.get_connector(conn.connection_id)
if not connector or not connector.is_authenticated:
logger.info(
- "Skipping orphan compute — connection unauthenticated",
+ "Skipping remote listing — connection unauthenticated",
connector_type=connector_type,
connection_id=conn.connection_id,
)
@@ -502,11 +615,11 @@ async def compute_orphans_for_connector_type(
# The flat default of list_files() only returns the *root* listing
# (e.g. /drive/root/children for SharePoint, files-only, no folder
# traversal), so any folder-internal file in OpenSearch would be
- # absent from remote_ids and wrongly flagged as an orphan.
+ # absent from the result and wrongly flagged as an orphan.
# list_selected_files iterates each id via _get_file_metadata_by_id
- # and silently drops missing ids, so the resulting `remote_ids` is
- # exactly "the subset of existing_file_ids that still exists at
- # source" — which is what orphan detection actually needs.
+ # and silently drops missing ids, so the result is exactly "the
+ # subset of existing_file_ids that still exists at source" — which
+ # is what orphan detection actually needs.
# cfg is None on bucket connectors (BaseConnector declares it as a
# class default), so guard on cfg-is-not-None rather than hasattr:
# otherwise bucket connectors route through list_selected_files ->
@@ -518,7 +631,7 @@ async def compute_orphans_for_connector_type(
for f in page.get("files", []):
fid = f.get("id")
if fid:
- remote_ids.add(fid)
+ remote_files[fid] = f
else:
page_token = None
while True:
@@ -526,20 +639,56 @@ async def compute_orphans_for_connector_type(
for f in page.get("files", []):
fid = f.get("id")
if fid:
- remote_ids.add(fid)
+ remote_files[fid] = f
page_token = page.get("nextPageToken") or page.get("next_page_token")
if not page_token:
break
except Exception as e:
logger.warning(
- "Skipping orphan compute — listing failed",
+ "Skipping remote listing — listing failed",
connector_type=connector_type,
connection_id=conn.connection_id,
error=str(e),
)
return None
- orphan_ids = [fid for fid in existing_file_ids if fid not in remote_ids]
+ return remote_files
+
+
+async def compute_orphans_for_connector_type(
+ connector_type: str,
+ user_id: str,
+ connector_service,
+ session_manager,
+ jwt_token: str | None,
+ existing_file_ids: list[str],
+ id_to_filename: dict[str, str] | None = None,
+ remote_files: dict[str, dict[str, Any]] | None = None,
+) -> list[dict[str, str]] | None:
+ """Compute orphan documents (ingested but no longer present at the source)
+ for this connector_type without deleting them.
+
+ Returns a list of {"document_id", "filename"} dicts. Returns None when strict
+ gating aborts the pass (unauthenticated connection or listing exception) so
+ callers can distinguish "no orphans" from "could not determine safely".
+
+ ``remote_files`` lets a caller that has already enumerated the source pass
+ the listing in instead of paying for a second pass.
+ """
+ if not existing_file_ids:
+ return []
+
+ if remote_files is None:
+ remote_files = await list_remote_files_for_connector_type(
+ connector_type=connector_type,
+ user_id=user_id,
+ connector_service=connector_service,
+ existing_file_ids=existing_file_ids,
+ )
+ if remote_files is None:
+ return None
+
+ orphan_ids = [fid for fid in existing_file_ids if fid not in remote_files]
if not orphan_ids:
return []
@@ -554,7 +703,8 @@ async def delete_orphan_documents(
jwt_token: str | None,
*,
connector_type: str | None = None,
- shared: bool = False,
+ shared: bool | None = False,
+ allow_anonymous_delete: bool = False,
) -> int:
"""Delete OpenSearch chunks for the given orphan IDs. Returns the number of
chunks deleted (0 on failure).
@@ -563,11 +713,39 @@ async def delete_orphan_documents(
ingest path, ``document_id`` for the Langflow path) via
``connectors.chunk_cleanup``, so callers no longer need to track which
field the ids came from.
+
+ ``shared`` is the caller's explicit intent and is None on the re-sync paths,
+ which have no record of how each file was ingested.
+
+ ``allow_anonymous_delete`` is the resolved ``knowledge:delete:anonymous``
+ permission and is what puts ownerless (share-all) chunks in scope at all.
+ Without it this stays owner-scoped, even on a re-sync that would otherwise
+ widen to "owned by me OR ownerless".
+
+ That gate is not paranoia about the caller's own files. The orphan id set
+ comes from ``get_synced_file_ids_for_connector``, which filters only on
+ ``connector_type`` and reads through the user-scoped client — and DLS makes
+ every ownerless chunk visible to every authenticated user
+ (securityconfig/roles.yml). So the set can include shared files ingested by
+ a *different* user's connection, which this user's own remote listing will
+ not contain and orphan detection will therefore classify as deleted at
+ source. Widening the delete scope without the permission would let any
+ authenticated user erase another connection's shared documents by running
+ an ordinary sync.
"""
if not orphan_ids:
return 0
from connectors.chunk_cleanup import delete_connector_file_chunks
+ include_shared = allow_anonymous_delete and not shared
+ if shared and not allow_anonymous_delete:
+ logger.warning(
+ "Skipping shared orphan cleanup — caller lacks knowledge:delete:anonymous",
+ connector_type=connector_type,
+ orphan_count=len(orphan_ids),
+ )
+ return 0
+
try:
opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
return await delete_connector_file_chunks(
@@ -575,7 +753,8 @@ async def delete_orphan_documents(
opensearch_client,
connector_type=connector_type,
owner_user_id=None if shared else user_id,
- shared=shared,
+ shared=bool(shared),
+ include_shared=include_shared,
refresh=True,
)
except Exception as e:
@@ -595,12 +774,17 @@ async def reconcile_orphans_for_connector_type(
jwt_token: str | None,
existing_file_ids: list[str],
*,
- shared: bool = False,
+ shared: bool | None = False,
+ allow_anonymous_delete: bool = False,
) -> list[str]:
"""Compute and delete orphans for a connector type. Thin wrapper around
compute_orphans_for_connector_type + delete_orphan_documents preserved for
callers that perform sync immediately after reconcile.
+ ``allow_anonymous_delete`` (the resolved ``knowledge:delete:anonymous``
+ permission) is forwarded to the deletion; see delete_orphan_documents for
+ why ownerless chunks must not be in scope without it.
+
Returns the list of orphan file IDs that were deleted (or []).
"""
orphans = await compute_orphans_for_connector_type(
@@ -616,6 +800,7 @@ async def reconcile_orphans_for_connector_type(
orphan_ids = [o["document_id"] for o in orphans]
deleted = await delete_orphan_documents(
+ allow_anonymous_delete=allow_anonymous_delete,
orphan_ids=orphan_ids,
user_id=user_id,
session_manager=session_manager,
@@ -646,7 +831,8 @@ async def _sync_existing_connector_files(
id_field: str,
*,
ingest_settings: dict[str, Any] | None = None,
- shared: bool = False,
+ shared: bool | None = None,
+ allow_anonymous_delete: bool = False,
reconcile: bool = True,
max_files: int | None = None,
) -> dict[str, Any]:
@@ -659,6 +845,13 @@ async def _sync_existing_connector_files(
full re-sync runs depending on the connector's ``CHANGE_DETECTION``
capability; connectors with no stored ids fall back to filename filtering.
+ ``shared`` defaults to None — "inherit". This flow re-syncs files that are
+ already indexed, and nothing records whether each was ingested with the COS
+ share-all toggle, so each file's current indexed state decides (see
+ ``ConnectorFileProcessor._resolve_shared``). Passing False here instead
+ would make sync treat every shared COS file as private, skip it as an
+ unreplaceable duplicate, and leave the stale content in the index.
+
Returns an outcome dict the caller maps onto its own response shape:
* ``{"outcome": "synced", "task_id": ...}``
* ``{"outcome": "deleted_only"}`` — orphan cleanup removed every file;
@@ -686,6 +879,7 @@ async def _sync_existing_connector_files(
jwt_token=jwt_token,
existing_file_ids=existing_file_ids,
shared=shared,
+ allow_anonymous_delete=allow_anonymous_delete,
)
if orphan_ids:
orphan_id_set = set(orphan_ids)
@@ -771,8 +965,11 @@ class ConnectorSyncBody(BaseModel):
preview: bool = False
# When True (COS only), index chunks without an owner field so OpenSearch DLS
# makes them visible to all users in the instance. Temporary CIO mechanism;
- # not a full ACL feature. Defaults to False (private).
- shared: bool = False
+ # not a full ACL feature.
+ # Omitted/None means "no explicit intent": new files are indexed private,
+ # and files already in the index keep the sharing state they have. The
+ # connector UI already sends the flag only when the toggle is on screen.
+ shared: bool | None = None
class ConnectorCheckDuplicatesBody(BaseModel):
@@ -1290,9 +1487,15 @@ async def connector_sync(
status_code=400,
)
- if body.shared and not await has_effective_permission(
+ # Resolved for every sync, not just an explicitly shared one: orphan
+ # cleanup on a plain re-sync can also reach ownerless chunks (see
+ # delete_orphan_documents), and those are visible to every authenticated
+ # user under DLS regardless of who ingested them.
+ allow_anonymous_delete = await has_effective_permission(
request, user, rbac, "knowledge:delete:anonymous"
- ):
+ )
+
+ if body.shared and not allow_anonymous_delete:
return JSONResponse(
{"error": "Shared sync requires the knowledge:delete:anonymous permission"},
status_code=403,
@@ -1443,7 +1646,7 @@ async def connector_sync(
jwt_token=jwt_token,
)
existing_set = set(existing_ids)
- modified_map = await get_synced_id_to_modified_time_map(
+ state_map = await get_synced_file_state_map(
connector_type=connector_type,
user_id=user.user_id,
session_manager=session_manager,
@@ -1460,7 +1663,8 @@ async def connector_sync(
fid,
f.get("modified_time"),
fid in existing_set,
- modified_map,
+ state_map,
+ f.get("etag"),
)
if status == "new":
new_ids.append(fid)
@@ -1560,6 +1764,7 @@ async def connector_sync(
id_field=id_field,
ingest_settings=body.settings,
shared=body.shared,
+ allow_anonymous_delete=allow_anonymous_delete,
# Strict gating: skip orphan reconcile when sync is capped — we'd
# see a partial remote listing and delete legitimate files.
reconcile=body.max_files is None,
@@ -1577,7 +1782,10 @@ async def connector_sync(
return JSONResponse(
{
"status": "no_files",
- "message": f"All {connector_type} files are already up to date.",
+ # Scoped to indexed files. Unlike the bucket_filter branch
+ # above, this path never enumerates new remote files, so
+ # it cannot claim the whole connector is up to date.
+ "message": (f"All indexed {connector_type} files are up to date."),
},
status_code=200,
)
@@ -1848,12 +2056,16 @@ async def connector_webhook(
# Trigger incremental sync for affected files. The webhook fires
# because the file changed, so replace the indexed copy instead of
# tripping the duplicate-filename guard meant for manual uploads.
+ # shared=None like the other re-sync paths: no user is in the
+ # loop to express an intent, so each file keeps the sharing state
+ # it already has.
task_id = await connector_service.sync_specific_files(
connection.connection_id,
connection.user_id,
in_scope,
jwt_token=jwt_token,
replace_duplicates=_connector_sync_should_replace(connector_type),
+ shared=None,
)
result = {
@@ -2011,6 +2223,7 @@ async def sync_all_connectors(
session_manager=Depends(get_session_manager),
user: User = Depends(require_permission("connectors:use")),
session: AsyncSession = Depends(get_db_session),
+ rbac=Depends(get_rbac_service),
):
"""
Sync files from all active cloud connector connections.
@@ -2020,6 +2233,11 @@ async def sync_all_connectors(
Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_START
)
jwt_token = user.jwt_token
+ # Sync-all reconciles orphans across every connector type, so it can
+ # reach ownerless chunks too — same gate as the per-connector endpoint.
+ allow_anonymous_delete = await has_effective_permission(
+ request, user, rbac, "knowledge:delete:anonymous"
+ )
all_task_ids = []
synced_connectors = []
@@ -2098,6 +2316,7 @@ async def sync_all_connectors(
existing_file_ids=existing_file_ids,
existing_filenames=existing_filenames,
id_field=id_field,
+ allow_anonymous_delete=allow_anonymous_delete,
)
if sync_result["outcome"] == "deleted_only":
deleted_only_connectors.append(connector_type)
@@ -2181,17 +2400,36 @@ def _cloud_connector_types() -> list[str]:
return [cls.CONNECTOR_TYPE for cls in get_connector_classes()]
-async def _preview_orphans_for_connector_type(
+class ConnectorSyncPreview(NamedTuple):
+ """What a sync would do to one connector type, without doing any of it.
+
+ ``orphans`` is None when strict gating aborted the pass (so the caller can
+ say "couldn't determine" rather than "nothing to delete"). ``updates`` is
+ None when this connector can't answer the question ahead of time — see
+ ``_preview_for_connector_type``.
+ """
+
+ orphans: list[dict[str, str]] | None
+ updates: list[dict[str, str]] | None
+ synced_count: int
+
+
+async def _preview_for_connector_type(
connector_type: str,
user_id: str,
connector_service,
session_manager,
jwt_token: str | None,
-) -> tuple[list[dict[str, str]] | None, int]:
- """Helper: compute orphans (no deletion) + return total synced count.
+) -> ConnectorSyncPreview:
+ """Compute what a sync would delete and re-ingest, without touching anything.
+
+ Deletions and updates come out of a single remote listing, so previewing
+ costs the same as the orphan pass used to.
- Returns (orphans, synced_count). `orphans` is None when strict gating aborts
- (so the caller can surface a "couldn't determine" state); [] when no orphans.
+ Updates are only knowable for connectors that declare timestamp change
+ detection: they compare a listing against stored signals. ``replace_always``
+ connectors decide per file after downloading it, so their update count is
+ reported as unavailable rather than guessed at.
"""
existing_file_ids, existing_filenames, _ = await get_synced_file_ids_for_connector(
connector_type=connector_type,
@@ -2204,7 +2442,7 @@ async def _preview_orphans_for_connector_type(
if not existing_file_ids:
# No document_ids to diff against (e.g. Langflow-only ingest). Filename-only
# fallback can't detect orphans safely — surface empty list.
- return [], synced_count
+ return ConnectorSyncPreview(orphans=[], updates=None, synced_count=synced_count)
id_to_filename = await get_synced_id_to_filename_map(
connector_type=connector_type,
@@ -2213,6 +2451,13 @@ async def _preview_orphans_for_connector_type(
jwt_token=jwt_token,
)
+ remote_files = await list_remote_files_for_connector_type(
+ connector_type=connector_type,
+ user_id=user_id,
+ connector_service=connector_service,
+ existing_file_ids=existing_file_ids,
+ )
+
orphans = await compute_orphans_for_connector_type(
connector_type=connector_type,
user_id=user_id,
@@ -2221,10 +2466,34 @@ async def _preview_orphans_for_connector_type(
jwt_token=jwt_token,
existing_file_ids=existing_file_ids,
id_to_filename=id_to_filename,
+ remote_files=remote_files,
)
if orphans is not None:
synced_count = max(0, synced_count - len(orphans))
- return orphans, synced_count
+
+ updates = None
+ if remote_files is not None and _connector_uses_timestamp_change_detection(connector_type):
+ state_map = await get_synced_file_state_map(
+ connector_type=connector_type,
+ user_id=user_id,
+ session_manager=session_manager,
+ jwt_token=jwt_token,
+ )
+ updates = [
+ {"document_id": fid, "filename": id_to_filename.get(fid, "")}
+ for fid in existing_file_ids
+ if fid in remote_files
+ and classify_remote_file_change(
+ fid,
+ remote_files[fid].get("modified_time"),
+ True,
+ state_map,
+ remote_files[fid].get("etag"),
+ )
+ == "changed"
+ ]
+
+ return ConnectorSyncPreview(orphans=orphans, updates=updates, synced_count=synced_count)
async def connector_sync_preview(
@@ -2236,14 +2505,15 @@ async def connector_sync_preview(
session: AsyncSession = Depends(get_db_session),
):
"""Preview the impact of syncing a connector type without performing any
- deletion or ingest. Returns the list of orphan files (present in OpenSearch
- but no longer at the source) by filename, plus the total synced count.
+ deletion or ingest. Returns the orphan files (present in OpenSearch but no
+ longer at the source) and the files whose source copy has changed, both by
+ filename, plus the total synced count.
"""
if denied := await _connector_access_denied(request, session, connector_type):
return denied
try:
- orphans, synced_count = await _preview_orphans_for_connector_type(
+ preview = await _preview_for_connector_type(
connector_type=connector_type,
user_id=user.user_id,
connector_service=connector_service,
@@ -2253,9 +2523,13 @@ async def connector_sync_preview(
return JSONResponse(
{
"connector_type": connector_type,
- "synced_count": synced_count,
- "orphans": orphans or [],
- "orphans_available": orphans is not None,
+ "synced_count": preview.synced_count,
+ "orphans": preview.orphans or [],
+ "orphans_available": preview.orphans is not None,
+ "updates": preview.updates or [],
+ # False means this connector can't tell ahead of time (it decides
+ # per file during ingest), not that nothing will be updated.
+ "updates_available": preview.updates is not None,
},
status_code=200,
)
@@ -2272,20 +2546,22 @@ async def connectors_sync_all_preview(
session: AsyncSession = Depends(get_db_session),
):
"""Preview the impact of sync-all-connectors across every cloud connector
- type. Returns orphan filenames grouped by connector_type plus a per-type
- synced count.
+ type. Returns orphan and changed filenames grouped by connector_type plus a
+ per-type synced count.
"""
try:
orphans_by_type: dict[str, list[dict[str, str]]] = {}
synced_count_by_type: dict[str, int] = {}
orphans_available_by_type: dict[str, bool] = {}
+ updates_by_type: dict[str, list[dict[str, str]]] = {}
+ updates_available_by_type: dict[str, bool] = {}
connector_types = await _allowed_connector_types_for_request(
request, session, _cloud_connector_types()
)
for connector_type in connector_types:
try:
- orphans, synced_count = await _preview_orphans_for_connector_type(
+ preview = await _preview_for_connector_type(
connector_type=connector_type,
user_id=user.user_id,
connector_service=connector_service,
@@ -2298,21 +2574,25 @@ async def connectors_sync_all_preview(
connector_type=connector_type,
error=str(e),
)
- orphans, synced_count = None, 0
+ preview = ConnectorSyncPreview(orphans=None, updates=None, synced_count=0)
# Only include connector types that have something synced.
- if synced_count == 0 and not orphans:
+ if preview.synced_count == 0 and not preview.orphans:
continue
- synced_count_by_type[connector_type] = synced_count
- orphans_by_type[connector_type] = orphans or []
- orphans_available_by_type[connector_type] = orphans is not None
+ synced_count_by_type[connector_type] = preview.synced_count
+ orphans_by_type[connector_type] = preview.orphans or []
+ orphans_available_by_type[connector_type] = preview.orphans is not None
+ updates_by_type[connector_type] = preview.updates or []
+ updates_available_by_type[connector_type] = preview.updates is not None
return JSONResponse(
{
"orphans_by_type": orphans_by_type,
"synced_count_by_type": synced_count_by_type,
"orphans_available_by_type": orphans_available_by_type,
+ "updates_by_type": updates_by_type,
+ "updates_available_by_type": updates_available_by_type,
},
status_code=200,
)
@@ -2507,8 +2787,9 @@ async def browse_connection_files(
)
ingested_set = set(ingested_ids) | set(ingested_filenames)
- # Stored modified_time per ingested source id, for "update available" detection.
- modified_map = await get_synced_id_to_modified_time_map(
+ # Stored change-detection state per ingested source id, for "update
+ # available" detection.
+ state_map = await get_synced_file_state_map(
connector_type=connector_type,
user_id=user.user_id,
session_manager=session_manager,
@@ -2520,11 +2801,15 @@ async def browse_connection_files(
for f in remote_files:
file_id = f.get("id", "")
is_ingested = file_id in ingested_set or f.get("name", "") in ingested_set
- # "Update available": ingested, but the source version is newer than what
- # we indexed. The frontend keeps unchanged files disabled but lets the user
- # re-ingest stale ones (with replace_duplicates).
- is_stale = is_ingested and remote_is_newer_than_synced(
- file_id, f.get("modified_time"), modified_map
+ # "Update available": ingested, but the source copy differs from what we
+ # indexed. Uses the same classification as sync so the browser and the
+ # sync agree. The frontend keeps unchanged files disabled but lets the
+ # user re-ingest stale ones (with replace_duplicates).
+ is_stale = (
+ classify_remote_file_change(
+ file_id, f.get("modified_time"), is_ingested, state_map, f.get("etag")
+ )
+ == "changed"
)
enriched_files.append(
{
diff --git a/src/config/settings.py b/src/config/settings.py
index 2c7478f71..e66dc1f62 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -800,6 +800,8 @@ INDEX_BODY = {
"connector_type": {"type": "keyword"},
"ingest_run_id": {"type": "keyword"},
"connector_file_id": {"type": "keyword"},
+ # Object-store entity tag of the source file, for sync change detection.
+ "content_etag": {"type": "keyword"},
"owner": {"type": "keyword"},
"allowed_users": {"type": "keyword"},
"allowed_groups": {"type": "keyword"},
diff --git a/src/connectors/aws_s3/connector.py b/src/connectors/aws_s3/connector.py
index 8267a3879..29980f93b 100644
--- a/src/connectors/aws_s3/connector.py
+++ b/src/connectors/aws_s3/connector.py
@@ -7,7 +7,13 @@ from posixpath import basename
from typing import Any
from config.settings import IBM_AUTH_ENABLED
-from connectors.base import BaseConnector, ConnectorDocument, DocumentACL
+from connectors.base import (
+ CONTENT_ETAG_METADATA_KEY,
+ BaseConnector,
+ ConnectorDocument,
+ DocumentACL,
+ normalize_etag,
+)
from utils.logging_config import get_logger
from .auth import create_s3_client, create_s3_resource
@@ -198,6 +204,9 @@ class S3Connector(BaseConnector):
"modified_time": obj.last_modified.isoformat()
if obj.last_modified
else None,
+ # Change detection compares this against the tag
+ # stored at ingest; it moves on every overwrite.
+ "etag": normalize_etag(getattr(obj, "e_tag", None)),
}
)
if max_files and len(files) >= max_files:
@@ -249,6 +258,9 @@ class S3Connector(BaseConnector):
"s3_bucket": bucket_name,
"s3_key": key,
"size": size,
+ # Persisted at ingest so a later sync can tell an overwritten
+ # object from an untouched one without trusting clocks.
+ CONTENT_ETAG_METADATA_KEY: normalize_etag(response.get("ETag")),
},
)
diff --git a/src/connectors/base.py b/src/connectors/base.py
index 9b7a420f9..ff4add17f 100644
--- a/src/connectors/base.py
+++ b/src/connectors/base.py
@@ -5,6 +5,33 @@ from dataclasses import dataclass
from datetime import datetime
from typing import Any
+# Key under which a connector reports its object's entity tag in
+# ``ConnectorDocument.metadata``. Bucket connectors should set it from the
+# store's ETag; sync change detection persists it and compares it against a
+# fresh listing, which is how a modified object is noticed even when the two
+# timestamps can't be compared. See api.connectors.classify_remote_file_change.
+CONTENT_ETAG_METADATA_KEY = "content_etag"
+
+
+def normalize_etag(value: Any) -> str | None:
+ """Normalize an object-store entity tag for comparison, or None if empty.
+
+ Strips the weak-validator prefix and the surrounding quotes that S3-style
+ and HTTP entity tags carry, so a tag read back out of the index compares
+ equal to the one a fresh listing reports even if the two APIs quote it
+ differently.
+
+ The multipart suffix (``-12``) is deliberately kept: it is part of the
+ identity, so identical bytes uploaded with different part sizes compare as
+ different and get re-ingested, rather than being skipped on a guess.
+ """
+ if not isinstance(value, str):
+ return None
+ tag = value.strip()
+ if tag[:2].upper() == "W/":
+ tag = tag[2:].strip()
+ return tag.strip('"').strip() or None
+
@dataclass
class DocumentACL:
@@ -78,9 +105,14 @@ class BaseConnector(ABC):
# "replace_always" — re-process every indexed file on sync, replacing the
# indexed copy (the content-hash short-circuit still
# skips identical bytes on the non-Langflow path).
- # "timestamp" — list the source once and re-ingest only files whose
- # remote modified_time is strictly newer than the
- # stored one (see bucket_changed_file_ids).
+ # "timestamp" — list the source once and re-ingest only the files
+ # that actually differ: by entity tag where the
+ # listing and the index both have one, otherwise by a
+ # strictly newer remote modified_time. Connectors on
+ # this mode should report "etag" in list_files() and
+ # CONTENT_ETAG_METADATA_KEY in get_file_content()
+ # metadata; without them detection falls back to
+ # timestamps alone (see bucket_changed_file_ids).
CHANGE_DETECTION: str = "replace_always"
# Connector-specific keys in the config dict that must be encrypted at rest.
SECRET_CONFIG_KEYS: tuple = ()
diff --git a/src/connectors/chunk_cleanup.py b/src/connectors/chunk_cleanup.py
index 6ef2d3aa0..9be0ab295 100644
--- a/src/connectors/chunk_cleanup.py
+++ b/src/connectors/chunk_cleanup.py
@@ -9,7 +9,10 @@ cleanup — but exactly one deletion *semantic*, owned here:
holds the content hash), the Langflow path stores it in ``document_id``.
Matching a single field misses chunks from the other layout.
* Optionally owner-scoped: ``owner_user_id`` restricts to that owner's chunks;
- ``shared=True`` targets ownerless (instance-shared) chunks instead.
+ ``shared=True`` targets ownerless (instance-shared) chunks instead, and
+ ``include_shared=True`` covers both (the right boundary when the caller has
+ already pinned the file down by its connector id — see
+ ``build_owner_or_shared_filter``).
* DLS-safe: visible chunk ``_id``s are enumerated with the caller's
(user-scoped) client, then deleted individually with the trusted backend
write client — ``delete_by_query`` is silently no-opped under DLS.
@@ -25,6 +28,7 @@ def build_connector_file_chunks_query(
connector_type: str | None = None,
owner_user_id: str | None = None,
shared: bool = False,
+ include_shared: bool = False,
keep_filenames: Iterable[str] | None = None,
) -> dict[str, Any]:
"""Query matching every chunk of the given connector file ids.
@@ -32,9 +36,17 @@ def build_connector_file_chunks_query(
``connector_type`` narrows the match to a single connector type so IDs
that happen to collide across connectors are not affected.
+ ``include_shared`` widens ``owner_user_id`` from "owned by this user" to
+ "owned by this user OR ownerless". Sync paths need it: a file ingested with
+ the COS share-all toggle has no ``owner`` at all, so a plain owner term
+ matches none of its chunks and the cleanup silently no-ops. Ignored when
+ ``shared=True`` (already ownerless-only) or when no owner is given.
+
``keep_filenames`` excludes chunks indexed under those names — used by
rename cleanup to drop only the stale old-name chunks of a file id.
"""
+ from utils.opensearch_queries import build_owner_or_shared_filter
+
ids = [fid for fid in file_ids if fid]
filters: list[dict[str, Any]] = [
{
@@ -56,7 +68,11 @@ def build_connector_file_chunks_query(
if shared:
filters.append({"bool": {"must_not": {"exists": {"field": "owner"}}}})
elif owner_user_id:
- filters.append({"term": {"owner": owner_user_id}})
+ filters.append(
+ build_owner_or_shared_filter(owner_user_id)
+ if include_shared
+ else {"term": {"owner": owner_user_id}}
+ )
query: dict[str, Any] = {"bool": {"filter": filters}}
keep = [name for name in (keep_filenames or []) if name]
if keep:
@@ -71,6 +87,7 @@ async def delete_connector_file_chunks(
connector_type: str | None = None,
owner_user_id: str | None = None,
shared: bool = False,
+ include_shared: bool = False,
keep_filenames: Iterable[str] | None = None,
refresh: bool = False,
) -> int:
@@ -99,6 +116,7 @@ async def delete_connector_file_chunks(
connector_type=connector_type,
owner_user_id=owner_user_id,
shared=shared,
+ include_shared=include_shared,
keep_filenames=keep_filenames,
),
)
diff --git a/src/connectors/service.py b/src/connectors/service.py
index 535e8ebac..11d0d2224 100644
--- a/src/connectors/service.py
+++ b/src/connectors/service.py
@@ -4,7 +4,12 @@ from config.settings import get_index_name
from utils.file_utils import clean_connector_filename
from utils.logging_config import get_logger
-from .base import BaseConnector, ConnectorDocument
+from .base import (
+ CONTENT_ETAG_METADATA_KEY,
+ BaseConnector,
+ ConnectorDocument,
+ normalize_etag,
+)
from .connection_manager import ConnectionManager
logger = get_logger(__name__)
@@ -159,6 +164,9 @@ class ConnectorService:
if (params.modified_time != null) {
ctx._source.modified_time = params.modified_time;
}
+ if (params.content_etag != null) {
+ ctx._source.content_etag = params.content_etag;
+ }
if (params.metadata != null) {
ctx._source.metadata = params.metadata;
}
@@ -173,6 +181,13 @@ class ConnectorService:
"modified_time": document.modified_time.isoformat()
if document.modified_time
else None,
+ # Promoted out of metadata to a top-level keyword so
+ # sync can read it back with a terms aggregation (the
+ # same shape connector_file_id uses) instead of
+ # fetching every chunk's _source.
+ "content_etag": normalize_etag(
+ (document.metadata or {}).get(CONTENT_ETAG_METADATA_KEY)
+ ),
"metadata": document.metadata,
},
},
@@ -196,7 +211,7 @@ class ConnectorService:
filename_filter: set = None,
ingest_settings: dict[str, Any] | None = None,
replace_duplicates: bool = False,
- shared: bool = False,
+ shared: bool | None = False,
) -> str:
"""
Sync files from a connector connection using existing task tracking system.
@@ -335,7 +350,7 @@ class ConnectorService:
ingest_settings: dict[str, Any] | None = None,
replace_duplicates: bool = False,
preview_mode: bool = False,
- shared: bool = False,
+ shared: bool | None = False,
) -> str:
"""
Sync specific files by their IDs (used for webhook-triggered syncs or manual selection).
diff --git a/src/models/processors.py b/src/models/processors.py
index 4b781feb0..e1b4d41d3 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -321,12 +321,20 @@ class TaskProcessor:
shared: bool = False,
) -> int:
"""Delete all chunks of a document with the given filename from
- OpenSearch. Returns the number of chunks deleted."""
+ OpenSearch. Returns the number of chunks deleted.
+
+ ``shared`` describes how the replacement is about to be *written*, not
+ what is already indexed, so it must not narrow what we delete: with an
+ owner in hand the scope is always "owned by this user OR ownerless".
+ Choosing an owner-only scope for a shared document matched none of its
+ chunks, and the caller read that zero as "nothing to replace" and
+ skipped the file — leaving the stale copy in the index even though the
+ duplicate check (which is owner-agnostic) had just found it and the user
+ had confirmed the overwrite."""
from config.settings import clients, get_index_name
from utils.opensearch_delete import collect_visible_document_ids, delete_document_ids
from utils.opensearch_queries import (
build_anonymous_filename_query,
- build_owned_filename_query,
build_replace_filename_query,
)
@@ -348,7 +356,7 @@ class TaskProcessor:
return 0
else:
- build_query = build_replace_filename_query if shared else build_owned_filename_query
+ build_query = build_replace_filename_query
candidate_filenames = get_filename_aliases(filename)
if not candidate_filenames:
@@ -398,6 +406,15 @@ class TaskProcessor:
``connector_type`` scopes the match to one connector type — the same
value the chunks were indexed under — so an id that collides with a
different connector's id can't take its chunks down with it.
+
+ Callers pass the sharing mode resolved from what is actually indexed
+ (``_resolve_shared``), so this deletes under exactly that layout —
+ ownerless chunks for a share-all file, owner-scoped ones otherwise.
+ It must NOT widen to "owned OR ownerless" on top of that: the resolved
+ mode already matches the file, and widening would let a sync of an owned
+ file reach ownerless chunks that another user's connection ingested,
+ with none of the ``knowledge:delete:anonymous`` gating that deliberate
+ shared deletion goes through.
"""
from connectors.chunk_cleanup import delete_connector_file_chunks
@@ -867,7 +884,7 @@ class ConnectorFileProcessor(TaskProcessor):
replace_duplicates: bool = False,
connector_type: str | None = None,
preview_mode: bool = False,
- shared: bool = False,
+ shared: bool | None = False,
):
super().__init__(
document_service=document_service,
@@ -887,9 +904,75 @@ class ConnectorFileProcessor(TaskProcessor):
self.preview_mode = preview_mode
self.shared = shared
- async def _reconcile_shared_owner(self, filename: str) -> None:
+ async def _indexed_shared_state(
+ self,
+ file_id: str,
+ opensearch_client,
+ connector_type: str | None,
+ ) -> bool | None:
+ """Whether this connector file is ALREADY indexed as shared (ownerless).
+
+ Returns True when the indexed chunks carry no ``owner`` (the share-all
+ layout), False when they carry one, and None when the file isn't indexed
+ yet or the lookup failed — callers fall back to their own default.
+
+ Read with the caller's (user-scoped) client, so DLS keeps documents we
+ cannot see out of the answer.
+ """
+ if not file_id:
+ return None
+ from connectors.chunk_cleanup import build_connector_file_chunks_query
+
+ try:
+ response = await opensearch_client.search(
+ index=get_index_name(),
+ body={
+ "size": 1,
+ "_source": ["owner"],
+ "query": build_connector_file_chunks_query(
+ [file_id], connector_type=connector_type
+ ),
+ },
+ )
+ except Exception as e:
+ logger.warning(
+ "Could not read indexed sharing state for connector file",
+ file_id=file_id,
+ error=str(e),
+ )
+ return None
+ hits = response.get("hits", {}).get("hits", [])
+ if not hits:
+ return None
+ return not hits[0].get("_source", {}).get("owner")
+
+ async def _resolve_shared(
+ self,
+ file_id: str,
+ opensearch_client,
+ connector_type: str | None,
+ ) -> bool:
+ """Resolve the sharing mode to index this file under.
+
+ ``self.shared`` is the caller's explicit intent — the connector upload
+ UI's "Make documents available to all users" toggle. It is None on the
+ re-sync paths, which run with no user in the loop and no record of what
+ each file was ingested as; those inherit whatever is already in the
+ index. Without that inheritance a sync would treat a shared file as
+ private, and every owner-scoped step downstream (the replace-delete, the
+ chunk cleanup, the owner fields on re-index) would either match nothing
+ or silently un-share the document.
+
+ Files not yet indexed fall back to private, matching the previous
+ default for a caller that expressed no intent.
+ """
+ if self.shared is not None:
+ return self.shared
+ return bool(await self._indexed_shared_state(file_id, opensearch_client, connector_type))
+
+ async def _reconcile_shared_owner(self, filename: str, shared: bool) -> None:
"""Update owner fields on already-indexed chunks for `filename` to match
- the connector's current `shared` setting.
+ the `shared` setting resolved for this file.
Called on the duplicate/unchanged skip paths below, where a file's
content and name haven't changed since a prior sync but the connector's
@@ -900,12 +983,16 @@ class ConnectorFileProcessor(TaskProcessor):
this user or already ownerless (matching the same boundary
delete_document_by_filename uses), so it can't touch another user's
private document that happens to share this filename.
+
+ `shared` comes from _resolve_shared, never straight from self.shared: on
+ a re-sync it reflects the file's current indexed state, so this is a
+ no-op rather than an unrequested ownership change.
"""
write_client = clients.opensearch
if write_client is None:
return
owner, owner_name, owner_email = resolve_shared_owner_fields(
- self.user_id, self.owner_name, self.owner_email, self.shared
+ self.user_id, self.owner_name, self.owner_email, shared
)
for candidate in get_filename_aliases(filename):
try:
@@ -924,7 +1011,7 @@ class ConnectorFileProcessor(TaskProcessor):
ctx._source.owner_email = params.owner_email;
""",
"params": {
- "shared": self.shared,
+ "shared": shared,
"owner": owner,
"owner_name": owner_name,
"owner_email": owner_email,
@@ -1020,7 +1107,9 @@ class ConnectorFileProcessor(TaskProcessor):
file_id,
opensearch_client,
self.user_id,
- shared=self.shared,
+ shared=await self._resolve_shared(
+ file_id, opensearch_client, connector_type
+ ),
connector_type=connector_type,
)
@@ -1069,15 +1158,21 @@ class ConnectorFileProcessor(TaskProcessor):
self.user_id, self.jwt_token
)
+ # Resolve once, up front: every owner-scoped step below (the
+ # replace-delete, the rename cleanup, the owner fields written at
+ # index time) has to agree on how this file is shared, or the sync
+ # skips it and leaves the stale copy in place.
+ shared = await self._resolve_shared(document.id, opensearch_client, connector_type)
+
duplicate_action = await self.resolve_duplicate_filename(
file_task.filename,
opensearch_client,
replace=self.replace_duplicates,
owner_user_id=self.user_id,
- shared=self.shared,
+ shared=shared,
)
if duplicate_action == "skip":
- await self._reconcile_shared_owner(file_task.filename)
+ await self._reconcile_shared_owner(file_task.filename, shared)
self.mark_duplicate_skipped(upload_task, file_task)
return
@@ -1096,7 +1191,7 @@ class ConnectorFileProcessor(TaskProcessor):
opensearch_client,
self.user_id,
keep_filenames=get_filename_aliases(file_task.filename),
- shared=self.shared,
+ shared=shared,
connector_type=connector_type,
)
> 0
@@ -1115,7 +1210,7 @@ class ConnectorFileProcessor(TaskProcessor):
file_hash = hash_id(tmp_path)
if not renamed and await self.check_document_exists(file_hash, opensearch_client):
- await self._reconcile_shared_owner(file_task.filename)
+ await self._reconcile_shared_owner(file_task.filename, shared)
file_task.status = TaskStatus.COMPLETED
file_task.result = {"status": "unchanged", "id": file_hash}
file_task.updated_at = time.time()
@@ -1214,7 +1309,7 @@ class ConnectorFileProcessor(TaskProcessor):
effective_owner, effective_owner_name, effective_owner_email = (
resolve_shared_owner_fields(
- self.user_id, self.owner_name, self.owner_email, self.shared
+ self.user_id, self.owner_name, self.owner_email, shared
)
)
file_task.document_id = document.id
@@ -1309,7 +1404,7 @@ class ConnectorFileProcessor(TaskProcessor):
connector_type=connector_type,
acl=document.acl,
connector_file_id=document.id,
- shared=self.shared,
+ shared=shared,
**standard_kwargs,
)
diff --git a/src/utils/embeddings.py b/src/utils/embeddings.py
index 74f3ffd14..7bff5e6ec 100644
--- a/src/utils/embeddings.py
+++ b/src/utils/embeddings.py
@@ -49,6 +49,8 @@ async def create_index_body(
"source_url": {"type": "keyword"},
"connector_type": {"type": "keyword"},
"ingest_run_id": {"type": "keyword"},
+ # Object-store entity tag of the source file, for sync change detection.
+ "content_etag": {"type": "keyword"},
"owner": {"type": "keyword"},
"owner_email": {"type": "keyword"},
"allowed_users": {"type": "keyword"},
diff --git a/src/utils/opensearch_queries.py b/src/utils/opensearch_queries.py
index 32e852e1a..d16dfb7b5 100644
--- a/src/utils/opensearch_queries.py
+++ b/src/utils/opensearch_queries.py
@@ -74,30 +74,45 @@ def build_anonymous_filename_query(filename: str) -> dict:
}
+def build_owner_or_shared_filter(owner: str) -> dict:
+ """Build the owner-scope clause covering a user's own chunks AND shared ones.
+
+ Matches ``owner == owner`` OR the ``owner`` field being absent. The
+ ownerless branch is what keeps replace/cleanup working on documents ingested
+ with the COS "make available to all users" toggle, which indexes chunks
+ without an owner at all; naming the current user in the other branch keeps
+ another user's *private* document — merely visible to us via allowed_users
+ DLS — out of scope.
+
+ This is the deletion boundary for anything we have already matched by a
+ stable identity (filename alias, connector file id): both layouts are ours
+ to replace, neither is someone else's to remove.
+ """
+ return {
+ "bool": {
+ "should": [
+ {"term": {"owner": owner}},
+ {"bool": {"must_not": {"exists": {"field": "owner"}}}},
+ ],
+ "minimum_should_match": 1,
+ }
+ }
+
+
def build_replace_filename_query(filename: str, owner: str) -> dict:
"""Build a delete-scope query for replace_duplicates that covers both private
and shared (ownerless) chunks with this filename.
Matches chunks where filename matches AND (owner == current user OR owner
- field is absent). Combining both cases is necessary because the same
- filename may have been previously ingested as shared (no owner field) and
- is now being replaced by the same user. The owner-field branch protects
- against accidentally deleting documents owned by *other* users that are
- merely visible to the current user via allowed_users DLS.
+ field is absent) — see build_owner_or_shared_filter. Combining both cases
+ is necessary because the same filename may have been previously ingested as
+ shared (no owner field) and is now being replaced by the same user.
"""
return {
"bool": {
"filter": [
build_filename_query(filename),
- {
- "bool": {
- "should": [
- {"term": {"owner": owner}},
- {"bool": {"must_not": {"exists": {"field": "owner"}}}},
- ],
- "minimum_should_match": 1,
- }
- },
+ build_owner_or_shared_filter(owner),
]
}
}
diff --git a/tests/unit/api/test_connector_change_detection.py b/tests/unit/api/test_connector_change_detection.py
index 50ea274c9..52e0d4ba1 100644
--- a/tests/unit/api/test_connector_change_detection.py
+++ b/tests/unit/api/test_connector_change_detection.py
@@ -3,7 +3,7 @@
Covers:
- the pure timestamp/classification helpers (`_parse_iso_to_epoch_ms`,
`remote_is_newer_than_synced`, `classify_remote_file_change`),
-- the `get_synced_id_to_modified_time_map` aggregation helper,
+- the `get_synced_file_state_map` aggregation helper,
- the whole-container (`bucket_filter`) reconciliation in `connector_sync`: only
new + changed blobs are ingested (new as a plain batch, changed with
replace_duplicates=True), unchanged blobs are skipped.
@@ -17,6 +17,25 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
+
+def _rbac_allowing(*perms: str):
+ """RBAC stub granting exactly `perms`.
+
+ Unit tests run with OPENRAG_RBAC_ENFORCE=true (tests/unit/conftest.py), so
+ permission checks are live and every sync now resolves
+ knowledge:delete:anonymous to decide whether ownerless chunks may be
+ deleted — see delete_orphan_documents.
+ """
+ rbac = MagicMock()
+ granted = set(perms)
+
+ async def has_permission(user_id, perm, role_override=None):
+ return perm in granted
+
+ rbac.has_permission = AsyncMock(side_effect=has_permission)
+ return rbac
+
+
ROOT = Path(__file__).resolve().parent.parent.parent.parent
SRC = ROOT / "src"
if str(SRC) not in sys.path:
@@ -49,17 +68,23 @@ def test_parse_iso_to_epoch_ms_returns_none_for_bad_input():
assert _parse_iso_to_epoch_ms("not-a-date") is None
+def _stored(modified_ms=None, etag=None):
+ from api.connectors import SyncedFileState
+
+ return SyncedFileState(modified_time_ms=modified_ms, content_etag=etag)
+
+
def test_remote_is_newer_than_synced_true_when_strictly_newer():
from api.connectors import remote_is_newer_than_synced
- stored = {"c::a": 1704067200000.0} # 2024-01-01
+ stored = {"c::a": _stored(1704067200000.0)} # 2024-01-01
assert remote_is_newer_than_synced("c::a", "2024-06-01T00:00:00Z", stored) is True
def test_remote_is_newer_than_synced_false_when_same_or_older():
from api.connectors import remote_is_newer_than_synced
- stored = {"c::a": 1704067200000.0}
+ stored = {"c::a": _stored(1704067200000.0)}
assert remote_is_newer_than_synced("c::a", "2024-01-01T00:00:00Z", stored) is False
assert remote_is_newer_than_synced("c::a", "2023-01-01T00:00:00Z", stored) is False
@@ -67,7 +92,7 @@ def test_remote_is_newer_than_synced_false_when_same_or_older():
def test_remote_is_newer_than_synced_tolerance_absorbs_subsecond_jitter():
from api.connectors import remote_is_newer_than_synced
- stored = {"c::a": 1704067200000.0}
+ stored = {"c::a": _stored(1704067200000.0)}
# 500ms newer is within tolerance → not "changed".
assert remote_is_newer_than_synced("c::a", "2024-01-01T00:00:00.500Z", stored) is False
# 2s newer exceeds tolerance → changed.
@@ -79,13 +104,13 @@ def test_remote_is_newer_than_synced_false_when_no_stored_token():
# Missing id, or ingested-but-no-token (None) → backfill-safe False.
assert remote_is_newer_than_synced("c::missing", "2024-06-01T00:00:00Z", {}) is False
- assert remote_is_newer_than_synced("c::a", "2024-06-01T00:00:00Z", {"c::a": None}) is False
+ assert remote_is_newer_than_synced("c::a", "2024-06-01T00:00:00Z", {"c::a": _stored()}) is False
def test_classify_remote_file_change():
from api.connectors import classify_remote_file_change
- stored = {"c::a": 1704067200000.0}
+ stored = {"c::a": _stored(1704067200000.0)}
# Not ingested → new (regardless of timestamps).
assert classify_remote_file_change("c::new", "2024-06-01T00:00:00Z", False, stored) == "new"
# Ingested + newer → changed.
@@ -97,12 +122,112 @@ def test_classify_remote_file_change():
# ---------------------------------------------------------------------------
-# get_synced_id_to_modified_time_map
+# Entity-tag change detection
+# ---------------------------------------------------------------------------
+
+
+def test_classify_uses_etag_when_both_sides_have_one():
+ """A differing etag is "changed" even when the timestamps say otherwise."""
+ from api.connectors import classify_remote_file_change
+
+ stored = {"c::a": _stored(1704067200000.0, "abc123")}
+ # Same (stale) timestamp, different bytes → changed.
+ assert (
+ classify_remote_file_change("c::a", "2024-01-01T00:00:00Z", True, stored, "def456")
+ == "changed"
+ )
+ # No timestamp stored at all, but the tags disagree → still changed. This is
+ # the case the timestamp-only check could never see.
+ assert (
+ classify_remote_file_change("c::a", None, True, {"c::a": _stored(None, "abc123")}, "def456")
+ == "changed"
+ )
+
+
+def test_classify_equal_etags_beat_a_newer_timestamp():
+ """Identical bytes are unchanged even if the object's timestamp moved.
+
+ A copy or a re-upload of the same file bumps LastModified without changing
+ the content; re-ingesting it is pure waste.
+ """
+ from api.connectors import classify_remote_file_change
+
+ stored = {"c::a": _stored(1704067200000.0, "abc123")}
+ assert (
+ classify_remote_file_change("c::a", "2024-06-01T00:00:00Z", True, stored, "abc123")
+ == "unchanged"
+ )
+
+
+def test_classify_etag_comparison_ignores_quoting_and_weak_prefix():
+ """S3 quotes its tags and HTTP may mark them weak; neither is a content change."""
+ from api.connectors import classify_remote_file_change
+
+ stored = {"c::a": _stored(1704067200000.0, "abc123")}
+ assert (
+ classify_remote_file_change("c::a", "2024-06-01T00:00:00Z", True, stored, '"abc123"')
+ == "unchanged"
+ )
+ assert (
+ classify_remote_file_change("c::a", "2024-06-01T00:00:00Z", True, stored, 'W/"abc123"')
+ == "unchanged"
+ )
+
+
+def test_classify_falls_back_to_timestamp_when_an_etag_is_missing():
+ """One-sided tags prove nothing, so the timestamp still decides."""
+ from api.connectors import classify_remote_file_change
+
+ stored_without_etag = {"c::a": _stored(1704067200000.0)}
+ assert (
+ classify_remote_file_change("c::a", "2024-06-01T00:00:00Z", True, stored_without_etag, "x")
+ == "changed"
+ )
+ stored_with_etag = {"c::a": _stored(1704067200000.0, "abc123")}
+ assert (
+ classify_remote_file_change("c::a", "2024-06-01T00:00:00Z", True, stored_with_etag, None)
+ == "changed"
+ )
+
+
+def test_has_comparable_change_signal():
+ """Whether a modification could be noticed at all — what the warning counts."""
+ from api.connectors import has_comparable_change_signal
+
+ # Nothing ingested for this id.
+ assert has_comparable_change_signal("c::missing", "2024-06-01T00:00:00Z", {}) is False
+ # Ingested with neither signal → undetectable.
+ assert (
+ has_comparable_change_signal("c::a", "2024-06-01T00:00:00Z", {"c::a": _stored()}) is False
+ )
+ # Stored etag but the listing reports none → falls back to timestamps, and
+ # there is no stored timestamp either.
+ assert (
+ has_comparable_change_signal("c::a", "2024-06-01T00:00:00Z", {"c::a": _stored(None, "e1")})
+ is False
+ )
+ # Either signal comparable on both sides → detectable.
+ assert (
+ has_comparable_change_signal(
+ "c::a", "2024-06-01T00:00:00Z", {"c::a": _stored(None, "e1")}, "e2"
+ )
+ is True
+ )
+ assert (
+ has_comparable_change_signal(
+ "c::a", "2024-06-01T00:00:00Z", {"c::a": _stored(1704067200000.0)}
+ )
+ is True
+ )
+
+
+# ---------------------------------------------------------------------------
+# get_synced_file_state_map
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
-async def test_modified_time_map_prefers_connector_file_id_over_document_id(monkeypatch):
+async def test_state_map_prefers_connector_file_id_over_document_id(monkeypatch):
from api import connectors as connectors_api
monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
@@ -113,9 +238,18 @@ async def test_modified_time_map_prefers_connector_file_id_over_document_id(monk
"aggregations": {
"by_connector_file_id": {
"buckets": [
- {"key": "c::a", "latest_modified": {"value": 1704067200000.0}},
- # connector_file_id present but no modified_time → None
- {"key": "c::b", "latest_modified": {"value": None}},
+ {
+ "key": "c::a",
+ "latest_modified": {"value": 1704067200000.0},
+ # S3 hands back quoted tags; the helper normalizes.
+ "etag": {"buckets": [{"key": '"abc123"'}]},
+ },
+ # connector_file_id present but neither signal stored → both None
+ {
+ "key": "c::b",
+ "latest_modified": {"value": None},
+ "etag": {"buckets": []},
+ },
]
},
"by_document_id": {
@@ -133,7 +267,7 @@ async def test_modified_time_map_prefers_connector_file_id_over_document_id(monk
sm = MagicMock()
sm.get_user_opensearch_client = MagicMock(return_value=opensearch_client)
- result = await connectors_api.get_synced_id_to_modified_time_map(
+ result = await connectors_api.get_synced_file_state_map(
connector_type="azure_blob",
user_id="alice",
session_manager=sm,
@@ -141,63 +275,178 @@ async def test_modified_time_map_prefers_connector_file_id_over_document_id(monk
)
# connector_file_id wins for c::a (1704067200000, not the 999 from document_id).
- assert result["c::a"] == 1704067200000.0
- assert result["c::b"] is None
- assert result["c::lf"] == 1704153600000.0
+ assert result["c::a"].modified_time_ms == 1704067200000.0
+ assert result["c::a"].content_etag == "abc123"
+ assert result["c::b"].modified_time_ms is None
+ assert result["c::b"].content_etag is None
+ assert result["c::lf"].modified_time_ms == 1704153600000.0
+
+
+# A terms agg on an analyzed `text` field raises; one on a field the index does
+# not have returns no buckets and no error. So a field may only be switched to
+# its `.keyword` sub-field once an error has proven the drift, and the two
+# fields have to be retried independently — switching both together silently
+# empties whichever aggregation was fine.
+
+
+def _text_field_error(field: str) -> Exception:
+ """The error OpenSearch raises for a terms agg on an analyzed text field."""
+ return Exception(
+ "RequestError(400, 'search_phase_execution_exception', 'Text fields "
+ "are not optimised for operations that require per-document field "
+ "data like aggregations and sorting, so these operations are "
+ "disabled by default. Please use a keyword field instead. "
+ f"Alternatively, set fielddata=true on [{field}]...')"
+ )
-async def test_modified_time_map_falls_back_to_keyword_subfield_on_text_field_error(
- monkeypatch,
-):
- """Indices that predate connector_file_id's addition to the explicit
- mapping (config/settings.py) have it dynamically mapped as analyzed text,
- and OpenSearch rejects a terms agg on that field outright. The helper
- must retry the aggregation against connector_file_id.keyword instead of
- treating this as a fatal error (which would make every blob look "new")."""
- from api import connectors as connectors_api
+def _mapping_drift_client(text_mapped: set[str]):
+ """Fake OpenSearch whose aggregations behave like a real index's mappings.
- monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+ Fields in ``text_mapped`` raise when aggregated directly (they are analyzed
+ text). A `.keyword` sub-field only resolves for those; asking for it on a
+ properly mapped field returns no buckets, exactly as OpenSearch does for a
+ field that does not exist — which is what makes a speculative switch silent.
- opensearch_client = AsyncMock()
- called_fields = []
+ Records every (id_field, etag_field) pair attempted.
+ """
+ attempts: list[tuple[str, str]] = []
+ client = AsyncMock()
async def fake_search(*, index, body):
- called_fields.append(body["aggs"]["by_connector_file_id"]["terms"]["field"])
- if len(called_fields) == 1:
- raise Exception(
- "RequestError(400, 'search_phase_execution_exception', 'Text fields "
- "are not optimised for operations that require per-document field "
- "data like aggregations and sorting, so these operations are "
- "disabled by default. Please use a keyword field instead. "
- "Alternatively, set fielddata=true on [connector_file_id]...')"
- )
+ id_field = body["aggs"]["by_connector_file_id"]["terms"]["field"]
+ etag_field = body["aggs"]["by_connector_file_id"]["aggs"]["etag"]["terms"]["field"]
+ attempts.append((id_field, etag_field))
+ for field in (id_field, etag_field):
+ base = field.removesuffix(".keyword")
+ if base in text_mapped and field == base:
+ raise _text_field_error(base)
+ etag_resolves = etag_field.removesuffix(".keyword") in text_mapped or (
+ not etag_field.endswith(".keyword")
+ )
+ id_resolves = id_field.removesuffix(".keyword") in text_mapped or (
+ not id_field.endswith(".keyword")
+ )
+ etag_buckets = [{"key": '"abc123"'}] if etag_resolves else []
return {
"aggregations": {
"by_connector_file_id": {
- "buckets": [{"key": "c::a", "latest_modified": {"value": 1704067200000.0}}]
+ "buckets": [
+ {
+ "key": "c::a",
+ "latest_modified": {"value": 1704067200000.0},
+ "etag": {"buckets": etag_buckets},
+ }
+ ]
+ if id_resolves
+ else []
},
"by_document_id": {"buckets": []},
}
}
- opensearch_client.search = AsyncMock(side_effect=fake_search)
+ client.search = AsyncMock(side_effect=fake_search)
+ return client, attempts
+
+
+async def _state_map_with(monkeypatch, text_mapped: set[str]):
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+ client, attempts = _mapping_drift_client(text_mapped)
sm = MagicMock()
- sm.get_user_opensearch_client = MagicMock(return_value=opensearch_client)
+ sm.get_user_opensearch_client = MagicMock(return_value=client)
- result = await connectors_api.get_synced_id_to_modified_time_map(
- connector_type="azure_blob",
+ result = await connectors_api.get_synced_file_state_map(
+ connector_type="ibm_cos",
user_id="alice",
session_manager=sm,
jwt_token=None,
)
+ return result, attempts
- assert result == {"c::a": 1704067200000.0}
- assert called_fields == ["connector_file_id", "connector_file_id.keyword"]
+
+async def test_state_map_uses_plain_fields_when_nothing_has_drifted(monkeypatch):
+ result, attempts = await _state_map_with(monkeypatch, text_mapped=set())
+
+ assert attempts == [("connector_file_id", "content_etag")]
+ assert result["c::a"].content_etag == "abc123"
+
+
+async def test_state_map_keeps_etags_on_an_index_predating_content_etag(monkeypatch):
+ """The legacy case: connector_file_id drifted, content_etag did not.
+
+ content_etag was added to the explicit mapping long after connector_file_id,
+ so an index old enough to have the text-mapped id has a correctly mapped (or
+ absent) content_etag with no `.keyword` sub-field. Retrying both fields
+ together asks for content_etag.keyword, which resolves to nothing and drops
+ every stored etag — silently regressing change detection to timestamps on
+ exactly the indices that need it most.
+ """
+ result, attempts = await _state_map_with(monkeypatch, text_mapped={"connector_file_id"})
+
+ assert attempts == [
+ ("connector_file_id", "content_etag"),
+ ("connector_file_id.keyword", "content_etag"),
+ ]
+ assert result["c::a"].modified_time_ms == 1704067200000.0
+ assert result["c::a"].content_etag == "abc123"
+
+
+async def test_state_map_keeps_source_ids_when_only_content_etag_drifted(monkeypatch):
+ """The inverse: switching connector_file_id too would empty the id buckets.
+
+ With no source ids in the map every file classifies as unchanged, which is
+ the original "overwrite in COS is not reflected" defect all over again.
+ """
+ result, attempts = await _state_map_with(monkeypatch, text_mapped={"content_etag"})
+
+ assert attempts == [
+ ("connector_file_id", "content_etag"),
+ ("connector_file_id.keyword", "content_etag"),
+ ("connector_file_id", "content_etag.keyword"),
+ ]
+ assert "c::a" in result
+ assert result["c::a"].content_etag == "abc123"
@pytest.mark.asyncio
-async def test_modified_time_map_returns_empty_on_error(monkeypatch):
+async def test_state_map_switches_both_fields_only_when_both_drifted(monkeypatch):
+ result, attempts = await _state_map_with(
+ monkeypatch, text_mapped={"connector_file_id", "content_etag"}
+ )
+
+ assert attempts[-1] == ("connector_file_id.keyword", "content_etag.keyword")
+ assert result["c::a"].content_etag == "abc123"
+
+
+async def test_state_map_gives_up_after_exhausting_field_forms(monkeypatch):
+ """An unmapped-keyword error no candidate can fix must surface, not hang."""
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+ client = AsyncMock()
+ client.search = AsyncMock(side_effect=_text_field_error("connector_file_id"))
+ sm = MagicMock()
+ sm.get_user_opensearch_client = MagicMock(return_value=client)
+
+ # The outer handler converts a failed lookup into an empty map (backfill-safe).
+ result = await connectors_api.get_synced_file_state_map(
+ connector_type="ibm_cos",
+ user_id="alice",
+ session_manager=sm,
+ jwt_token=None,
+ )
+ assert result == {}
+ assert client.search.await_count == 4
+
+
+async def test_state_map_returns_empty_on_error(monkeypatch):
from api import connectors as connectors_api
monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
@@ -206,7 +455,7 @@ async def test_modified_time_map_returns_empty_on_error(monkeypatch):
sm = MagicMock()
sm.get_user_opensearch_client = MagicMock(return_value=opensearch_client)
- result = await connectors_api.get_synced_id_to_modified_time_map(
+ result = await connectors_api.get_synced_file_state_map(
connector_type="azure_blob",
user_id="alice",
session_manager=sm,
@@ -224,7 +473,7 @@ async def test_modified_time_map_returns_empty_on_error(monkeypatch):
async def test_synced_file_ids_falls_back_to_keyword_subfield_on_text_field_error(
monkeypatch,
):
- """Same text-field mapping-drift issue as get_synced_id_to_modified_time_map,
+ """Same text-field mapping-drift issue as get_synced_file_state_map,
but here a fatal (uncaught) error would make orphan detection and bucket
reconciliation treat every connector file as never-synced."""
from api import connectors as connectors_api
@@ -302,11 +551,11 @@ async def test_bucket_filter_ingests_only_new_and_changed(monkeypatch):
)
monkeypatch.setattr(
connectors_api,
- "get_synced_id_to_modified_time_map",
+ "get_synced_file_state_map",
AsyncMock(
return_value={
- "c::ingested_unchanged": 1704067200000.0, # 2024-01-01
- "c::ingested_changed": 1704067200000.0, # 2024-01-01
+ "c::ingested_unchanged": _stored(1704067200000.0), # 2024-01-01
+ "c::ingested_changed": _stored(1704067200000.0), # 2024-01-01
}
),
)
@@ -324,8 +573,9 @@ async def test_bucket_filter_ingests_only_new_and_changed(monkeypatch):
request=MagicMock(),
connector_service=service,
session_manager=MagicMock(),
- user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),
session=MagicMock(),
+ rbac=_rbac_allowing("knowledge:delete:anonymous"),
)
assert response.status_code == 201
@@ -353,8 +603,10 @@ async def test_bucket_filter_all_unchanged_returns_no_files(monkeypatch):
)
monkeypatch.setattr(
connectors_api,
- "get_synced_id_to_modified_time_map",
- AsyncMock(return_value={"c::a": 1704067200000.0, "c::b": 1704067200000.0}),
+ "get_synced_file_state_map",
+ AsyncMock(
+ return_value={"c::a": _stored(1704067200000.0), "c::b": _stored(1704067200000.0)}
+ ),
)
remote_files = [
@@ -369,8 +621,9 @@ async def test_bucket_filter_all_unchanged_returns_no_files(monkeypatch):
request=MagicMock(),
connector_service=service,
session_manager=MagicMock(),
- user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),
session=MagicMock(),
+ rbac=_rbac_allowing("knowledge:delete:anonymous"),
)
assert response.status_code == 200
@@ -393,7 +646,7 @@ async def test_bucket_filter_only_new_files_single_batch(monkeypatch):
)
monkeypatch.setattr(
connectors_api,
- "get_synced_id_to_modified_time_map",
+ "get_synced_file_state_map",
AsyncMock(return_value={}),
)
@@ -409,8 +662,9 @@ async def test_bucket_filter_only_new_files_single_batch(monkeypatch):
request=MagicMock(),
connector_service=service,
session_manager=MagicMock(),
- user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),
session=MagicMock(),
+ rbac=_rbac_allowing("knowledge:delete:anonymous"),
)
assert response.status_code == 201
@@ -433,9 +687,12 @@ async def test_bucket_changed_file_ids_filters_to_changed_ingested(monkeypatch):
monkeypatch.setattr(
connectors_api,
- "get_synced_id_to_modified_time_map",
+ "get_synced_file_state_map",
AsyncMock(
- return_value={"c::a": 1704067200000.0, "c::b": 1704067200000.0} # 2024-01-01
+ return_value={
+ "c::a": _stored(1704067200000.0),
+ "c::b": _stored(1704067200000.0),
+ } # 2024-01-01
),
)
@@ -462,6 +719,90 @@ async def test_bucket_changed_file_ids_filters_to_changed_ingested(monkeypatch):
assert changed == ["c::b"]
+async def test_bucket_changed_file_ids_detects_overwrite_by_etag(monkeypatch):
+ """An overwritten object is caught by its entity tag alone.
+
+ This is the case the timestamp check cannot see: nothing was stored to
+ compare against (the file predates change detection, or an ingest path
+ didn't persist the timestamp), so the file would report "unchanged" forever.
+ """
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_state_map",
+ AsyncMock(
+ return_value={
+ "c::a": _stored(None, "etag-a"),
+ "c::b": _stored(None, "etag-b"),
+ }
+ ),
+ )
+
+ connector = MagicMock()
+ connector.list_files = AsyncMock(
+ return_value={
+ "files": [
+ {"id": "c::a", "modified_time": None, "etag": "etag-a"}, # untouched
+ {"id": "c::b", "modified_time": None, "etag": "etag-b-v2"}, # overwritten
+ ],
+ "next_page_token": None,
+ }
+ )
+
+ changed = await connectors_api.bucket_changed_file_ids(
+ connector, "ibm_cos", "alice", MagicMock(), "token", ["c::a", "c::b"]
+ )
+ assert changed == ["c::b"]
+
+
+async def test_bucket_changed_file_ids_warns_about_undetectable_files(monkeypatch):
+ """Files with nothing to compare are reported, not silently called up to date."""
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_state_map",
+ AsyncMock(return_value={"c::a": _stored(), "c::b": _stored(1704067200000.0)}),
+ )
+
+ connector = MagicMock()
+ connector.list_files = AsyncMock(
+ return_value={
+ "files": [
+ # Ingested before any signal was recorded, and the listing has
+ # no etag either — a modification here is unknowable.
+ {"id": "c::a", "modified_time": "2024-06-01T00:00:00Z"},
+ {"id": "c::b", "modified_time": "2024-01-01T00:00:00Z"},
+ ],
+ "next_page_token": None,
+ }
+ )
+
+ # The project logger writes straight to stderr rather than propagating to the
+ # stdlib root logger, so assert on the call instead of on caplog.
+ warnings = []
+ monkeypatch.setattr(
+ connectors_api.logger,
+ "warning",
+ lambda msg, **kw: warnings.append((msg, kw)),
+ )
+
+ changed = await connectors_api.bucket_changed_file_ids(
+ connector, "ibm_cos", "alice", MagicMock(), "token", ["c::a", "c::b"]
+ )
+
+ assert changed == []
+ assert len(warnings) == 1
+ msg, fields = warnings[0]
+ assert "no stored change-detection signal" in msg
+ assert fields["count"] == 1
+ # c::b has a comparable timestamp — it is genuinely unchanged, not unknowable.
+ assert fields["sample"] == ["c::a"]
+
+
@pytest.mark.asyncio
async def test_bucket_changed_file_ids_empty_when_no_existing():
from api import connectors as connectors_api
@@ -499,8 +840,10 @@ async def test_sync_button_bucket_reingests_only_changed(monkeypatch):
)
monkeypatch.setattr(
connectors_api,
- "get_synced_id_to_modified_time_map",
- AsyncMock(return_value={"c::a": 1704067200000.0, "c::b": 1704067200000.0}),
+ "get_synced_file_state_map",
+ AsyncMock(
+ return_value={"c::a": _stored(1704067200000.0), "c::b": _stored(1704067200000.0)}
+ ),
)
remote_files = [
@@ -515,8 +858,9 @@ async def test_sync_button_bucket_reingests_only_changed(monkeypatch):
request=MagicMock(),
connector_service=service,
session_manager=MagicMock(),
- user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),
session=MagicMock(),
+ rbac=_rbac_allowing("knowledge:delete:anonymous"),
)
assert response.status_code == 201
@@ -544,8 +888,10 @@ async def test_sync_button_bucket_all_unchanged_returns_no_files(monkeypatch):
)
monkeypatch.setattr(
connectors_api,
- "get_synced_id_to_modified_time_map",
- AsyncMock(return_value={"c::a": 1704067200000.0, "c::b": 1704067200000.0}),
+ "get_synced_file_state_map",
+ AsyncMock(
+ return_value={"c::a": _stored(1704067200000.0), "c::b": _stored(1704067200000.0)}
+ ),
)
remote_files = [
@@ -560,8 +906,9 @@ async def test_sync_button_bucket_all_unchanged_returns_no_files(monkeypatch):
request=MagicMock(),
connector_service=service,
session_manager=MagicMock(),
- user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),
session=MagicMock(),
+ rbac=_rbac_allowing("knowledge:delete:anonymous"),
)
assert response.status_code == 200
@@ -598,8 +945,10 @@ async def test_sync_all_bucket_reingests_only_changed(monkeypatch):
)
monkeypatch.setattr(
connectors_api,
- "get_synced_id_to_modified_time_map",
- AsyncMock(return_value={"c::a": 1704067200000.0, "c::b": 1704067200000.0}),
+ "get_synced_file_state_map",
+ AsyncMock(
+ return_value={"c::a": _stored(1704067200000.0), "c::b": _stored(1704067200000.0)}
+ ),
)
remote_files = [
@@ -612,8 +961,9 @@ async def test_sync_all_bucket_reingests_only_changed(monkeypatch):
request=MagicMock(),
connector_service=service,
session_manager=MagicMock(),
- user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),
session=MagicMock(),
+ rbac=_rbac_allowing("knowledge:delete:anonymous"),
)
assert response.status_code == 201
diff --git a/tests/unit/api/test_connector_sync_response.py b/tests/unit/api/test_connector_sync_response.py
index c68fdc602..4324b36e0 100644
--- a/tests/unit/api/test_connector_sync_response.py
+++ b/tests/unit/api/test_connector_sync_response.py
@@ -13,6 +13,19 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
+
+def _permissive_rbac():
+ """RBAC stub that grants everything.
+
+ Unit tests pin OPENRAG_RBAC_ENFORCE=true (tests/unit/conftest.py), and every
+ sync now resolves knowledge:delete:anonymous up front to decide whether
+ orphan cleanup may touch ownerless chunks (see delete_orphan_documents).
+ """
+ rbac = MagicMock()
+ rbac.has_permission = AsyncMock(return_value=True)
+ return rbac
+
+
ROOT = Path(__file__).resolve().parent.parent.parent.parent
SRC = ROOT / "src"
if str(SRC) not in sys.path:
@@ -79,8 +92,9 @@ async def test_connector_sync_reports_one_connection_with_multiple_active(monkey
request=MagicMock(),
connector_service=service,
session_manager=MagicMock(),
- user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),
session=MagicMock(),
+ rbac=_permissive_rbac(),
)
assert response.status_code == 201
@@ -138,8 +152,9 @@ async def test_connector_sync_passes_preview_mode_to_sync_specific_files(monkeyp
request=MagicMock(),
connector_service=service,
session_manager=MagicMock(),
- user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),
session=MagicMock(),
+ rbac=_permissive_rbac(),
)
assert response.status_code == 201
@@ -165,8 +180,9 @@ async def test_connector_sync_ignores_preview_when_disabled(monkeypatch):
request=MagicMock(),
connector_service=service,
session_manager=MagicMock(),
- user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),
session=MagicMock(),
+ rbac=_permissive_rbac(),
)
assert response.status_code == 201
diff --git a/tests/unit/api/test_reconcile_orphans_for_connector_type.py b/tests/unit/api/test_reconcile_orphans_for_connector_type.py
index 3559f95bd..a4e215675 100644
--- a/tests/unit/api/test_reconcile_orphans_for_connector_type.py
+++ b/tests/unit/api/test_reconcile_orphans_for_connector_type.py
@@ -18,6 +18,25 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
+
+def _rbac_allowing(*perms: str):
+ """RBAC stub granting exactly `perms`.
+
+ Unit tests run with OPENRAG_RBAC_ENFORCE=true (tests/unit/conftest.py), so
+ permission checks are live and every sync now resolves
+ knowledge:delete:anonymous to decide whether ownerless chunks may be
+ deleted — see delete_orphan_documents.
+ """
+ rbac = MagicMock()
+ granted = set(perms)
+
+ async def has_permission(user_id, perm, role_override=None):
+ return perm in granted
+
+ rbac.has_permission = AsyncMock(side_effect=has_permission)
+ return rbac
+
+
ROOT = Path(__file__).resolve().parent.parent.parent.parent
SRC = ROOT / "src"
if str(SRC) not in sys.path:
@@ -238,8 +257,14 @@ async def test_happy_path_deletes_orphans(monkeypatch):
@pytest.mark.asyncio
-async def test_happy_path_private_scopes_to_owner(monkeypatch):
- """Private orphan cleanup must filter by connector_type and owner."""
+async def test_happy_path_private_scopes_to_owner_or_ownerless(monkeypatch):
+ """With knowledge:delete:anonymous, cleanup covers "owned by me OR ownerless".
+
+ The ownerless branch matters for COS: a file ingested with the share-all
+ toggle has no owner field, and sync has no record of that, so an owner-only
+ filter would leave its chunks behind after the object is deleted at source.
+ Another user's private document stays out of scope via the owner term.
+ """
from api.connectors import reconcile_orphans_for_connector_type
conn = _make_connection("c1")
@@ -256,12 +281,21 @@ async def test_happy_path_private_scopes_to_owner(monkeypatch):
session_manager=sm,
jwt_token=None,
existing_file_ids=["a", "b"],
+ allow_anonymous_delete=True,
)
search_body = opensearch_client.search.await_args.kwargs["body"]
filters = search_body["query"]["bool"]["filter"]
assert {"term": {"connector_type": "google_drive"}} in filters
- assert {"term": {"owner": "alice"}} in filters
+ assert {
+ "bool": {
+ "should": [
+ {"term": {"owner": "alice"}},
+ {"bool": {"must_not": {"exists": {"field": "owner"}}}},
+ ],
+ "minimum_should_match": 1,
+ }
+ } in filters
@pytest.mark.asyncio
@@ -284,6 +318,7 @@ async def test_happy_path_shared_scopes_to_ownerless(monkeypatch):
jwt_token=None,
existing_file_ids=["a", "b"],
shared=True,
+ allow_anonymous_delete=True,
)
search_body = opensearch_client.search.await_args.kwargs["body"]
@@ -445,7 +480,7 @@ async def test_orphan_delete_matches_both_id_layouts(monkeypatch):
@pytest.mark.asyncio
async def test_preview_subtracts_orphans_from_resync_count(monkeypatch):
- from api.connectors import _preview_orphans_for_connector_type
+ from api.connectors import _preview_for_connector_type
monkeypatch.setattr(
"api.connectors.get_synced_file_ids_for_connector",
@@ -455,6 +490,10 @@ async def test_preview_subtracts_orphans_from_resync_count(monkeypatch):
"api.connectors.get_synced_id_to_filename_map",
AsyncMock(return_value={"b": "b.pdf", "e": "e.pdf"}),
)
+ monkeypatch.setattr(
+ "api.connectors.list_remote_files_for_connector_type",
+ AsyncMock(return_value={"a": {"id": "a"}, "c": {"id": "c"}, "d": {"id": "d"}}),
+ )
monkeypatch.setattr(
"api.connectors.compute_orphans_for_connector_type",
AsyncMock(
@@ -465,7 +504,7 @@ async def test_preview_subtracts_orphans_from_resync_count(monkeypatch):
),
)
- orphans, synced_count = await _preview_orphans_for_connector_type(
+ preview = await _preview_for_connector_type(
connector_type="google_drive",
user_id="alice",
connector_service=MagicMock(),
@@ -473,11 +512,88 @@ async def test_preview_subtracts_orphans_from_resync_count(monkeypatch):
jwt_token="token",
)
- assert synced_count == 4
- assert orphans == [
+ assert preview.synced_count == 4
+ assert preview.orphans == [
{"document_id": "b", "filename": "b.pdf"},
{"document_id": "e", "filename": "e.pdf"},
]
+ # google_drive re-processes every file on sync and decides per file after
+ # downloading it, so the update count cannot be known in advance.
+ assert preview.updates is None
+
+
+async def test_preview_reports_updates_for_timestamp_connectors(monkeypatch):
+ """Bucket connectors can say which files a sync will re-ingest, and the
+ preview reuses the orphan pass's listing rather than enumerating twice."""
+ from api.connectors import SyncedFileState, _preview_for_connector_type
+
+ monkeypatch.setattr(
+ "api.connectors.get_synced_file_ids_for_connector",
+ AsyncMock(return_value=(["c::a", "c::b", "c::gone"], [], "connector_file_id")),
+ )
+ monkeypatch.setattr(
+ "api.connectors.get_synced_id_to_filename_map",
+ AsyncMock(return_value={"c::a": "a.pdf", "c::b": "b.pdf", "c::gone": "gone.pdf"}),
+ )
+ listing = AsyncMock(
+ return_value={
+ "c::a": {"id": "c::a", "etag": "etag-a"}, # untouched
+ "c::b": {"id": "c::b", "etag": "etag-b-v2"}, # overwritten
+ }
+ )
+ monkeypatch.setattr("api.connectors.list_remote_files_for_connector_type", listing)
+ monkeypatch.setattr(
+ "api.connectors.get_synced_file_state_map",
+ AsyncMock(
+ return_value={
+ "c::a": SyncedFileState(content_etag="etag-a"),
+ "c::b": SyncedFileState(content_etag="etag-b"),
+ }
+ ),
+ )
+
+ preview = await _preview_for_connector_type(
+ connector_type="ibm_cos",
+ user_id="alice",
+ connector_service=MagicMock(),
+ session_manager=MagicMock(),
+ jwt_token="token",
+ )
+
+ assert preview.updates == [{"document_id": "c::b", "filename": "b.pdf"}]
+ # Deleted at source, so it is an orphan — not an update.
+ assert preview.orphans == [{"document_id": "c::gone", "filename": "gone.pdf"}]
+ assert preview.synced_count == 2
+ listing.assert_awaited_once()
+
+
+async def test_preview_reports_updates_unavailable_when_listing_aborts(monkeypatch):
+ """Strict gating: an unreachable source means unknown, not "no updates"."""
+ from api.connectors import _preview_for_connector_type
+
+ monkeypatch.setattr(
+ "api.connectors.get_synced_file_ids_for_connector",
+ AsyncMock(return_value=(["c::a"], [], "connector_file_id")),
+ )
+ monkeypatch.setattr(
+ "api.connectors.get_synced_id_to_filename_map", AsyncMock(return_value={"c::a": "a.pdf"})
+ )
+ monkeypatch.setattr(
+ "api.connectors.list_remote_files_for_connector_type", AsyncMock(return_value=None)
+ )
+
+ preview = await _preview_for_connector_type(
+ connector_type="ibm_cos",
+ user_id="alice",
+ connector_service=MagicMock(),
+ session_manager=MagicMock(),
+ jwt_token="token",
+ )
+
+ assert preview.orphans is None
+ assert preview.updates is None
@pytest.mark.asyncio
@@ -513,8 +629,9 @@ async def test_connector_sync_filters_orphan_ids_before_resync(monkeypatch):
request=MagicMock(),
connector_service=service,
session_manager=MagicMock(),
- user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),
session=MagicMock(),
+ rbac=_rbac_allowing("knowledge:delete:anonymous"),
)
assert response.status_code == 201
@@ -556,8 +673,9 @@ async def test_connector_sync_returns_no_files_when_all_ids_are_orphans(monkeypa
request=MagicMock(),
connector_service=service,
session_manager=MagicMock(),
- user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),
session=MagicMock(),
+ rbac=_rbac_allowing("knowledge:delete:anonymous"),
)
assert response.status_code == 200
@@ -598,8 +716,9 @@ async def test_sync_all_returns_deleted_only_without_error(monkeypatch):
request=MagicMock(),
connector_service=service,
session_manager=MagicMock(),
- user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),
session=MagicMock(),
+ rbac=_rbac_allowing("knowledge:delete:anonymous"),
)
body = _json(response)
@@ -609,3 +728,69 @@ async def test_sync_all_returns_deleted_only_without_error(monkeypatch):
assert body["deleted_only_connectors"] == ["google_drive"]
assert "Deleted stale cloud files" in body["message"]
service.sync_specific_files.assert_not_awaited()
+
+
+# ---------------------------------------------------------------------------
+# knowledge:delete:anonymous gates every path that can reach ownerless chunks
+#
+# Ownerless (share-all) chunks are visible to EVERY authenticated user under
+# DLS (securityconfig/roles.yml), and the orphan candidate set is filtered only
+# by connector_type — so it can contain shared files another user's connection
+# ingested. This user's own remote listing will not contain them, orphan
+# detection will call them deleted-at-source, and without this gate an ordinary
+# sync would erase another connection's shared documents.
+# ---------------------------------------------------------------------------
+
+
+async def test_private_sync_without_permission_never_widens_to_ownerless(monkeypatch):
+ from api.connectors import reconcile_orphans_for_connector_type
+
+ conn = _make_connection("c1")
+ connector = _make_connector(remote_file_ids=["a"])
+ service = _make_service([conn], connector_lookup={"c1": connector})
+ opensearch_client = _make_opensearch_client(chunk_ids=["chunk-b-1"])
+ _patch_write_client(monkeypatch)
+ sm = _make_session_manager(opensearch_client)
+
+ await reconcile_orphans_for_connector_type(
+ connector_type="ibm_cos",
+ user_id="alice",
+ connector_service=service,
+ session_manager=sm,
+ jwt_token=None,
+ existing_file_ids=["a", "b"],
+ # No knowledge:delete:anonymous — the default for an ordinary sync.
+ )
+
+ filters = opensearch_client.search.await_args.kwargs["body"]["query"]["bool"]["filter"]
+ # Owner-scoped only: an ownerless chunk cannot match this.
+ assert {"term": {"owner": "alice"}} in filters
+ assert not any("should" in str(f) and "owner" in str(f) for f in filters)
+
+
+async def test_shared_reconcile_without_permission_deletes_nothing(monkeypatch):
+ """An explicitly shared reconcile is refused outright, not silently narrowed."""
+ from api.connectors import reconcile_orphans_for_connector_type
+
+ conn = _make_connection("c1")
+ connector = _make_connector(remote_file_ids=["a"])
+ service = _make_service([conn], connector_lookup={"c1": connector})
+ opensearch_client = _make_opensearch_client(chunk_ids=["chunk-b-1"])
+ write_client = _patch_write_client(monkeypatch)
+ sm = _make_session_manager(opensearch_client)
+
+ deleted_ids = await reconcile_orphans_for_connector_type(
+ connector_type="ibm_cos",
+ user_id="alice",
+ connector_service=service,
+ session_manager=sm,
+ jwt_token=None,
+ existing_file_ids=["a", "b"],
+ shared=True,
+ allow_anonymous_delete=False,
+ )
+
+ assert deleted_ids == []
+ write_client.delete.assert_not_called()
diff --git a/tests/unit/api/test_sync_preview_updates.py b/tests/unit/api/test_sync_preview_updates.py
new file mode 100644
index 000000000..6e75fc0b1
--- /dev/null
+++ b/tests/unit/api/test_sync_preview_updates.py
@@ -0,0 +1,143 @@
+"""Sync preview reports updates, not just deletions.
+
+The preview endpoints feed the "Sync all connectors" confirmation dialog. They
+used to return orphans only, so the dialog could describe deletions but had
+nothing to say about modified files — it filled the gap with the *total synced
+count*, which reads as "all N files will be updated" no matter how many actually
+changed. These endpoints now return the changed files themselves.
+
+`updates_available: false` means the connector cannot answer ahead of time (it
+re-reads every file and decides during ingest), which the dialog must render
+differently from "nothing will be updated".
+"""
+
+import json
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from api import connectors as connectors_api
+
+
+def _json(response):
+ return json.loads(response.body.decode())
+
+
[email protected](autouse=True)
+def _allow_all_connectors(monkeypatch):
+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))
+ monkeypatch.setattr(
+ connectors_api, "_allowed_connector_types_for_request", AsyncMock(return_value=["ibm_cos"])
+ )
+
+
+def _patch_preview(monkeypatch, preview):
+ monkeypatch.setattr(
+ connectors_api, "_preview_for_connector_type", AsyncMock(return_value=preview)
+ )
+
+
+def _user():
+ return SimpleNamespace(user_id="alice", jwt_token="token")
+
+
+async def test_preview_returns_changed_files(monkeypatch):
+ _patch_preview(
+ monkeypatch,
+ connectors_api.ConnectorSyncPreview(
+ orphans=[{"document_id": "c::gone", "filename": "gone.pdf"}],
+ updates=[{"document_id": "c::b", "filename": "b.pdf"}],
+ synced_count=3,
+ ),
+ )
+
+ response = await connectors_api.connector_sync_preview(
+ "ibm_cos",
+ MagicMock(),
+ connector_service=MagicMock(),
+ session_manager=MagicMock(),
+ user=_user(),
+ session=MagicMock(),
+ )
+
+ body = _json(response)
+ assert response.status_code == 200
+ assert body["updates"] == [{"document_id": "c::b", "filename": "b.pdf"}]
+ assert body["updates_available"] is True
+ assert body["orphans"] == [{"document_id": "c::gone", "filename": "gone.pdf"}]
+ assert body["orphans_available"] is True
+ assert body["synced_count"] == 3
+
+
+async def test_preview_marks_updates_unavailable_for_replace_always_connectors(monkeypatch):
+ """Unknowable is not the same as none — the dialog needs to tell them apart."""
+ _patch_preview(
+ monkeypatch,
+ connectors_api.ConnectorSyncPreview(orphans=[], updates=None, synced_count=7),
+ )
+
+ response = await connectors_api.connector_sync_preview(
+ "google_drive",
+ MagicMock(),
+ connector_service=MagicMock(),
+ session_manager=MagicMock(),
+ user=_user(),
+ session=MagicMock(),
+ )
+
+ body = _json(response)
+ assert body["updates"] == []
+ assert body["updates_available"] is False
+ # The synced total is still reported, so the dialog can say "7 re-checked".
+ assert body["synced_count"] == 7
+
+
+async def test_sync_all_preview_groups_updates_by_connector_type(monkeypatch):
+ _patch_preview(
+ monkeypatch,
+ connectors_api.ConnectorSyncPreview(
+ orphans=[],
+ updates=[{"document_id": "c::b", "filename": "b.pdf"}],
+ synced_count=4,
+ ),
+ )
+
+ response = await connectors_api.connectors_sync_all_preview(
+ MagicMock(),
+ connector_service=MagicMock(),
+ session_manager=MagicMock(),
+ user=_user(),
+ session=MagicMock(),
+ )
+
+ body = _json(response)
+ assert body["updates_by_type"] == {"ibm_cos": [{"document_id": "c::b", "filename": "b.pdf"}]}
+ assert body["updates_available_by_type"] == {"ibm_cos": True}
+
+
+async def test_sync_all_preview_survives_a_failing_connector(monkeypatch):
+ """A connector that blows up must not take the whole preview down."""
+ monkeypatch.setattr(
+ connectors_api,
+ "_preview_for_connector_type",
+ AsyncMock(side_effect=RuntimeError("listing exploded")),
+ )
+
+ response = await connectors_api.connectors_sync_all_preview(
+ MagicMock(),
+ connector_service=MagicMock(),
+ session_manager=MagicMock(),
+ user=_user(),
+ session=MagicMock(),
+ )
+
+ assert response.status_code == 200
+ body = _json(response)
+ # Nothing synced and no orphans → the type is omitted entirely.
+ assert body["updates_by_type"] == {}
+ assert body["orphans_by_type"] == {}
diff --git a/tests/unit/connectors/test_chunk_cleanup.py b/tests/unit/connectors/test_chunk_cleanup.py
index ff7370a05..d18c615f5 100644
--- a/tests/unit/connectors/test_chunk_cleanup.py
+++ b/tests/unit/connectors/test_chunk_cleanup.py
@@ -57,6 +57,39 @@ def test_query_shared_targets_ownerless_chunks():
assert {"term": {"owner": "alice"}} not in query["bool"]["filter"]
+def test_query_include_shared_covers_owned_and_ownerless():
+ """Sync scope: the file id pins the document down, so both layouts are ours.
+
+ A COS file ingested with the share-all toggle has no owner field at all; an
+ owner-only term matches none of its chunks and the cleanup silently no-ops.
+ """
+ query = build_connector_file_chunks_query(["f"], owner_user_id="alice", include_shared=True)
+ assert {
+ "bool": {
+ "should": [
+ {"term": {"owner": "alice"}},
+ {"bool": {"must_not": {"exists": {"field": "owner"}}}},
+ ],
+ "minimum_should_match": 1,
+ }
+ } in query["bool"]["filter"]
+ assert {"term": {"owner": "alice"}} not in query["bool"]["filter"]
+
+
+def test_query_include_shared_ignored_without_owner():
+ query = build_connector_file_chunks_query(["f"], include_shared=True)
+ assert len(query["bool"]["filter"]) == 1
+
+
+def test_query_shared_wins_over_include_shared():
+ """shared=True is already ownerless-only; include_shared must not widen it."""
+ query = build_connector_file_chunks_query(
+ ["f"], owner_user_id="alice", shared=True, include_shared=True
+ )
+ assert {"bool": {"must_not": {"exists": {"field": "owner"}}}} in query["bool"]["filter"]
+ assert not any("should" in str(f) and "owner" in str(f) for f in query["bool"]["filter"][1:])
+
+
def test_query_keep_filenames_excluded():
query = build_connector_file_chunks_query(["f"], keep_filenames=["new.pdf", "new_.pdf"])
assert query["bool"]["must_not"] == [{"terms": {"filename": ["new.pdf", "new_.pdf"]}}]
diff --git a/tests/unit/connectors/test_connector_file_type_validation.py b/tests/unit/connectors/test_connector_file_type_validation.py
index fc74f3632..7dccbe00c 100644
--- a/tests/unit/connectors/test_connector_file_type_validation.py
+++ b/tests/unit/connectors/test_connector_file_type_validation.py
@@ -6,6 +6,19 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
+
+def _permissive_rbac():
+ """RBAC stub that grants everything.
+
+ Unit tests pin OPENRAG_RBAC_ENFORCE=true (tests/unit/conftest.py), and every
+ sync now resolves knowledge:delete:anonymous up front to decide whether
+ orphan cleanup may touch ownerless chunks (see delete_orphan_documents).
+ """
+ rbac = MagicMock()
+ rbac.has_permission = AsyncMock(return_value=True)
+ return rbac
+
+
ROOT = Path(__file__).resolve().parent.parent.parent.parent
SRC = ROOT / "src"
if str(SRC) not in sys.path:
@@ -328,7 +341,8 @@ async def test_connector_sync_skip_duplicates_returns_no_files_when_all_selected
request=MagicMock(),
connector_service=connector_service,
session_manager=session_manager,
- user=SimpleNamespace(user_id="user-id", jwt_token="jwt-token"),
+ user=SimpleNamespace(user_id="user-id", jwt_token="jwt-token", db_user_id="user-id"),
+ rbac=_permissive_rbac(),
)
assert response.status_code == 200
@@ -393,7 +407,8 @@ async def test_connector_sync_skip_duplicates_submits_only_expanded_non_duplicat
request=MagicMock(),
connector_service=connector_service,
session_manager=session_manager,
- user=SimpleNamespace(user_id="user-id", jwt_token="jwt-token"),
+ user=SimpleNamespace(user_id="user-id", jwt_token="jwt-token", db_user_id="user-id"),
+ rbac=_permissive_rbac(),
)
assert response.status_code == 201
@@ -449,7 +464,8 @@ async def test_connector_sync_does_not_report_all_duplicates_when_expansion_is_e
request=MagicMock(),
connector_service=connector_service,
session_manager=session_manager,
- user=SimpleNamespace(user_id="user-id", jwt_token="jwt-token"),
+ user=SimpleNamespace(user_id="user-id", jwt_token="jwt-token", db_user_id="user-id"),
+ rbac=_permissive_rbac(),
)
assert response.status_code == 201
diff --git a/tests/unit/connectors/test_content_etag_change_signal.py b/tests/unit/connectors/test_content_etag_change_signal.py
new file mode 100644
index 000000000..1caf4591d
--- /dev/null
+++ b/tests/unit/connectors/test_content_etag_change_signal.py
@@ -0,0 +1,274 @@
+"""Entity tags are the change signal sync can rely on.
+
+Timestamp-only change detection silently fails closed: with no stored
+``modified_time`` to compare against, an overwritten object reports "unchanged"
+forever. Object stores change the ETag on every overwrite, so bucket connectors
+report it in listings and persist it at ingest, and sync compares the two.
+
+Covers the plumbing end to end: normalization, what each connector emits, and
+the enrichment that writes it onto the indexed chunks.
+"""
+
+import sys
+from datetime import datetime
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+
+# ---------------------------------------------------------------------------
+# normalize_etag
+# ---------------------------------------------------------------------------
+
+
+def test_normalize_etag_strips_quotes_and_weak_prefix():
+ """The same tag must compare equal however the API that returned it spells it."""
+ from connectors.base import normalize_etag
+
+ assert normalize_etag('"abc123"') == "abc123"
+ assert normalize_etag("abc123") == "abc123"
+ assert normalize_etag('W/"abc123"') == "abc123"
+ assert normalize_etag('w/"abc123"') == "abc123"
+ assert normalize_etag(' "abc123" ') == "abc123"
+
+
+def test_normalize_etag_keeps_multipart_suffix():
+ """The part count is part of the identity, so it must survive normalization.
+
+ Identical bytes uploaded with different part sizes produce different tags;
+ re-ingesting in that case is wasteful but safe, while treating them as equal
+ would skip a real change.
+ """
+ from connectors.base import normalize_etag
+
+ assert (
+ normalize_etag('"d41d8cd98f00b204e9800998ecf8427e-12"')
+ == "d41d8cd98f00b204e9800998ecf8427e-12"
+ )
+
+
+def test_normalize_etag_returns_none_for_empty_values():
+ from connectors.base import normalize_etag
+
+ assert normalize_etag(None) is None
+ assert normalize_etag("") is None
+ assert normalize_etag('""') is None
+ assert normalize_etag(" ") is None
+ assert normalize_etag(12345) is None
+
+
+# ---------------------------------------------------------------------------
+# Connectors report the tag in listings and persist it on the document
+# ---------------------------------------------------------------------------
+
+
+async def test_ibm_cos_hmac_listing_and_download_carry_the_etag():
+ from connectors.base import CONTENT_ETAG_METADATA_KEY
+ from enhancements.connectors.ibm_cos.connector import IBMCOSConnector
+
+ connector = IBMCOSConnector({"bucket_names": ["b1"], "auth_mode": "hmac"})
+
+ obj = SimpleNamespace(
+ key="reports/q3.pdf",
+ size=10,
+ last_modified=datetime(2026, 5, 7),
+ e_tag='"abc123"',
+ )
+ bucket = MagicMock()
+ bucket.objects.all.return_value = [obj]
+ resource = MagicMock()
+ resource.Bucket.return_value = bucket
+ resource.Object.return_value.get.return_value = {
+ "Body": SimpleNamespace(read=lambda: b"%PDF-1.4"),
+ "ContentType": "application/pdf",
+ "ContentLength": 8,
+ "LastModified": datetime(2026, 5, 7),
+ "ETag": '"abc123"',
+ }
+ resource.meta.client.get_object_acl.side_effect = RuntimeError("acls disabled")
+ connector._handle = resource
+
+ listing = await connector.list_files()
+ assert listing["files"][0]["etag"] == "abc123"
+
+ document = await connector.get_file_content("b1::reports/q3.pdf")
+ assert document.metadata[CONTENT_ETAG_METADATA_KEY] == "abc123"
+
+
+async def test_ibm_cos_iam_listing_carries_the_etag():
+ from enhancements.connectors.ibm_cos.connector import IBMCOSConnector
+
+ connector = IBMCOSConnector({"bucket_names": ["b1"]}) # iam is the default
+ client = MagicMock()
+ client.list_objects_v2.return_value = {
+ "Contents": [
+ {
+ "Key": "reports/q3.pdf",
+ "Size": 10,
+ "LastModified": datetime(2026, 5, 7),
+ "ETag": '"abc123"',
+ }
+ ],
+ "IsTruncated": False,
+ }
+ connector._handle = client
+
+ listing = await connector.list_files()
+ assert listing["files"][0]["etag"] == "abc123"
+
+
+async def test_aws_s3_listing_and_download_carry_the_etag():
+ from connectors.aws_s3.connector import S3Connector
+ from connectors.base import CONTENT_ETAG_METADATA_KEY
+
+ connector = S3Connector({"bucket_names": ["b1"]})
+
+ obj = SimpleNamespace(
+ key="reports/q3.pdf",
+ size=10,
+ last_modified=datetime(2026, 5, 7),
+ e_tag='"abc123"',
+ )
+ bucket = MagicMock()
+ bucket.objects.all.return_value = [obj]
+ resource = MagicMock()
+ resource.Bucket.return_value = bucket
+ resource.Object.return_value.get.return_value = {
+ "Body": SimpleNamespace(read=lambda: b"%PDF-1.4"),
+ "ContentType": "application/pdf",
+ "ContentLength": 8,
+ "LastModified": datetime(2026, 5, 7),
+ "ETag": '"abc123"',
+ }
+ connector._resource = resource
+ connector._client = MagicMock()
+ connector._client.get_object_acl.side_effect = RuntimeError("acls disabled")
+
+ listing = await connector.list_files()
+ assert listing["files"][0]["etag"] == "abc123"
+
+ document = await connector.get_file_content("b1::reports/q3.pdf")
+ assert document.metadata[CONTENT_ETAG_METADATA_KEY] == "abc123"
+
+
+async def test_azure_blob_listing_and_download_carry_the_etag():
+ from connectors.base import CONTENT_ETAG_METADATA_KEY
+ from enhancements.connectors.azure_blob.connector import AzureBlobConnector
+
+ connector = AzureBlobConnector({"container_names": ["c1"]})
+
+ blob = SimpleNamespace(
+ name="reports/q3.pdf",
+ size=10,
+ last_modified=datetime(2026, 5, 7),
+ # Azure marks its tags weak.
+ etag='W/"abc123"',
+ )
+ container_client = MagicMock()
+ container_client.list_blobs.return_value = [blob]
+ client = MagicMock()
+ client.get_container_client.return_value = container_client
+ downloader = MagicMock()
+ downloader.readall.return_value = b"%PDF-1.4"
+ downloader.properties = SimpleNamespace(
+ content_settings=SimpleNamespace(content_type="application/pdf"),
+ last_modified=datetime(2026, 5, 7),
+ etag='W/"abc123"',
+ size=8,
+ )
+ client.get_blob_client.return_value.download_blob.return_value = downloader
+ connector._client = client
+
+ listing = await connector.list_files()
+ assert listing["files"][0]["etag"] == "abc123"
+
+ document = await connector.get_file_content("c1::reports/q3.pdf")
+ assert document.metadata[CONTENT_ETAG_METADATA_KEY] == "abc123"
+
+
+# ---------------------------------------------------------------------------
+# The enrichment persists it onto the indexed chunks
+# ---------------------------------------------------------------------------
+
+
+def _make_service():
+ from connectors.service import ConnectorService
+
+ service = ConnectorService.__new__(ConnectorService)
+ service.session_manager = MagicMock()
+ service.index_name = "test-index"
+ opensearch_client = AsyncMock()
+ service.session_manager.get_user_opensearch_client = MagicMock(return_value=opensearch_client)
+ service.clients = MagicMock()
+ service.clients.opensearch = opensearch_client
+ return service, opensearch_client
+
+
+def _make_document(metadata):
+ from connectors.base import ConnectorDocument, DocumentACL
+
+ return ConnectorDocument(
+ id="b1::reports/q3.pdf",
+ filename="q3.pdf",
+ mimetype="application/pdf",
+ content=b"",
+ source_url="cos://b1/reports/q3.pdf",
+ acl=DocumentACL(owner="alice"),
+ modified_time=datetime(2026, 5, 7),
+ created_time=datetime(2026, 5, 1),
+ metadata=metadata,
+ )
+
+
+async def test_metadata_update_promotes_content_etag_to_a_top_level_field(monkeypatch):
+ """Stored as its own keyword so sync can read it back with a terms agg."""
+ from connectors.base import CONTENT_ETAG_METADATA_KEY
+
+ service, opensearch_client = _make_service()
+
+ async def _noop_acl(**_kwargs):
+ return {"status": "unchanged"}
+
+ monkeypatch.setattr("utils.acl_utils.update_document_acl", _noop_acl)
+
+ document = _make_document({"size": 8, CONTENT_ETAG_METADATA_KEY: '"abc123"'})
+ await service._update_connector_metadata(
+ document, owner_user_id="alice", connector_type="ibm_cos"
+ )
+
+ script = opensearch_client.update_by_query.await_args.kwargs["body"]["script"]
+ assert "ctx._source.content_etag = params.content_etag" in script["source"]
+ # Normalized on the way in so it compares equal to a fresh listing's tag.
+ assert script["params"]["content_etag"] == "abc123"
+
+
+async def test_metadata_update_omits_content_etag_when_connector_reports_none(monkeypatch):
+ """OAuth connectors report no tag; the field must stay absent, not empty."""
+ service, opensearch_client = _make_service()
+
+ async def _noop_acl(**_kwargs):
+ return {"status": "unchanged"}
+
+ monkeypatch.setattr("utils.acl_utils.update_document_acl", _noop_acl)
+
+ await service._update_connector_metadata(
+ _make_document({"site": "marketing"}), owner_user_id="alice", connector_type="sharepoint"
+ )
+
+ script = opensearch_client.update_by_query.await_args.kwargs["body"]["script"]
+ # The script guards on null, so a None param leaves existing chunks alone.
+ assert script["params"]["content_etag"] is None
+ assert "if (params.content_etag != null)" in script["source"]
diff --git a/tests/unit/test_connector_sync_bucket_filter_shared.py b/tests/unit/test_connector_sync_bucket_filter_shared.py
index cc622e63b..a8ec7e82f 100644
--- a/tests/unit/test_connector_sync_bucket_filter_shared.py
+++ b/tests/unit/test_connector_sync_bucket_filter_shared.py
@@ -63,7 +63,7 @@ async def test_bucket_filter_sync_forwards_shared_to_new_and_changed_batches():
new=AsyncMock(return_value=(["changed-file-1"], [], "connector_file_id")),
),
patch(
- "api.connectors.get_synced_id_to_modified_time_map",
+ "api.connectors.get_synced_file_state_map",
new=AsyncMock(return_value={"changed-file-1": 0.0}),
),
patch(
diff --git a/tests/unit/test_connector_sync_shared_resync.py b/tests/unit/test_connector_sync_shared_resync.py
new file mode 100644
index 000000000..c66106008
--- /dev/null
+++ b/tests/unit/test_connector_sync_shared_resync.py
@@ -0,0 +1,227 @@
+"""Re-syncing a COS file that was ingested as shared must update it in place.
+
+Regression test for "overwrite a file in COS is not reflected after syncing
+connectors". The share-all toggle indexes chunks with no ``owner`` field, and
+nothing persists that choice, so the re-sync paths (Sync all connectors, and
+``/connectors/{type}/sync`` with an empty body) used to run with shared=False.
+Every owner-scoped step downstream then missed the ownerless chunks: the
+replace-delete matched nothing, ``resolve_duplicate_filename`` reported the file
+as an unreplaceable duplicate and skipped it, and ``_reconcile_shared_owner``
+stamped the current user onto the chunks on the way out — so the modified object
+never reached the index and the document silently lost its shared status.
+
+Those paths now pass shared=None ("no explicit intent") and each file inherits
+the sharing state it already has in the index. An explicit toggle value still
+wins, so a user can still flip a document between shared and private.
+"""
+
+from datetime import datetime
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from connectors.base import ConnectorDocument, DocumentACL
+from models.processors import ConnectorFileProcessor
+from models.tasks import FileTask, TaskStatus, UploadTask
+
+COS_FILE_ID = "my-bucket::reports/q3.pdf"
+COS_FILENAME = "q3.pdf"
+
+
[email protected](autouse=True)
+def backend_write_client(monkeypatch):
+ """Backend OpenSearch write client (clients.opensearch) used for deletes."""
+ import config.settings as cfg
+
+ client = AsyncMock()
+ client.delete = AsyncMock(return_value={"result": "deleted"})
+ monkeypatch.setattr(cfg.clients, "opensearch", client)
+ return client
+
+
+def _make_document() -> ConnectorDocument:
+ return ConnectorDocument(
+ id=COS_FILE_ID,
+ filename=COS_FILENAME,
+ mimetype="application/pdf",
+ content=b"%PDF-1.4 updated bytes",
+ source_url=f"cos://{COS_FILE_ID}",
+ acl=DocumentACL(owner="service-instance-1"),
+ modified_time=datetime.now(),
+ created_time=datetime.now(),
+ )
+
+
+def _build_processor(*, shared: bool | None) -> ConnectorFileProcessor:
+ document_service = MagicMock()
+ document_service.docling_service = MagicMock()
+ document_service.session_manager = MagicMock()
+ document_service.session_manager.get_user_opensearch_client = MagicMock()
+ connector_service = MagicMock()
+ connector_service._update_connector_metadata = AsyncMock()
+ return ConnectorFileProcessor(
+ connector_service=connector_service,
+ connection_id="conn-1",
+ files_to_process=[COS_FILE_ID],
+ user_id="user-1",
+ jwt_token="jwt",
+ owner_name="Alice",
+ owner_email="[email protected]",
+ document_service=document_service,
+ models_service=MagicMock(),
+ # Timestamp change detection re-ingests changed blobs with replace on.
+ replace_duplicates=True,
+ connector_type="ibm_cos",
+ shared=shared,
+ )
+
+
+def _owner_clause_matches(query: dict, indexed_owner) -> bool:
+ """Apply a query's owner scoping to the indexed chunks, the way OpenSearch would.
+
+ The whole defect lives in this filter, so the fake has to honor it rather
+ than returning hits unconditionally: an owner-only term does not match
+ ownerless (shared) chunks, which is what made the replace-delete remove
+ nothing and the file get skipped as an unreplaceable duplicate.
+ """
+ for clause in query.get("bool", {}).get("filter", []):
+ term_owner = clause.get("term", {}).get("owner")
+ if term_owner is not None:
+ return indexed_owner == term_owner
+ inner = clause.get("bool", {})
+ if inner.get("must_not") == {"exists": {"field": "owner"}}:
+ return indexed_owner is None
+ shoulds = inner.get("should")
+ if shoulds and any("owner" in str(s) for s in shoulds):
+ # owner == me OR owner absent
+ owners = [s.get("term", {}).get("owner") for s in shoulds]
+ return indexed_owner is None or indexed_owner in owners
+ return True # unscoped query
+
+
+def _wire(processor: ConnectorFileProcessor, document: ConnectorDocument, *, indexed_owner):
+ """Wire an OpenSearch client for a file already indexed under ``indexed_owner``.
+
+ ``indexed_owner`` is the ``owner`` value on the existing chunks: None models
+ a share-all ingest (field absent), a string models a private one, and
+ ``"absent"`` models a file that is not indexed at all.
+ """
+ opensearch_client = AsyncMock()
+
+ async def mock_search(index, body, **kwargs):
+ # 1. Indexed-sharing-state probe: the only search asking for _source owner.
+ if body.get("_source") == ["owner"]:
+ if indexed_owner == "absent":
+ return {"hits": {"hits": []}}
+ source = {} if indexed_owner is None else {"owner": indexed_owner}
+ return {"hits": {"hits": [{"_id": "chunk-1", "_source": source}]}}
+ # 2. Bulk filename duplicate check (terms aggregation).
+ if "aggs" in body:
+ buckets = [] if indexed_owner == "absent" else [{"key": COS_FILENAME, "doc_count": 1}]
+ return {"aggregations": {"filenames": {"buckets": buckets}}}
+ query = body.get("query", {})
+ query_str = str(query)
+ # 3. Rename cleanup by connector file id — nothing stale under an old name.
+ if "connector_file_id" in query_str:
+ return {"hits": {"hits": []}}
+ # 4. Replace-delete id collection for the filename, owner scoping applied.
+ if "filename" in query_str:
+ if indexed_owner == "absent" or not _owner_clause_matches(query, indexed_owner):
+ return {"hits": {"hits": []}}
+ return {"hits": {"hits": [{"_id": "chunk-1"}, {"_id": "chunk-2"}]}}
+ # 5. Content-hash check: the overwritten bytes hash to something new.
+ return {"hits": {"hits": []}}
+
+ opensearch_client.search = mock_search
+ opensearch_client.delete = AsyncMock(return_value={"result": "deleted"})
+
+ connector = MagicMock()
+ connector.get_file_content = AsyncMock(return_value=document)
+ processor.connector_service.get_connector = AsyncMock(return_value=connector)
+ connection = MagicMock()
+ connection.connector_type = "ibm_cos"
+ processor.connector_service.connection_manager = MagicMock()
+ processor.connector_service.connection_manager.get_connection = AsyncMock(
+ return_value=connection
+ )
+ processor.document_service.session_manager.get_user_opensearch_client.return_value = (
+ opensearch_client
+ )
+ return opensearch_client
+
+
+async def _run(processor) -> tuple[FileTask, UploadTask, AsyncMock]:
+ file_task = FileTask(file_path=COS_FILE_ID)
+ upload_task = UploadTask(task_id="task-1", total_files=1)
+ with patch.object(
+ processor,
+ "process_document_standard",
+ new=AsyncMock(return_value={"status": "indexed", "id": "hash-new"}),
+ ) as mock_process:
+ await processor.process_item(upload_task, COS_FILE_ID, file_task)
+ return file_task, upload_task, mock_process
+
+
+async def test_resync_replaces_shared_cos_file_and_keeps_it_shared(
+ monkeypatch, backend_write_client
+):
+ """The defect: an overwritten COS object indexed as shared must be re-ingested."""
+ monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", True)
+ processor = _build_processor(shared=None)
+ _wire(processor, _make_document(), indexed_owner=None)
+
+ file_task, upload_task, mock_process = await _run(processor)
+
+ # Not skipped as a duplicate — the stale chunks were deleted and the new
+ # bytes went through ingestion.
+ assert file_task.status == TaskStatus.COMPLETED
+ assert file_task.result["status"] == "indexed"
+ assert upload_task.successful_files == 1
+ backend_write_client.delete.assert_awaited()
+ mock_process.assert_awaited_once()
+ # ...and it stays shared, rather than being quietly reassigned to the
+ # user who happened to run the sync.
+ assert mock_process.await_args.kwargs["shared"] is True
+
+
+async def test_resync_keeps_private_cos_file_private(monkeypatch):
+ """A file indexed with an owner is re-ingested private, not shared."""
+ monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", True)
+ processor = _build_processor(shared=None)
+ _wire(processor, _make_document(), indexed_owner="user-1")
+
+ file_task, _, mock_process = await _run(processor)
+
+ assert file_task.status == TaskStatus.COMPLETED
+ mock_process.assert_awaited_once()
+ assert mock_process.await_args.kwargs["shared"] is False
+
+
+async def test_explicit_shared_false_overrides_indexed_state(monkeypatch):
+ """The UI toggle still wins: an explicit False makes a shared file private."""
+ monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", True)
+ processor = _build_processor(shared=False)
+ _wire(processor, _make_document(), indexed_owner=None)
+
+ file_task, _, mock_process = await _run(processor)
+
+ assert file_task.status == TaskStatus.COMPLETED
+ mock_process.assert_awaited_once()
+ assert mock_process.await_args.kwargs["shared"] is False
+
+
+async def test_new_file_with_no_intent_falls_back_to_private(monkeypatch):
+ """A file that isn't indexed yet has no state to inherit → private."""
+ monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", True)
+ processor = _build_processor(shared=None)
+ _wire(processor, _make_document(), indexed_owner="absent")
+
+ file_task, _, mock_process = await _run(processor)
+
+ assert file_task.status == TaskStatus.COMPLETED
+ mock_process.assert_awaited_once()
+ assert mock_process.await_args.kwargs["shared"] is False
diff --git a/tests/unit/test_reconcile_shared_owner.py b/tests/unit/test_reconcile_shared_owner.py
index 4aa6e43ad..1e4471c46 100644
--- a/tests/unit/test_reconcile_shared_owner.py
+++ b/tests/unit/test_reconcile_shared_owner.py
@@ -40,7 +40,7 @@ async def test_reconcile_shared_true_omits_owner_via_script():
settings_module.clients.opensearch = write_client
processor = _make_processor(shared=True)
- await processor._reconcile_shared_owner("report.pdf")
+ await processor._reconcile_shared_owner("report.pdf", True)
assert write_client.update_by_query.await_count >= 1
call = write_client.update_by_query.await_args_list[0]
@@ -60,7 +60,7 @@ async def test_reconcile_shared_false_sets_owner_via_script():
settings_module.clients.opensearch = write_client
processor = _make_processor(shared=False)
- await processor._reconcile_shared_owner("report.pdf")
+ await processor._reconcile_shared_owner("report.pdf", False)
call = write_client.update_by_query.await_args_list[0]
params = call.kwargs["body"]["script"]["params"]
@@ -76,7 +76,7 @@ async def test_reconcile_shared_owner_noop_without_write_client():
processor = _make_processor(shared=True)
# Must not raise even though there's nothing to write to.
- await processor._reconcile_shared_owner("report.pdf")
+ await processor._reconcile_shared_owner("report.pdf", True)
@pytest.mark.asyncio
@@ -87,4 +87,4 @@ async def test_reconcile_shared_owner_swallows_update_errors():
processor = _make_processor(shared=True)
# A failed reconciliation must not fail the (already-skipped) sync item.
- await processor._reconcile_shared_owner("report.pdf")
+ await processor._reconcile_shared_owner("report.pdf", True)
diff --git a/tests/unit/test_shared_flag.py b/tests/unit/test_shared_flag.py
index 407c25894..2fc330736 100644
--- a/tests/unit/test_shared_flag.py
+++ b/tests/unit/test_shared_flag.py
@@ -151,11 +151,18 @@ def test_build_replace_filename_query_differs_from_owned_query():
# ---------------------------------------------------------------------------
-def test_connector_sync_body_defaults_shared_false():
+def test_connector_sync_body_defaults_shared_to_no_intent():
+ """Omitting shared means "no explicit intent", not "private".
+
+ The connector UI sends the flag only while the share-all toggle is on
+ screen, and the re-sync paths never send it at all. None lets each file keep
+ the sharing state it already has in the index instead of being silently
+ un-shared; a file that isn't indexed yet still falls back to private.
+ """
from api.connectors import ConnectorSyncBody
body = ConnectorSyncBody()
- assert body.shared is False
+ assert body.shared is None
def test_connector_sync_body_shared_true():
@@ -165,11 +172,11 @@ def test_connector_sync_body_shared_true():
assert body.shared is True
-def test_connector_sync_body_shared_backwards_compat():
- """Existing clients that omit shared get False."""
+def test_connector_sync_body_shared_false_is_explicit():
+ """An explicitly sent False still means "index this privately"."""
from api.connectors import ConnectorSyncBody
- body = ConnectorSyncBody(selected_files=["file-1"])
+ body = ConnectorSyncBody(selected_files=["file-1"], shared=False)
assert body.shared is False
diff --git a/tests/unit/test_traditional_duplicate_handling.py b/tests/unit/test_traditional_duplicate_handling.py
index ea6f6ef27..608385aa6 100644
--- a/tests/unit/test_traditional_duplicate_handling.py
+++ b/tests/unit/test_traditional_duplicate_handling.py
@@ -206,6 +206,54 @@ async def test_delete_document_by_filename_shared_without_owner(monkeypatch):
assert {"bool": {"must_not": {"exists": {"field": "owner"}}}} in filters
+async def test_delete_document_by_filename_replaces_owned_and_ownerless(monkeypatch):
+ """With an owner in hand the replace scope is "mine OR ownerless", always.
+
+ ``shared`` says how the replacement will be written; it must not narrow what
+ gets deleted. An owner-only scope matched none of a shared document's
+ chunks, so the caller saw zero deletions, reported "nothing to replace" and
+ skipped the file — even though the (owner-agnostic) duplicate check had just
+ found it and the user had confirmed the overwrite.
+ """
+ from types import SimpleNamespace
+
+ from models.processors import TaskProcessor
+
+ admin_client = AsyncMock()
+ admin_client.delete = AsyncMock(return_value={"result": "deleted"})
+ monkeypatch.setattr(
+ "config.settings.clients",
+ SimpleNamespace(opensearch=admin_client),
+ )
+ monkeypatch.setattr("config.settings.get_index_name", lambda: "test-index")
+
+ opensearch_client = AsyncMock()
+ opensearch_client.search = AsyncMock(
+ return_value={"_scroll_id": None, "hits": {"hits": [{"_id": "chunk-1"}]}}
+ )
+
+ processor = TaskProcessor()
+ deleted = await processor.delete_document_by_filename(
+ "report.pdf",
+ opensearch_client,
+ owner_user_id="user-123",
+ shared=False,
+ )
+
+ assert deleted == 1
+ filters = opensearch_client.search.await_args.kwargs["body"]["query"]["bool"]["filter"]
+ assert {
+ "bool": {
+ "should": [
+ {"term": {"owner": "user-123"}},
+ {"bool": {"must_not": {"exists": {"field": "owner"}}}},
+ ],
+ "minimum_should_match": 1,
+ }
+ } in filters
+
+
def _build_s3_processor(replace_duplicates: bool) -> S3FileProcessor:
document_service = MagicMock()
document_service.session_manager = MagicMock()