Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

feat(kb): defer ingestion while a collection is frozen (#3757)

蓝鱼bluefish686e2dc73769
diff --git a/example.env b/example.env
index 77b86473c..a287c28b1 100644
--- a/example.env
+++ b/example.env
@@ -426,6 +426,11 @@ XAGENT_REDIS_URL="redis://localhost:6379/0"
# XAGENT_BACKGROUND_JOB_STALE_SECONDS="7200"
# Scheduler interval for scanning stale background jobs.
# XAGENT_BACKGROUND_JOB_SWEEP_INTERVAL_SECONDS="300"
+# An ingest job refused because its collection is frozen for an embedding
+# migration retries after this delay, until the budget (seconds since the
+# first refusal) is spent.
+# XAGENT_KB_MIGRATION_DEFER_DELAY_SECONDS="15"
+# XAGENT_KB_MIGRATION_DEFER_BUDGET_SECONDS="600"
# Milvus only: purge cadence for leftover rows, scheduled through Celery Beat
# (no Beat means no purge). Deletes wait twice XAGENT_BACKGROUND_JOB_STALE_SECONDS; ignored by LanceDB.
# XAGENT_KB_MILVUS_CLEANUP_INTERVAL_SECONDS="3600"
diff --git a/frontend/src/components/kb/knowledge-base-creation-dialog.test.tsx b/frontend/src/components/kb/knowledge-base-creation-dialog.test.tsx
index 60c6e3491..5eb7e3a96 100644
--- a/frontend/src/components/kb/knowledge-base-creation-dialog.test.tsx
+++ b/frontend/src/components/kb/knowledge-base-creation-dialog.test.tsx
@@ -14,6 +14,7 @@ vi.mock("@/contexts/auth-context", () => ({
vi.mock("@/lib/api-wrapper", () => ({
apiRequest: apiRequestMock,
+ uploadWithCollectionMigrationRetry: apiRequestMock,
parseApiResponse: async (response: { json: () => Promise<unknown> }) => ({
data: await response.json(),
text: null,
@@ -621,6 +622,8 @@ describe("KnowledgeBaseCreationDialog collection naming", () => {
expect(apiRequestMock).toHaveBeenCalledWith(
"http://api.local/api/kb/ingest-cloud",
expect.any(Object),
+ expect.any(Function),
+ expect.any(Object),
)
})
const cloudCall = apiRequestMock.mock.calls.find(
@@ -707,6 +710,56 @@ describe("KnowledgeBaseCreationDialog collection naming", () => {
consoleErrorSpy.mockRestore()
}
})
+
+ it("clears the upgrading notice once a retried web import is accepted", async () => {
+ const base = apiRequestMock.getMockImplementation()!
+ apiRequestMock.mockImplementation(
+ (url: string, options?: RequestInit, onUpgrading?: () => void) => {
+ if (url === "http://api.local/api/kb/ingest-web/jobs") {
+ onUpgrading?.()
+ return Promise.resolve(
+ createJsonResponse({
+ id: "job-2",
+ user_id: 1,
+ job_type: "kb.ingest.web",
+ queue: "kb",
+ status: "running",
+ progress: { message: "Crawling", completed: 0, total: 1 },
+ result: null,
+ error_message: null,
+ celery_task_id: "task-2",
+ attempts: 1,
+ max_attempts: 3,
+ })
+ )
+ }
+ if (url === "http://api.local/api/jobs/job-2") {
+ return new Promise(() => {})
+ }
+ return base(url, options)
+ }
+ )
+
+ const { container } = render(
+ <KnowledgeBaseCreationDialog open={true} onOpenChange={vi.fn()} onSuccess={vi.fn()} />
+ )
+
+ fireEvent.change(container.querySelector("#collection_name") as HTMLInputElement, {
+ target: { value: "web_collection" },
+ })
+
+ await goToStep3(container, "web")
+ fireEvent.click(screen.getByText("kb.dialog.createButton"))
+
+ await waitFor(() => {
+ expect(
+ apiRequestMock.mock.calls.some(
+ ([url]) => url === "http://api.local/api/jobs/job-2"
+ )
+ ).toBe(true)
+ }, { timeout: 5000 })
+ expect(screen.queryByText(/kb\.migration\.upgrading/)).toBeNull()
+ })
})
const RESERVE_URL = "http://api.local/api/knowledge-bases/team-docs/reserve-team"
diff --git a/frontend/src/components/kb/knowledge-base-creation-dialog.tsx b/frontend/src/components/kb/knowledge-base-creation-dialog.tsx
index 99e475d7c..2ec2feabc 100644
--- a/frontend/src/components/kb/knowledge-base-creation-dialog.tsx
+++ b/frontend/src/components/kb/knowledge-base-creation-dialog.tsx
@@ -31,7 +31,7 @@ import {
import { useI18n } from "@/contexts/i18n-context"
import type { TranslationKey } from "@/i18n/translations"
import { useAuth } from "@/contexts/auth-context"
-import { apiRequest, getUploadErrorMessage, isJsonRecord, parseApiResponse, UPLOAD_ERROR_MESSAGES } from "@/lib/api-wrapper"
+import { apiRequest, getUploadErrorMessage, isJsonRecord, parseApiResponse, UPLOAD_ERROR_MESSAGES, uploadWithCollectionMigrationRetry } from "@/lib/api-wrapper"
import { Model } from "@/lib/models"
import {
Upload,
@@ -236,6 +236,15 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: K
const [ingestionResults, setIngestionResults] = useState<IngestionResult[]>([])
const [isDragging, setIsDragging] = useState(false)
const fileInputRef = useRef<HTMLInputElement>(null)
+ const uploadAbortRef = useRef<AbortController | null>(null)
+ const mountedRef = useRef(true)
+ useEffect(() => {
+ mountedRef.current = true
+ return () => {
+ mountedRef.current = false
+ uploadAbortRef.current?.abort()
+ }
+ }, [])
const [currentUploadFileName, setCurrentUploadFileName] = useState<string | null>(null)
const [currentUploadCollection, setCurrentUploadCollection] = useState<string | null>(null)
const [completedUploadCount, setCompletedUploadCount] = useState(0)
@@ -487,10 +496,19 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: K
toast.warning(t("kb.ownership.releaseFailed", { name: collection }))
}
+ const beginAbortableUpload = () => {
+ uploadAbortRef.current?.abort()
+ const controller = new AbortController()
+ uploadAbortRef.current = controller
+ return controller
+ }
+
const resetState = () => {
+ uploadAbortRef.current?.abort()
setSelectedFiles([])
setOwnership("personal")
setUploadProgress(0)
+ setUploadProgressDetail(null)
setIngestionResults([])
setWebIngestionResult(null)
setNewCollectionName("")
@@ -521,6 +539,7 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: K
toast.error(t("kb.errors.uploadFileRequired"))
return
}
+ const controller = beginAbortableUpload()
setIsUploading(true)
// Classify on the status code, not on the backend's English wording.
@@ -554,13 +573,17 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: K
normalizeIngestionConfigForFilename(ingestionConfig, file.name)
)
- const response = await apiRequest(
+ const response = await uploadWithCollectionMigrationRetry(
`${apiUrl}/api/kb/ingest${useBackgroundJobs ? "/jobs" : ""}`,
{
method: "POST",
body: formData
- }
+ },
+ () => setUploadProgressDetail(t("kb.migration.upgrading")),
+ controller.signal
)
+ setUploadProgressDetail(null)
+ if (controller.signal.aborted) return
const parsed = await parseApiResponse(response)
@@ -669,10 +692,12 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: K
onSuccess?.(successfulCollections)
}
} finally {
- setIsUploading(false)
- setCurrentUploadFileName(null)
- setCurrentUploadCollection(null)
- setUploadProgressDetail(null)
+ if (mountedRef.current) {
+ setIsUploading(false)
+ setCurrentUploadFileName(null)
+ setCurrentUploadCollection(null)
+ setUploadProgressDetail(null)
+ }
}
}
@@ -681,11 +706,13 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: K
toast.error(t("kb.errors.startUrlRequired"))
return
}
+ const controller = beginAbortableUpload()
setIsWebIngesting(true)
let failedStatus: number | undefined
setWebIngestionProgress(0)
setWebIngestionResult(null)
+ setUploadProgressDetail(null)
const collectionName = trimmedCollectionName
const teamClaimed = { current: false }
@@ -722,13 +749,17 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: K
setWebIngestionProgress(10)
- const response = await apiRequest(
+ const response = await uploadWithCollectionMigrationRetry(
`${apiUrl}/api/kb/ingest-web${useBackgroundJobs ? "/jobs" : ""}`,
{
method: "POST",
body: formData
- }
+ },
+ () => setUploadProgressDetail(t("kb.migration.upgrading")),
+ controller.signal
)
+ setUploadProgressDetail(null)
+ if (controller.signal.aborted) return
const parsed = await parseApiResponse(response)
@@ -802,17 +833,21 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: K
})
if (teamClaimed.current) void releaseTeamName(collectionName)
} finally {
- setIsWebIngesting(false)
- setWebIngestionProgress(0)
+ if (mountedRef.current) {
+ setIsWebIngesting(false)
+ setWebIngestionProgress(0)
+ }
}
}
const handleCloudIngest = async () => {
if (totalCloudFiles === 0) return
+ const controller = beginAbortableUpload()
setIsCloudConnecting(true)
let failedStatus: number | undefined
setIngestionResults([])
+ setUploadProgressDetail(null)
const collectionName = trimmedCollectionName
const teamClaimed = { current: false }
@@ -859,13 +894,15 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: K
retry_delay: ingestionConfig.retry_delay
}
- const response = await apiRequest(`${getApiUrl()}/api/kb/ingest-cloud`, {
+ const response = await uploadWithCollectionMigrationRetry(`${getApiUrl()}/api/kb/ingest-cloud`, {
method: "POST",
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify(requestBody)
- })
+ }, () => setUploadProgressDetail(t("kb.migration.upgrading")), controller.signal)
+ setUploadProgressDetail(null)
+ if (controller.signal.aborted) return
const parsed = await parseApiResponse(response)
@@ -935,7 +972,7 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: K
void releaseTeamName(collectionName)
}
} finally {
- setIsCloudConnecting(false)
+ if (mountedRef.current) setIsCloudConnecting(false)
}
}
diff --git a/frontend/src/components/kb/knowledge-base-detail.component.test.tsx b/frontend/src/components/kb/knowledge-base-detail.component.test.tsx
index c2a0ec349..c147d9edd 100644
--- a/frontend/src/components/kb/knowledge-base-detail.component.test.tsx
+++ b/frontend/src/components/kb/knowledge-base-detail.component.test.tsx
@@ -7,6 +7,7 @@ const toastErrorMock = vi.hoisted(() => vi.fn())
vi.mock("@/lib/api-wrapper", () => ({
apiRequest: apiRequestMock,
+ uploadWithCollectionMigrationRetry: apiRequestMock,
}))
vi.mock("@/lib/utils", () => ({
diff --git a/frontend/src/components/kb/knowledge-base-detail.tsx b/frontend/src/components/kb/knowledge-base-detail.tsx
index a9c0eeb32..89915aae3 100644
--- a/frontend/src/components/kb/knowledge-base-detail.tsx
+++ b/frontend/src/components/kb/knowledge-base-detail.tsx
@@ -13,7 +13,7 @@ import { ScrollArea } from "@/components/ui/scroll-area"
import { Select } from "@/components/ui/select"
import { Badge } from "@/components/ui/badge"
import { ConfirmDialog } from "@/components/ui/confirm-dialog"
-import { apiRequest, getUploadErrorMessage, isJsonRecord, parseApiResponse, UPLOAD_ERROR_MESSAGES } from "@/lib/api-wrapper"
+import { apiRequest, getUploadErrorMessage, isJsonRecord, parseApiResponse, UPLOAD_ERROR_MESSAGES, uploadWithCollectionMigrationRetry } from "@/lib/api-wrapper"
import { getApiUrl } from "@/lib/utils"
import {
getBackgroundJobFailureMessage,
@@ -162,6 +162,15 @@ export function KnowledgeBaseDetailContent({ collectionName }: { collectionName:
const [ingestionResults, setIngestionResults] = useState<IngestionResult[]>([])
const [currentUploadFileName, setCurrentUploadFileName] = useState<string | null>(null)
const [completedUploadCount, setCompletedUploadCount] = useState(0)
+ const uploadAbortRef = useRef<AbortController | null>(null)
+ const mountedRef = useRef(true)
+ useEffect(() => {
+ mountedRef.current = true
+ return () => {
+ mountedRef.current = false
+ uploadAbortRef.current?.abort()
+ }
+ }, [])
const [isAddSourceOpen, setIsAddSourceOpen] = useState(false)
const [activeAddSourceMode, setActiveAddSourceMode] = useState<"web" | "file" | null>(null)
const [isDragging, setIsDragging] = useState(false)
@@ -214,6 +223,7 @@ export function KnowledgeBaseDetailContent({ collectionName }: { collectionName:
const [isWebIngesting, setIsWebIngesting] = useState(false)
const [webIngestionProgress, setWebIngestionProgress] = useState(0)
const [webIngestionResult, setWebIngestionResult] = useState<WebIngestionResult | null>(null)
+ const [webIngestionNotice, setWebIngestionNotice] = useState<string | null>(null)
const [webIngestionConfig, setWebIngestionConfig] = useState({
start_url: "",
max_pages: 100,
@@ -405,8 +415,16 @@ export function KnowledgeBaseDetailContent({ collectionName }: { collectionName:
setSelectedFiles(prev => prev.filter((_, i) => i !== index))
}
+ const beginAbortableUpload = () => {
+ uploadAbortRef.current?.abort()
+ const controller = new AbortController()
+ uploadAbortRef.current = controller
+ return controller
+ }
+
const doUpload = async () => {
if (selectedFiles.length === 0) return
+ const controller = beginAbortableUpload()
setIsUploading(true)
setUploadProgress(0)
@@ -430,13 +448,17 @@ export function KnowledgeBaseDetailContent({ collectionName }: { collectionName:
normalizeIngestionConfigForFilename(ingestionConfig, file.name)
)
- const response = await apiRequest(
+ const response = await uploadWithCollectionMigrationRetry(
`${apiUrl}/api/kb/ingest${useBackgroundJobs ? "/jobs" : ""}`,
{
method: "POST",
body: formData
- }
+ },
+ () => setUploadProgressDetail(t("kb.migration.upgrading")),
+ controller.signal
)
+ setUploadProgressDetail(null)
+ if (controller.signal.aborted) return
const parsed = await parseApiResponse(response)
@@ -533,9 +555,11 @@ export function KnowledgeBaseDetailContent({ collectionName }: { collectionName:
description: toastContent.description,
})
} finally {
- setIsUploading(false)
- setCurrentUploadFileName(null)
- setUploadProgressDetail(null)
+ if (mountedRef.current) {
+ setIsUploading(false)
+ setCurrentUploadFileName(null)
+ setUploadProgressDetail(null)
+ }
}
}
@@ -599,6 +623,8 @@ export function KnowledgeBaseDetailContent({ collectionName }: { collectionName:
setIsWebIngesting(true)
setWebIngestionProgress(0)
setWebIngestionResult(null)
+ setWebIngestionNotice(null)
+ const controller = beginAbortableUpload()
try {
const apiUrl = getApiUrl()
@@ -632,13 +658,17 @@ export function KnowledgeBaseDetailContent({ collectionName }: { collectionName:
setWebIngestionProgress(10)
- const response = await apiRequest(
+ const response = await uploadWithCollectionMigrationRetry(
`${apiUrl}/api/kb/ingest-web${useBackgroundJobs ? "/jobs" : ""}`,
{
method: "POST",
body: formData
- }
+ },
+ () => setWebIngestionNotice(t("kb.migration.upgrading")),
+ controller.signal
)
+ setWebIngestionNotice(null)
+ if (controller.signal.aborted) return
const parsed = await parseApiResponse(response)
@@ -730,8 +760,10 @@ export function KnowledgeBaseDetailContent({ collectionName }: { collectionName:
description: toastContent.description,
})
} finally {
- setIsWebIngesting(false)
- setWebIngestionProgress(0)
+ if (mountedRef.current) {
+ setIsWebIngesting(false)
+ setWebIngestionProgress(0)
+ }
}
}
@@ -1568,7 +1600,7 @@ export function KnowledgeBaseDetailContent({ collectionName }: { collectionName:
{isWebIngesting ? (
<div className="flex items-center gap-2">
<Loader2 className="h-4 w-4 animate-spin" />
- {t("kb.dialog.webImport.status.crawling")} ({Math.round(webIngestionProgress)}%)
+ {webIngestionNotice || t("kb.dialog.webImport.status.crawling")} ({Math.round(webIngestionProgress)}%)
</div>
) : (
t("kb.index.startImport")
diff --git a/frontend/src/components/kb/knowledge-base-detail.web-ingest.test.tsx b/frontend/src/components/kb/knowledge-base-detail.web-ingest.test.tsx
index a1e448d84..7421a998c 100644
--- a/frontend/src/components/kb/knowledge-base-detail.web-ingest.test.tsx
+++ b/frontend/src/components/kb/knowledge-base-detail.web-ingest.test.tsx
@@ -7,6 +7,7 @@ const toastErrorMock = vi.hoisted(() => vi.fn())
vi.mock("@/lib/api-wrapper", () => ({
apiRequest: apiRequestMock,
+ uploadWithCollectionMigrationRetry: apiRequestMock,
parseApiResponse: async (response: { json: () => Promise<unknown> }) => ({
data: await response.json(),
text: null,
@@ -290,6 +291,94 @@ describe("KnowledgeBaseDetailContent web ingest", () => {
expect.anything()
)
})
+
+ it("clears the upgrading notice once a retried web import is accepted", async () => {
+ const previous = apiRequestMock.getMockImplementation()!
+ apiRequestMock.mockImplementation(
+ (url: string, options?: RequestInit, onUpgrading?: () => void) => {
+ if (url === "http://api.local/api/kb/ingest-web/jobs") {
+ onUpgrading?.()
+ return Promise.resolve(
+ createJsonResponse({
+ id: "job-2",
+ user_id: 1,
+ job_type: "kb.ingest.web",
+ queue: "kb",
+ status: "running",
+ progress: { message: "Crawling", completed: 0, total: 1 },
+ result: null,
+ error_message: null,
+ celery_task_id: "task-2",
+ attempts: 1,
+ max_attempts: 3,
+ })
+ )
+ }
+ if (url === "http://api.local/api/jobs/job-2") {
+ return new Promise(() => {})
+ }
+ return previous(url, options)
+ }
+ )
+
+ render(<KnowledgeBaseDetailContent collectionName="demo" />)
+
+ await waitFor(() => {
+ expect(screen.getByText("kb.detail.files.addSource")).toBeInTheDocument()
+ })
+
+ fireEvent.click(screen.getByText("kb.detail.files.addSource"))
+ fireEvent.click(screen.getByText("kb.dialog.tabs.web"))
+ fireEvent.change(screen.getByLabelText("kb.dialog.webImport.basic.startUrl *"), {
+ target: { value: "https://example.com/docs" },
+ })
+ fireEvent.click(screen.getByText("kb.index.startImport"))
+
+ await waitFor(() => {
+ expect(
+ apiRequestMock.mock.calls.some(
+ ([url]) => url === "http://api.local/api/jobs/job-2"
+ )
+ ).toBe(true)
+ }, { timeout: 5000 })
+ expect(screen.queryByText(/kb\.migration\.upgrading/)).toBeNull()
+ })
+
+ it("aborts the pending upload retry when the page unmounts", async () => {
+ const previous = apiRequestMock.getMockImplementation()!
+ let signal: AbortSignal | undefined
+ apiRequestMock.mockImplementation(
+ (
+ url: string,
+ options?: RequestInit,
+ _onUpgrading?: () => void,
+ captured?: AbortSignal
+ ) => {
+ if (url === "http://api.local/api/kb/ingest-web/jobs") {
+ signal = captured
+ return new Promise(() => {})
+ }
+ return previous(url, options)
+ }
+ )
+
+ const { unmount } = render(<KnowledgeBaseDetailContent collectionName="demo" />)
+
+ await waitFor(() => {
+ expect(screen.getByText("kb.detail.files.addSource")).toBeInTheDocument()
+ })
+
+ fireEvent.click(screen.getByText("kb.detail.files.addSource"))
+ fireEvent.click(screen.getByText("kb.dialog.tabs.web"))
+ fireEvent.change(screen.getByLabelText("kb.dialog.webImport.basic.startUrl *"), {
+ target: { value: "https://example.com/docs" },
+ })
+ fireEvent.click(screen.getByText("kb.index.startImport"))
+
+ await waitFor(() => expect(signal).toBeDefined())
+ unmount()
+ expect(signal?.aborted).toBe(true)
+ })
})
describe("KnowledgeBaseDetailContent config save", () => {
diff --git a/frontend/src/i18n/locales/en.ts b/frontend/src/i18n/locales/en.ts
index b0be4f09b..42ff6f352 100644
--- a/frontend/src/i18n/locales/en.ts
+++ b/frontend/src/i18n/locales/en.ts
@@ -2509,6 +2509,9 @@ Build when you need.`,
},
},
kb: {
+ migration: {
+ upgrading: "Knowledge base is upgrading. The upload retries automatically.",
+ },
ownership: {
label: "Ownership",
personal: "Personal",
diff --git a/frontend/src/i18n/locales/zh.ts b/frontend/src/i18n/locales/zh.ts
index 3ad06b35c..9f70dba94 100644
--- a/frontend/src/i18n/locales/zh.ts
+++ b/frontend/src/i18n/locales/zh.ts
@@ -2504,6 +2504,9 @@ const zh = {
},
},
kb: {
+ migration: {
+ upgrading: "知识库升级中,上传将自动重试。",
+ },
ownership: {
label: "归属",
personal: "个人",
diff --git a/frontend/src/lib/api-wrapper.test.ts b/frontend/src/lib/api-wrapper.test.ts
index e3f4c782b..67a4c1a85 100644
--- a/frontend/src/lib/api-wrapper.test.ts
+++ b/frontend/src/lib/api-wrapper.test.ts
@@ -5,8 +5,12 @@ import {
classifyUploadError,
getApiErrorMessage,
getUploadErrorMessage,
+ KB_COLLECTION_MIGRATING_CODE,
+ KB_MIGRATION_RETRY_DELAY_MS,
+ KB_MIGRATION_RETRY_LIMIT,
parseApiResponse,
refreshStoredAccessToken,
+ uploadWithCollectionMigrationRetry,
} from "@/lib/api-wrapper"
import {
AUTH_CACHE_KEY,
@@ -260,6 +264,283 @@ describe("api-wrapper API error helpers", () => {
})
})
+describe("api-wrapper collection migration retry", () => {
+ function migratingResponse(retryAfterSeconds?: number): Response {
+ const headers: Record<string, string> = { "Content-Type": "application/json" }
+ if (retryAfterSeconds !== undefined) headers["Retry-After"] = String(retryAfterSeconds)
+ return new Response(JSON.stringify({
+ detail: {
+ code: KB_COLLECTION_MIGRATING_CODE,
+ message: "Knowledge base is being upgraded",
+ },
+ }), { status: 409, headers })
+ }
+
+ function successResponse(): Response {
+ return new Response(JSON.stringify({ status: "success" }), {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ })
+ }
+
+ it("reads the structured detail message for uploads", async () => {
+ const response = migratingResponse()
+ const parsed = await parseApiResponse(response)
+
+ expect(getUploadErrorMessage(response, parsed, MESSAGES)).toBe(
+ "Knowledge base is being upgraded",
+ )
+ })
+
+ it("retries an upload refused while the collection is frozen", async () => {
+ vi.useFakeTimers()
+ try {
+ const fetchMock = vi.spyOn(globalThis, "fetch")
+ .mockResolvedValueOnce(migratingResponse(5))
+ .mockResolvedValueOnce(successResponse())
+ const onUpgrading = vi.fn()
+
+ const options = { method: "POST", body: new FormData() }
+ const pending = uploadWithCollectionMigrationRetry(
+ "/api/kb/ingest/jobs",
+ options,
+ onUpgrading,
+ )
+ await vi.advanceTimersByTimeAsync(0)
+ expect(fetchMock).toHaveBeenCalledTimes(1)
+ // The server's Retry-After paces the retry; the default gap is not used.
+ await vi.advanceTimersByTimeAsync(4_999)
+ expect(fetchMock).toHaveBeenCalledTimes(1)
+ await vi.advanceTimersByTimeAsync(1)
+
+ const response = await pending
+ expect(response.status).toBe(200)
+ expect(fetchMock).toHaveBeenCalledTimes(2)
+ expect(onUpgrading).toHaveBeenCalledTimes(1)
+ const [retriedUrl, retriedOptions] = fetchMock.mock.calls[1]
+ expect(retriedUrl).toBe("/api/kb/ingest/jobs")
+ expect(retriedOptions?.method).toBe("POST")
+ expect(retriedOptions?.body).toBe(options.body)
+ } finally {
+ vi.useRealTimers()
+ vi.restoreAllMocks()
+ }
+ })
+
+ it("gives up after the retry limit and returns the refusal", async () => {
+ vi.useFakeTimers()
+ try {
+ const fetchMock = vi.spyOn(globalThis, "fetch")
+ for (let i = 0; i < KB_MIGRATION_RETRY_LIMIT + 1; i++) {
+ fetchMock.mockResolvedValueOnce(migratingResponse())
+ }
+ const onUpgrading = vi.fn()
+
+ const pending = uploadWithCollectionMigrationRetry(
+ "/api/kb/ingest/jobs",
+ { method: "POST", body: new FormData() },
+ onUpgrading,
+ )
+ for (let i = 0; i < KB_MIGRATION_RETRY_LIMIT; i++) {
+ await vi.advanceTimersByTimeAsync(KB_MIGRATION_RETRY_DELAY_MS)
+ }
+
+ const response = await pending
+ expect(response.status).toBe(409)
+ expect(fetchMock).toHaveBeenCalledTimes(KB_MIGRATION_RETRY_LIMIT + 1)
+ expect(onUpgrading).toHaveBeenCalledTimes(KB_MIGRATION_RETRY_LIMIT)
+ } finally {
+ vi.useRealTimers()
+ vi.restoreAllMocks()
+ }
+ })
+
+ it("stops retrying when another sign-in replaces the initiating session", async () => {
+ vi.useFakeTimers()
+ try {
+ writeAuthCache({ id: "1", username: "alice" }, "token-a")
+ const fetchMock = vi.spyOn(globalThis, "fetch")
+ .mockResolvedValueOnce(migratingResponse(5))
+ const onUpgrading = vi.fn()
+
+ const pending = uploadWithCollectionMigrationRetry(
+ "/api/kb/ingest/jobs",
+ { method: "POST", body: new FormData() },
+ onUpgrading,
+ )
+ await vi.advanceTimersByTimeAsync(0)
+ expect(fetchMock).toHaveBeenCalledTimes(1)
+
+ writeAuthCache({ id: "2", username: "bob" }, "token-b")
+ await vi.advanceTimersByTimeAsync(5_000)
+
+ const response = await pending
+ expect(response.status).toBe(409)
+ // A's body must never be resent with B's session.
+ expect(fetchMock).toHaveBeenCalledTimes(1)
+ expect(onUpgrading).toHaveBeenCalledTimes(1)
+ } finally {
+ vi.useRealTimers()
+ vi.restoreAllMocks()
+ }
+ })
+
+ it("stops retrying when the signal aborts during the wait", async () => {
+ vi.useFakeTimers()
+ try {
+ writeAuthCache({ id: "1", username: "alice" }, "token-a")
+ const controller = new AbortController()
+ const fetchMock = vi
+ .spyOn(globalThis, "fetch")
+ .mockResolvedValueOnce(migratingResponse(15))
+ const onUpgrading = vi.fn()
+
+ const pending = uploadWithCollectionMigrationRetry(
+ "/api/kb/ingest/jobs",
+ { method: "POST", body: new FormData() },
+ onUpgrading,
+ controller.signal,
+ )
+ await vi.advanceTimersByTimeAsync(0)
+ expect(fetchMock).toHaveBeenCalledTimes(1)
+
+ controller.abort()
+ await vi.advanceTimersByTimeAsync(0)
+
+ const response = await pending
+ expect(response.status).toBe(409)
+ expect(fetchMock).toHaveBeenCalledTimes(1)
+ expect(onUpgrading).toHaveBeenCalledTimes(1)
+ } finally {
+ vi.useRealTimers()
+ vi.restoreAllMocks()
+ }
+ })
+
+ it("sends no request at all when the signal is already aborted", async () => {
+ const controller = new AbortController()
+ controller.abort()
+ const fetchMock = vi.spyOn(globalThis, "fetch")
+
+ const response = await uploadWithCollectionMigrationRetry(
+ "/api/kb/ingest/jobs",
+ { method: "POST", body: new FormData() },
+ undefined,
+ controller.signal,
+ )
+
+ expect(response.status).toBe(409)
+ expect(fetchMock).not.toHaveBeenCalled()
+ vi.restoreAllMocks()
+ })
+
+ it("sends the credential from the snapshot it checked, not a later storage read", async () => {
+ writeAuthCache({ id: "1", username: "alice" }, "token-a")
+ const cacheA = localStorage.getItem(AUTH_CACHE_KEY)
+ writeAuthCache({ id: "2", username: "bob" }, "token-b")
+ const cacheB = localStorage.getItem(AUTH_CACHE_KEY)
+ if (!cacheA || !cacheB) throw new Error("auth cache missing")
+ localStorage.setItem(AUTH_CACHE_KEY, cacheA)
+
+ const originalGetItem = window.localStorage.getItem.bind(window.localStorage)
+ let authReads = 0
+ vi.spyOn(window.localStorage, "getItem").mockImplementation((key: string) => {
+ if (key === AUTH_CACHE_KEY) {
+ authReads += 1
+ // The storage probe plus the snapshot read are one logical read; any
+ // further read would pick up the other tab's session.
+ return authReads <= 2 ? cacheA : cacheB
+ }
+ return originalGetItem(key)
+ })
+ const fetchMock = vi
+ .spyOn(globalThis, "fetch")
+ .mockResolvedValueOnce(successResponse())
+
+ const response = await uploadWithCollectionMigrationRetry(
+ "/api/kb/ingest/jobs",
+ { method: "POST", body: new FormData() },
+ )
+
+ expect(response.status).toBe(200)
+ expect(authReads).toBe(2)
+ const headers = fetchMock.mock.calls[0][1]?.headers as Record<string, string>
+ expect(headers.Authorization).toBe("Bearer token-a")
+ vi.restoreAllMocks()
+ })
+
+ it("keeps retrying after a credential refresh within the initiating session", async () => {
+ vi.useFakeTimers()
+ try {
+ writeAuthCache({ id: "1", username: "alice" }, "token-a")
+ const cached = readAuthCache()
+ if (!cached) throw new Error("auth cache missing")
+ const fetchMock = vi.spyOn(globalThis, "fetch")
+ .mockResolvedValueOnce(migratingResponse(5))
+ .mockResolvedValueOnce(successResponse())
+
+ const pending = uploadWithCollectionMigrationRetry(
+ "/api/kb/ingest/jobs",
+ { method: "POST", body: new FormData() },
+ )
+ await vi.advanceTimersByTimeAsync(0)
+ localStorage.setItem(AUTH_CACHE_KEY, JSON.stringify({
+ ...cached,
+ credentialRevision: cached.credentialRevision + 1,
+ token: "token-a-refreshed",
+ }))
+ await vi.advanceTimersByTimeAsync(5_000)
+
+ const response = await pending
+ expect(response.status).toBe(200)
+ expect(fetchMock).toHaveBeenCalledTimes(2)
+ } finally {
+ vi.useRealTimers()
+ vi.restoreAllMocks()
+ }
+ })
+
+ it("does not retry a 409 that is not a migration refusal", async () => {
+ const fetchMock = vi.spyOn(globalThis, "fetch")
+ .mockResolvedValueOnce(new Response(JSON.stringify({
+ detail: { code: "kb_admission_conflict", message: "Another upload is in progress" },
+ }), { status: 409, headers: { "Content-Type": "application/json" } }))
+ const onUpgrading = vi.fn()
+
+ const response = await uploadWithCollectionMigrationRetry(
+ "/api/kb/ingest/jobs",
+ { method: "POST", body: new FormData() },
+ onUpgrading,
+ )
+
+ expect(response.status).toBe(409)
+ expect(fetchMock).toHaveBeenCalledTimes(1)
+ expect(onUpgrading).not.toHaveBeenCalled()
+ vi.restoreAllMocks()
+ })
+
+ it("does not retry a 409 whose body is not parseable JSON", async () => {
+ const fetchMock = vi.spyOn(globalThis, "fetch").mockResolvedValueOnce(
+ new Response("<html>upstream</html>", {
+ status: 409,
+ headers: { "Content-Type": "text/html" },
+ }),
+ )
+ const onUpgrading = vi.fn()
+
+ const response = await uploadWithCollectionMigrationRetry(
+ "/api/kb/ingest/jobs",
+ { method: "POST", body: new FormData() },
+ onUpgrading,
+ )
+
+ expect(response.status).toBe(409)
+ expect(fetchMock).toHaveBeenCalledTimes(1)
+ expect(onUpgrading).not.toHaveBeenCalled()
+ vi.restoreAllMocks()
+ })
+})
+
describe("api-wrapper auth refresh", () => {
const user = { id: "1", username: "alice", email: null, is_admin: false }
diff --git a/frontend/src/lib/api-wrapper.ts b/frontend/src/lib/api-wrapper.ts
index 4b81f4774..7cfaf7f16 100644
--- a/frontend/src/lib/api-wrapper.ts
+++ b/frontend/src/lib/api-wrapper.ts
@@ -11,6 +11,7 @@ import {
type AuthMutationUnavailableReason,
clearAuthSessionIfCurrent,
compareAuthSession, compareCredentialSession,
+ compareAuthSessionLineage,
commitAuthSessionRefresh,
getAuthStorageAvailability,
isSafeAuthExpirySeconds,
@@ -148,8 +149,12 @@ function withBearer(options: RequestInit, token: string): RequestInit {
return { ...options, headers: { ...options.headers, Authorization: `Bearer ${token}` } }
}
/** A request has at most one post-401 replay, bound to an exact immutable credential snapshot. */
-export async function apiRequest(url: string, options: RequestInit = {}): Promise<Response> {
- const session = readAuthSessionSnapshot()
+export async function apiRequest(
+ url: string,
+ options: RequestInit = {},
+ capturedSession?: AuthSessionSnapshot,
+): Promise<Response> {
+ const session = capturedSession ?? readAuthSessionSnapshot()
if (!session.accessToken) return fetch(url, options)
const response = await fetchWithRetry(url, withBearer(options, session.accessToken))
if (response.status !== 401 || shouldSkipRefresh(url)) return response
@@ -208,11 +213,69 @@ export function getUploadErrorMessage(response: Response, parsed: ParsedApiRespo
.filter(Boolean)
if (validationMessages.length) return validationMessages.join("; ")
}
+ if (isJsonRecord(parsed.data) && isJsonRecord(parsed.data.detail) && typeof parsed.data.detail.message === "string" && parsed.data.detail.message.trim()) return parsed.data.detail.message
if (isJsonRecord(parsed.data) && typeof parsed.data.message === "string" && parsed.data.message.trim()) return parsed.data.message
if (response.status === 413) return messages.tooLarge
if (parsed.isHtml) return messages.proxy
return parsed.text?.trim() ? truncateUploadMessage(parsed.text) : messages.generic
}
+export const KB_COLLECTION_MIGRATING_CODE = "kb_collection_migrating"
+export const KB_MIGRATION_RETRY_LIMIT = 8
+export const KB_MIGRATION_RETRY_DELAY_MS = 10_000
+function isCollectionMigratingResponse(parsed: ParsedApiResponse): boolean {
+ return isJsonRecord(parsed.data) && isJsonRecord(parsed.data.detail)
+ && parsed.data.detail.code === KB_COLLECTION_MIGRATING_CODE
+}
+function isSameSessionLineage(captured: AuthSessionSnapshot, current: AuthSessionSnapshot): boolean {
+ return compareAuthSessionLineage(captured, current) === "same"
+}
+function waitForRetry(ms: number, signal?: AbortSignal): Promise<void> {
+ return new Promise(resolve => {
+ if (signal?.aborted) { resolve(); return }
+ let timer: ReturnType<typeof setTimeout>
+ const done = () => {
+ clearTimeout(timer)
+ signal?.removeEventListener("abort", done)
+ resolve()
+ }
+ timer = setTimeout(done, ms)
+ signal?.addEventListener("abort", done, { once: true })
+ })
+}
+/** Retries an upload refused while its collection is frozen, staying with the initiating user. */
+export async function uploadWithCollectionMigrationRetry(
+ url: string,
+ options: RequestInit,
+ onUpgrading?: () => void,
+ signal?: AbortSignal,
+): Promise<Response> {
+ let initiatingSession: AuthSessionSnapshot | null = null
+ let boundToSession = false
+ let refusal = new Response(null, { status: 409 })
+ for (let attempt = 0; ; attempt += 1) {
+ if (signal?.aborted) return refusal
+ // One read per attempt: the checked snapshot is the one whose credential is sent.
+ const attemptSession = readAuthSessionSnapshot()
+ if (initiatingSession === null) {
+ initiatingSession = attemptSession
+ boundToSession = Boolean(attemptSession.sessionId)
+ } else if (boundToSession && !isSameSessionLineage(initiatingSession, attemptSession)) {
+ // Another tab may have signed in as someone else while this one waited.
+ return refusal
+ }
+ const response = await apiRequest(url, options, attemptSession)
+ if (response.status !== 409 || attempt >= KB_MIGRATION_RETRY_LIMIT) return response
+ if (!isCollectionMigratingResponse(await parseApiResponse(response.clone()))) return response
+ refusal = response
+ const retryAfterSeconds = Number.parseInt(response.headers.get("Retry-After") ?? "", 10)
+ const delayMs = Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0
+ ? Math.min(retryAfterSeconds, 60) * 1000
+ : KB_MIGRATION_RETRY_DELAY_MS
+ if (signal?.aborted) return refusal
+ onUpgrading?.()
+ await waitForRetry(delayMs, signal)
+ }
+}
export interface ClassifiedUploadError {
errorCode: ClientErrorCode
message: string
diff --git a/frontend/src/lib/auth-cache.ts b/frontend/src/lib/auth-cache.ts
index 3719dffc9..049c9bad3 100644
--- a/frontend/src/lib/auth-cache.ts
+++ b/frontend/src/lib/auth-cache.ts
@@ -307,19 +307,30 @@ export function readAuthSessionSnapshot(): AuthSessionSnapshot {
const inspection = inspectAuthSession()
return inspection.status === "valid" ? inspection.projection.snapshot : EMPTY_SNAPSHOT
}
-export function compareAuthSession(captured: AuthSessionSnapshot, inspection = inspectAuthSession()): AuthSessionComparison {
- if (inspection.status !== "valid") return { status: inspection.status, projection: null }
- const { cache, snapshot } = inspection.projection
- if (!captured.sessionId || cache.sessionId !== captured.sessionId) return { status: "replaced", projection: null }
- if (cache.user.id !== captured.userId || cache.credentialRevision < (captured.credentialRevision ?? -1) || cache.profileRevision < (captured.profileRevision ?? -1)) {
- return { status: "invalid", projection: null }
+export type AuthSessionLineage = "same" | "replaced" | "invalid"
+/** Compare two snapshots without touching shared storage. */
+export function compareAuthSessionLineage(captured: AuthSessionSnapshot, current: AuthSessionSnapshot): AuthSessionLineage {
+ if (!captured.sessionId || current.sessionId !== captured.sessionId) return "replaced"
+ if (current.userId !== captured.userId || current.credentialRevision === null || current.profileRevision === null) {
+ return "invalid"
+ }
+ if (current.credentialRevision < (captured.credentialRevision ?? -1) || current.profileRevision < (captured.profileRevision ?? -1)) {
+ return "invalid"
}
- if (cache.credentialRevision === captured.credentialRevision && (cache.token !== captured.accessToken || cache.refreshToken !== captured.refreshToken)) {
- return { status: "invalid", projection: null }
+ if (current.credentialRevision === captured.credentialRevision && (current.accessToken !== captured.accessToken || current.refreshToken !== captured.refreshToken)) {
+ return "invalid"
}
- if (cache.profileRevision === captured.profileRevision && snapshot.profileFingerprint !== captured.profileFingerprint) {
- return { status: "invalid", projection: null }
+ if (current.profileRevision === captured.profileRevision && current.profileFingerprint !== captured.profileFingerprint) {
+ return "invalid"
}
+ return "same"
+}
+export function compareAuthSession(captured: AuthSessionSnapshot, inspection = inspectAuthSession()): AuthSessionComparison {
+ if (inspection.status !== "valid") return { status: inspection.status, projection: null }
+ const { cache, snapshot } = inspection.projection
+ const lineage = compareAuthSessionLineage(captured, snapshot)
+ if (lineage === "replaced") return { status: "replaced", projection: null }
+ if (lineage === "invalid") return { status: "invalid", projection: null }
const credentialExact = cache.credentialRevision === captured.credentialRevision && cache.token === captured.accessToken && cache.refreshToken === captured.refreshToken
const profileExact = cache.profileRevision === captured.profileRevision
if (credentialExact && profileExact) return { status: "exact", projection: inspection.projection }
diff --git a/src/xagent/config.py b/src/xagent/config.py
index df64b631a..80afd15d6 100644
--- a/src/xagent/config.py
+++ b/src/xagent/config.py
@@ -250,6 +250,8 @@ BACKGROUND_JOB_VISIBILITY_TIMEOUT_SECONDS = (
BACKGROUND_JOB_MAX_RETRIES = "XAGENT_BACKGROUND_JOB_MAX_RETRIES"
BACKGROUND_JOB_STALE_SECONDS = "XAGENT_BACKGROUND_JOB_STALE_SECONDS"
BACKGROUND_JOB_SWEEP_INTERVAL_SECONDS = "XAGENT_BACKGROUND_JOB_SWEEP_INTERVAL_SECONDS"
+KB_MIGRATION_DEFER_DELAY_SECONDS = "XAGENT_KB_MIGRATION_DEFER_DELAY_SECONDS"
+KB_MIGRATION_DEFER_BUDGET_SECONDS = "XAGENT_KB_MIGRATION_DEFER_BUDGET_SECONDS"
KB_MILVUS_CLEANUP_INTERVAL_SECONDS = "XAGENT_KB_MILVUS_CLEANUP_INTERVAL_SECONDS"
KB_MILVUS_CLEANUP_ENABLED = "XAGENT_KB_MILVUS_CLEANUP_ENABLED"
TASKLESS_UPLOAD_TTL_SECONDS = "XAGENT_TASKLESS_UPLOAD_TTL_SECONDS"
@@ -1868,6 +1870,16 @@ def get_background_job_sweep_interval_seconds() -> int:
)
+def get_kb_migration_defer_delay_seconds() -> int:
+ """Return how long an ingest job waits before retrying a frozen collection."""
+ return _get_positive_int_env(KB_MIGRATION_DEFER_DELAY_SECONDS, 15)
+
+
+def get_kb_migration_defer_budget_seconds() -> int:
+ """Return the longest an ingest job defers for a collection migration."""
+ return _get_positive_int_env(KB_MIGRATION_DEFER_BUDGET_SECONDS, 10 * 60)
+
+
def get_kb_milvus_cleanup_interval_seconds() -> int:
"""Return how often the leftover Milvus rows of a Milvus deployment are purged."""
return _get_positive_int_env(
diff --git a/src/xagent/web/api/kb.py b/src/xagent/web/api/kb.py
index b28609dd2..8d587114f 100644
--- a/src/xagent/web/api/kb.py
+++ b/src/xagent/web/api/kb.py
@@ -2690,6 +2690,7 @@ def handle_kb_exceptions(func: T) -> T:
"code": "kb_collection_migrating",
"message": e.message,
},
+ headers={"Retry-After": "15"},
) from e
except RollbackFailureError as e:
logger.error("KB rollback failure in %s: %s", func.__name__, e)
diff --git a/src/xagent/web/app.py b/src/xagent/web/app.py
index 2d66f35b6..eeff30d7c 100644
--- a/src/xagent/web/app.py
+++ b/src/xagent/web/app.py
@@ -1630,6 +1630,8 @@ app.add_middleware(
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
+ # The upload retry loop paces itself by this hint across origins.
+ expose_headers=["Retry-After"],
)
app.add_middleware(PublicWidgetCORSMiddleware)
app.add_middleware(SessionMiddleware, secret_key=get_session_secret(), same_site="lax")
diff --git a/src/xagent/web/jobs/exceptions.py b/src/xagent/web/jobs/exceptions.py
index 20223dbda..38f466897 100644
--- a/src/xagent/web/jobs/exceptions.py
+++ b/src/xagent/web/jobs/exceptions.py
@@ -14,3 +14,11 @@ class BackgroundJobHandlerError(RuntimeError):
super().__init__(message)
self.result = result
self.retryable = retryable
+
+
+class BackgroundJobDeferred(RuntimeError):
+ """Run the same job again later without spending its retry budget."""
+
+ def __init__(self, message: str, *, delay_seconds: int) -> None:
+ super().__init__(message)
+ self.delay_seconds = max(1, int(delay_seconds))
diff --git a/src/xagent/web/jobs/kb_tasks.py b/src/xagent/web/jobs/kb_tasks.py
index 92981465f..d9abb78f9 100644
--- a/src/xagent/web/jobs/kb_tasks.py
+++ b/src/xagent/web/jobs/kb_tasks.py
@@ -4,11 +4,16 @@ import asyncio
import hashlib
import logging
import shutil
+from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Literal
from sqlalchemy.orm import Session
+from ...config import (
+ get_kb_migration_defer_budget_seconds,
+ get_kb_migration_defer_delay_seconds,
+)
from ...core.tools.core.RAG_tools.core.exceptions import CollectionMigratingError
from ...core.tools.core.RAG_tools.core.schemas import (
IngestionConfig,
@@ -46,13 +51,20 @@ from ..services.kb_ingest_targets import is_latest_kb_ingest_generation
from ..services.kb_reference_protection import coordinate_ingest_job
from ..services.quota_hooks import get_standalone_gate
from ..tracking.standalone_usage import StandaloneUsageRefused, usage_scope
-from .exceptions import BackgroundJobHandlerError
+from .exceptions import BackgroundJobDeferred, BackgroundJobHandlerError
from .progress import BackgroundJobProgressManager
logger = logging.getLogger(__name__)
_SUPERSEDED_STAGED_INGEST_MESSAGE = "KB ingest job superseded by a newer upload"
+# created_at also covers queue wait and stale requeues, so it cannot anchor
+# the wait budget; a deferred job carries its first refusal here instead.
+_MIGRATION_DEFERRED_AT_KEY = "_migration_deferred_at"
+_MIGRATION_GIVE_UP_MESSAGE = (
+ "Gave up waiting for the knowledge base upgrade; please re-submit."
+)
+
class StagedDocumentIngestSuperseded(RuntimeError):
pass
@@ -67,6 +79,43 @@ def _payload_collection(db: Session, job: BackgroundJob) -> str | None:
return str(collection) if collection is not None else None
+def _defer_collection_migration(
+ db: Session, job: BackgroundJob, exc: CollectionMigratingError
+) -> BackgroundJobDeferred | None:
+ now = datetime.now(timezone.utc)
+ payload = dict(job.payload or {})
+ first_deferral = payload.get(_MIGRATION_DEFERRED_AT_KEY)
+ if first_deferral is None:
+ first_at = now
+ payload[_MIGRATION_DEFERRED_AT_KEY] = first_at.isoformat()
+ setattr(job, "payload", payload)
+ db.add(job)
+ db.commit()
+ else:
+ first_at = datetime.fromisoformat(str(first_deferral))
+ if first_at.tzinfo is None:
+ first_at = first_at.replace(tzinfo=timezone.utc)
+ remaining = get_kb_migration_defer_budget_seconds() - int(
+ (now - first_at).total_seconds()
+ )
+ if remaining <= 0:
+ return None
+ return BackgroundJobDeferred(
+ str(exc),
+ delay_seconds=min(get_kb_migration_defer_delay_seconds(), remaining),
+ )
+
+
+def _clear_migration_deferral(db: Session, job: BackgroundJob) -> None:
+ payload = dict(job.payload or {})
+ if _MIGRATION_DEFERRED_AT_KEY not in payload:
+ return
+ payload.pop(_MIGRATION_DEFERRED_AT_KEY)
+ setattr(job, "payload", payload)
+ db.add(job)
+ db.commit()
+
+
def _collection_existed_before(payload: dict[str, Any]) -> bool:
"""Read the flag stamped when the job was submitted.
@@ -252,13 +301,18 @@ def handle_kb_ingest_document(db: Session, job: BackgroundJob) -> dict[str, Any]
raise StagedDocumentIngestSuperseded(_SUPERSEDED_STAGED_INGEST_MESSAGE)
# Before model resolution, which rewrites the collection's metadata row, and
- # outside the try: a job refused here was never admitted and must not compensate.
+ # outside the try: a refused job never compensates, only a give-up cleans up.
try:
ensure_collection_writable(str(payload["collection"]))
- except CollectionMigratingError:
- if int(job.attempts or 0) >= int(job.max_attempts or 1):
- _cleanup_staged_document_input(payload)
- raise
+ except CollectionMigratingError as exc:
+ deferred = _defer_collection_migration(db, job, exc)
+ if deferred is not None:
+ raise deferred from exc
+ _cleanup_staged_document_input(payload)
+ raise BackgroundJobHandlerError(
+ _MIGRATION_GIVE_UP_MESSAGE, retryable=False
+ ) from exc
+ _clear_migration_deferral(db, job)
try:
model_pin, model_ids = _resolve_job_ingestion_models(payload, ingestion_config)
# Billed to the member who submitted the job (the job's owner), not
@@ -447,7 +501,16 @@ def handle_kb_ingest_web(db: Session, job: BackgroundJob) -> dict[str, Any]:
db_session.close()
update_job_progress(db, job, message="Crawling website")
- ensure_collection_writable(collection)
+ try:
+ ensure_collection_writable(collection)
+ except CollectionMigratingError as exc:
+ deferred = _defer_collection_migration(db, job, exc)
+ if deferred is not None:
+ raise deferred from exc
+ raise BackgroundJobHandlerError(
+ _MIGRATION_GIVE_UP_MESSAGE, retryable=False
+ ) from exc
+ _clear_migration_deferral(db, job)
try:
model_pin, model_ids = _resolve_job_ingestion_models(payload, ingestion_config)
with (
diff --git a/src/xagent/web/jobs/tasks.py b/src/xagent/web/jobs/tasks.py
index 80d48d9eb..7c28916fa 100644
--- a/src/xagent/web/jobs/tasks.py
+++ b/src/xagent/web/jobs/tasks.py
@@ -13,12 +13,13 @@ from ..models.background_job import (
)
from ..models.database import get_session_local, init_db
from ..services.background_jobs import (
+ QUEUE_DEFAULT,
mark_job_failed,
mark_job_running,
mark_job_succeeded,
)
from .celery_app import celery_app
-from .exceptions import BackgroundJobHandlerError
+from .exceptions import BackgroundJobDeferred, BackgroundJobHandlerError
logger = logging.getLogger(__name__)
@@ -129,6 +130,36 @@ def execute_background_job(self: Any, job_id: str) -> dict[str, Any]:
mark_job_running(db, job)
try:
result = _execute_job_handler(db, job)
+ except BackgroundJobDeferred as exc:
+ db.refresh(job)
+ # A deferral is not a failed attempt; give the attempt back so the
+ # job's real retry budget survives a migration window.
+ setattr(job, "attempts", max(0, int(job.attempts or 0) - 1))
+ setattr(job, "status", BackgroundJobStatus.ENQUEUED.value)
+ setattr(job, "error_message", None)
+ setattr(
+ job,
+ "progress",
+ {"message": str(exc), "completed": 0, "total": 1},
+ )
+ db.add(job)
+ db.commit()
+ db.refresh(job)
+ if not celery_app.conf.task_always_eager:
+ try:
+ async_result = execute_background_job.apply_async(
+ args=[job.id],
+ queue=str(job.queue or QUEUE_DEFAULT),
+ countdown=exc.delay_seconds,
+ )
+ except Exception: # noqa: BLE001
+ # The stale sweep is the safety net; the row stays recoverable.
+ logger.exception("Failed to reschedule deferred job %s", job.id)
+ else:
+ setattr(job, "celery_task_id", async_result.id)
+ db.add(job)
+ db.commit()
+ return {"status": "deferred"}
except BackgroundJobHandlerError as exc:
db.refresh(job)
if exc.retryable and int(job.attempts or 0) < int(job.max_attempts or 1):
diff --git a/src/xagent/web/services/kb_reference_protection.py b/src/xagent/web/services/kb_reference_protection.py
index 72f88c1f0..417f7e784 100644
--- a/src/xagent/web/services/kb_reference_protection.py
+++ b/src/xagent/web/services/kb_reference_protection.py
@@ -364,7 +364,7 @@ def guard_upload_compensation(operation: _Operation) -> _Operation:
def coordinate_ingest_job(operation: _Operation) -> _Operation:
@wraps(operation)
def ingest(db: Session, job: Any) -> Any:
- from ..jobs.exceptions import BackgroundJobHandlerError
+ from ..jobs.exceptions import BackgroundJobDeferred, BackgroundJobHandlerError
from .kb_ingest_targets import release_kb_ingest_target_generation
payload = dict(job.payload or {})
@@ -382,6 +382,9 @@ def coordinate_ingest_job(operation: _Operation) -> _Operation:
result = operation(db, job)
release = True
return result
+ except BackgroundJobDeferred:
+ # The job will run again: do not release its reservation.
+ raise
except BackgroundJobHandlerError as exc:
release = final_attempt or not exc.retryable
raise
diff --git a/tests/core/test_config.py b/tests/core/test_config.py
index 3760a8d3c..cd5352529 100644
--- a/tests/core/test_config.py
+++ b/tests/core/test_config.py
@@ -59,6 +59,8 @@ from xagent.config import (
HOT_PATH_CACHE_TTL_SECONDS,
HOT_PATH_TASK_CACHE_TTL_SECONDS,
KB_COLLECTIONS_TIMEOUT_SECONDS,
+ KB_MIGRATION_DEFER_BUDGET_SECONDS,
+ KB_MIGRATION_DEFER_DELAY_SECONDS,
KB_MILVUS_CLEANUP_ENABLED,
KB_MILVUS_CLEANUP_INTERVAL_SECONDS,
KB_SEARCH_TIMEOUT_SECONDS,
@@ -175,6 +177,8 @@ from xagent.config import (
get_hot_path_cache_ttl_seconds,
get_hot_path_task_cache_ttl_seconds,
get_kb_collections_timeout_seconds,
+ get_kb_migration_defer_budget_seconds,
+ get_kb_migration_defer_delay_seconds,
get_kb_milvus_cleanup_enabled,
get_kb_milvus_cleanup_interval_seconds,
get_kb_search_timeout_seconds,
@@ -523,6 +527,14 @@ class TestEnvironmentVariableConstants:
BACKGROUND_JOB_SWEEP_INTERVAL_SECONDS
== "XAGENT_BACKGROUND_JOB_SWEEP_INTERVAL_SECONDS"
)
+ assert (
+ KB_MIGRATION_DEFER_DELAY_SECONDS
+ == "XAGENT_KB_MIGRATION_DEFER_DELAY_SECONDS"
+ )
+ assert (
+ KB_MIGRATION_DEFER_BUDGET_SECONDS
+ == "XAGENT_KB_MIGRATION_DEFER_BUDGET_SECONDS"
+ )
assert TRIGGER_DISPATCHER_ENABLED == "XAGENT_TRIGGER_DISPATCHER_ENABLED"
assert (
TRIGGER_DISPATCHER_INTERVAL_SECONDS
@@ -881,6 +893,18 @@ class TestCeleryBackgroundJobConfig:
assert get_background_job_stale_seconds() == 7200
assert get_background_job_sweep_interval_seconds() == 300
+ def test_kb_migration_defer_defaults(self, monkeypatch):
+ monkeypatch.delenv(KB_MIGRATION_DEFER_DELAY_SECONDS, raising=False)
+ monkeypatch.delenv(KB_MIGRATION_DEFER_BUDGET_SECONDS, raising=False)
+ assert get_kb_migration_defer_delay_seconds() == 15
+ assert get_kb_migration_defer_budget_seconds() == 600
+
+ def test_kb_migration_defer_env_overrides(self, monkeypatch):
+ monkeypatch.setenv(KB_MIGRATION_DEFER_DELAY_SECONDS, "20")
+ monkeypatch.setenv(KB_MIGRATION_DEFER_BUDGET_SECONDS, "1200")
+ assert get_kb_migration_defer_delay_seconds() == 20
+ assert get_kb_migration_defer_budget_seconds() == 1200
+
def test_kb_milvus_cleanup_interval_default(self, monkeypatch):
monkeypatch.delenv(KB_MILVUS_CLEANUP_INTERVAL_SECONDS, raising=False)
assert get_kb_milvus_cleanup_interval_seconds() == 3600
diff --git a/tests/web/api/test_kb_cloud_rollback_contract.py b/tests/web/api/test_kb_cloud_rollback_contract.py
index dfb97b2e4..12aebd09f 100644
--- a/tests/web/api/test_kb_cloud_rollback_contract.py
+++ b/tests/web/api/test_kb_cloud_rollback_contract.py
@@ -10,6 +10,7 @@ from __future__ import annotations
import asyncio
import inspect
from contextlib import ExitStack
+from datetime import datetime, timedelta, timezone
from pathlib import Path
from types import SimpleNamespace
from typing import Any, Optional
@@ -577,20 +578,27 @@ def test_staged_job_rollback_runs_inside_admission_after_mid_job_freeze(
assert str(err) == "ingestion failed"
[email protected]("job_type", ["document", "web"])
+ ("job_type", "give_up"),
+ [("document", False), ("document", True), ("web", False), ("web", True)],
+)
def test_frozen_job_refuses_before_model_resolution(
- test_env, tmp_path, monkeypatch, job_type
+ test_env, tmp_path, monkeypatch, job_type, give_up
) -> None:
- from xagent.core.tools.core.RAG_tools.core.exceptions import (
- CollectionMigratingError,
- )
from xagent.core.tools.core.RAG_tools.management.migration_state import (
FROZEN,
CollectionMigrationState,
clear_migration_state,
write_migration_state,
)
- from xagent.web.jobs.kb_tasks import handle_kb_ingest_web
+ from xagent.web.jobs.exceptions import (
+ BackgroundJobDeferred,
+ BackgroundJobHandlerError,
+ )
+ from xagent.web.jobs.kb_tasks import (
+ _MIGRATION_DEFERRED_AT_KEY,
+ handle_kb_ingest_web,
+ )
resolved: list[str] = []
monkeypatch.setattr(
@@ -598,6 +606,9 @@ def test_frozen_job_refuses_before_model_resolution(
lambda *a, **k: resolved.append("called"),
)
_, _, user, session_local = test_env
+ staged = tmp_path / "stage" / "doc.txt"
+ staged.parent.mkdir(parents=True)
+ staged.write_text("staged", encoding="utf-8")
payload: dict[str, Any] = {
"collection": "existing-kb",
"user_id": int(user.id),
@@ -605,8 +616,13 @@ def test_frozen_job_refuses_before_model_resolution(
"ingestion_config": IngestionConfig().model_dump(mode="json"),
"collection_existed_before": True,
}
+ if give_up:
+ payload[_MIGRATION_DEFERRED_AT_KEY] = (
+ datetime.now(timezone.utc) - timedelta(minutes=11)
+ ).isoformat()
if job_type == "document":
handler, kind = handle_kb_ingest_document, BackgroundJobType.KB_INGEST_DOCUMENT
+ payload["source_path"] = str(staged)
payload["target_path"] = str(tmp_path / "doc.txt")
else:
handler, kind = handle_kb_ingest_web, BackgroundJobType.KB_INGEST_WEB
@@ -617,24 +633,27 @@ def test_frozen_job_refuses_before_model_resolution(
job = create_background_job(
db, user_id=int(user.id), job_type=kind, payload=payload
)
- with pytest.raises(CollectionMigratingError):
+ expected_error = BackgroundJobHandlerError if give_up else BackgroundJobDeferred
+ with pytest.raises(expected_error) as raised:
handler(db, job)
finally:
clear_migration_state("existing-kb")
db.close()
assert resolved == []
+ if give_up:
+ assert raised.value.retryable is False
+ if job_type == "document":
+ # Deferred keeps the staged copy; only giving up removes it.
+ assert staged.exists() is not give_up
@pytest.mark.parametrize(
- ("frozen", "final"), [(True, True), (True, False), (False, True)]
+ ("frozen", "give_up"), [(True, False), (True, True), (False, False)]
)
def test_final_attempt_refusal_does_not_compensate_an_existing_document(
- test_env, tmp_path, monkeypatch, frozen, final
+ test_env, tmp_path, monkeypatch, frozen, give_up
) -> None:
- from xagent.core.tools.core.RAG_tools.core.exceptions import (
- CollectionMigratingError,
- )
from xagent.core.tools.core.RAG_tools.management.migration_state import (
FROZEN,
CollectionMigrationState,
@@ -648,6 +667,14 @@ def test_final_attempt_refusal_does_not_compensate_an_existing_document(
from xagent.core.tools.core.RAG_tools.utils.string_utils import (
generate_deterministic_doc_id,
)
+ from xagent.web.jobs.exceptions import (
+ BackgroundJobDeferred,
+ BackgroundJobHandlerError,
+ )
+ from xagent.web.jobs.kb_tasks import (
+ _MIGRATION_DEFERRED_AT_KEY,
+ _MIGRATION_GIVE_UP_MESSAGE,
+ )
_, _, user, session_local = test_env
file_id = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"
@@ -668,42 +695,58 @@ def test_final_attempt_refusal_does_not_compensate_an_existing_document(
db = session_local()
if frozen:
write_migration_state("existing-kb", CollectionMigrationState(state=FROZEN))
+ payload: dict[str, Any] = {
+ "collection": "existing-kb",
+ "source_path": str(staged),
+ "target_path": str(tmp_path / "canonical" / "doc.txt"),
+ "file_id": file_id,
+ "filename": "doc.txt",
+ "user_id": int(user.id),
+ "is_admin": False,
+ "ingestion_config": IngestionConfig().model_dump(mode="json"),
+ "collection_existed_before": True,
+ "document_existed_before": True,
+ }
+ if give_up:
+ payload[_MIGRATION_DEFERRED_AT_KEY] = (
+ datetime.now(timezone.utc) - timedelta(minutes=11)
+ ).isoformat()
try:
job = create_background_job(
db,
user_id=int(user.id),
job_type=BackgroundJobType.KB_INGEST_DOCUMENT,
- payload={
- "collection": "existing-kb",
- "source_path": str(staged),
- "target_path": str(tmp_path / "canonical" / "doc.txt"),
- "file_id": file_id,
- "filename": "doc.txt",
- "user_id": int(user.id),
- "is_admin": False,
- "ingestion_config": IngestionConfig().model_dump(mode="json"),
- "collection_existed_before": True,
- "document_existed_before": True,
- },
+ payload=payload,
max_attempts=2,
)
- job.attempts = 2 if final else 1
+ # Even the last allowed attempt is deferred while the budget holds: a
+ # freeze is not a failed attempt.
+ job.attempts = 2
+ if frozen and not give_up:
+ expected_error: type[Exception] = BackgroundJobDeferred
+ elif frozen:
+ expected_error = BackgroundJobHandlerError
+ else:
+ expected_error = RuntimeError
with (
patch(
"xagent.web.api.kb._rollback_failed_cloud_ingestion",
side_effect=_rollback,
),
- pytest.raises(CollectionMigratingError if frozen else RuntimeError),
+ pytest.raises(expected_error) as raised,
):
handle_kb_ingest_document(db, job)
finally:
clear_migration_state("existing-kb")
db.close()
+ if frozen and give_up:
+ assert raised.value.retryable is False
+ assert str(raised.value) == _MIGRATION_GIVE_UP_MESSAGE
# The unfrozen run is the positive control: this payload does reach compensation.
assert compensated == ([] if frozen else ["existing-kb"])
- # A retry still needs the staged copy; the last attempt must not leak it.
- assert staged.exists() is not final
+ # A deferred job still needs the staged copy; giving up or failing does not.
+ assert staged.exists() is (frozen and not give_up)
if frozen:
rows = load_ingestion_status("existing-kb", doc_id, user_id=user.id)
assert [r["status"] for r in rows] == ["success"]
diff --git a/tests/web/api/test_kb_exception_decorator.py b/tests/web/api/test_kb_exception_decorator.py
index 889348b54..fe5adae0f 100644
--- a/tests/web/api/test_kb_exception_decorator.py
+++ b/tests/web/api/test_kb_exception_decorator.py
@@ -419,6 +419,7 @@ async def test_handle_kb_exceptions_maps_migration_freeze_to_409() -> None:
assert exc_info.value.status_code == 409
assert exc_info.value.detail == _MIGRATION_DETAIL
+ assert exc_info.value.headers == {"Retry-After": "15"}
@pytest.mark.asyncio
@@ -454,6 +455,7 @@ async def test_ingest_route_maps_migration_freeze_to_409() -> None:
assert exc_info.value.status_code == 409
assert exc_info.value.detail == _MIGRATION_DETAIL
+ assert exc_info.value.headers == {"Retry-After": "15"}
@pytest.mark.asyncio
@@ -484,6 +486,7 @@ async def test_ingest_cloud_route_maps_migration_freeze_to_409() -> None:
assert exc_info.value.status_code == 409
assert exc_info.value.detail == _MIGRATION_DETAIL
+ assert exc_info.value.headers == {"Retry-After": "15"}
@pytest.mark.asyncio
@@ -514,6 +517,7 @@ async def test_ingest_web_route_maps_migration_freeze_to_409() -> None:
assert exc_info.value.status_code == 409
assert exc_info.value.detail == _MIGRATION_DETAIL
+ assert exc_info.value.headers == {"Retry-After": "15"}
@pytest.mark.asyncio
@@ -553,6 +557,7 @@ async def test_create_ingest_job_maps_migration_freeze_to_409() -> None:
assert exc_info.value.status_code == 409
assert exc_info.value.detail == _MIGRATION_DETAIL
+ assert exc_info.value.headers == {"Retry-After": "15"}
class _FakeRollbackFacade:
diff --git a/tests/web/test_api_cors.py b/tests/web/test_api_cors.py
new file mode 100644
index 000000000..ca8308ba7
--- /dev/null
+++ b/tests/web/test_api_cors.py
@@ -0,0 +1,16 @@
+"""Cross-origin clients must be able to read the retry hint."""
+
+from fastapi.testclient import TestClient
+
+from xagent.config import get_cors_origins
+from xagent.web.app import app
+
+
+def test_allowed_origin_exposes_retry_after() -> None:
+ origin = get_cors_origins()[0]
+
+ response = TestClient(app).get("/api/deployment-config", headers={"Origin": origin})
+
+ assert response.status_code == 200
+ assert response.headers["access-control-allow-origin"] == origin
+ assert "Retry-After" in response.headers["access-control-expose-headers"]
diff --git a/tests/web/test_background_jobs.py b/tests/web/test_background_jobs.py
index 94027cd35..ef40d4d6e 100644
--- a/tests/web/test_background_jobs.py
+++ b/tests/web/test_background_jobs.py
@@ -2120,3 +2120,590 @@ def test_kb_document_job_publishes_the_config_end_to_end(tmp_path, monkeypatch):
assert '"chunk_size":1234' in saved_config
finally:
db.close()
+
+
+def _freeze_collection(monkeypatch) -> None:
+ from xagent.core.tools.core.RAG_tools.core.exceptions import (
+ CollectionMigratingError,
+ )
+
+ def refuse(_collection: str) -> None:
+ raise CollectionMigratingError("Knowledge base 'kb' is being upgraded")
+
+ monkeypatch.setattr("xagent.web.jobs.kb_tasks.ensure_collection_writable", refuse)
+
+
+def _backdate_created_at(db, job, *, seconds: int) -> None:
+ db.query(BackgroundJob).filter(BackgroundJob.id == job.id).update(
+ {"created_at": datetime.now(timezone.utc) - timedelta(seconds=seconds)},
+ synchronize_session=False,
+ )
+ db.commit()
+ db.expire_all()
+
+
+def test_kb_document_job_defers_while_collection_is_migrating(tmp_path, monkeypatch):
+ from xagent.web.jobs.exceptions import BackgroundJobDeferred
+ from xagent.web.jobs.kb_tasks import (
+ _MIGRATION_DEFERRED_AT_KEY,
+ handle_kb_ingest_document,
+ )
+ from xagent.web.services.kb_ingest_targets import admit_kb_ingest_target
+
+ monkeypatch.setenv(CELERY_ENABLED, "false")
+ monkeypatch.delenv(CELERY_BROKER_URL, raising=False)
+ _freeze_collection(monkeypatch)
+
+ SessionLocal = _init_test_db(tmp_path / "kb-migration-defer.db")
+ db = SessionLocal()
+ try:
+ user = _create_user(db, username="kb-migration-defer")
+ staged_file = tmp_path / "stage" / "doc.txt"
+ target_file = tmp_path / "canonical" / "doc.txt"
+ staged_file.parent.mkdir(parents=True)
+ staged_file.write_text("staged content", encoding="utf-8")
+ file_id = "22222222-2222-4222-8222-222222222222"
+ generation_id = "33333333-3333-4333-8333-333333333333"
+ job = create_background_job(
+ db,
+ user_id=int(user.id),
+ job_type=BackgroundJobType.KB_INGEST_DOCUMENT,
+ payload={
+ "collection": "kb",
+ "source_path": str(staged_file),
+ "target_path": str(target_file),
+ "file_id": file_id,
+ "generation_id": generation_id,
+ "filename": "doc.txt",
+ "mime_type": "text/plain",
+ "file_size": staged_file.stat().st_size,
+ "user_id": int(user.id),
+ "is_admin": False,
+ "ingestion_config": IngestionConfig().model_dump(mode="json"),
+ "collection_existed_before": True,
+ },
+ )
+ admit_kb_ingest_target(
+ db,
+ user_id=int(user.id),
+ collection="kb",
+ target_path=str(target_file),
+ file_id=file_id,
+ generation_id=generation_id,
+ job_id=str(job.id),
+ file_sha256="0" * 64,
+ )
+ # Even on the last allowed attempt a deferral keeps the reservation:
+ # the job will run again, and the freeze is not the job's failure.
+ setattr(job, "attempts", int(job.max_attempts or 3))
+ db.commit()
+ db.refresh(job)
+ # A stale requeue can run a job hours after it was created; the wait
+ # budget must still start when the freeze is first hit, not at creation.
+ _backdate_created_at(db, job, seconds=2 * 3600)
+
+ releases: list[dict] = []
+ monkeypatch.setattr(
+ "xagent.web.services.kb_ingest_targets.release_kb_ingest_target_generation",
+ lambda *args, **kwargs: releases.append(kwargs) or True,
+ )
+ ingest_calls: list[dict] = []
+ monkeypatch.setattr(
+ "xagent.web.jobs.kb_tasks.run_document_ingestion",
+ lambda **kwargs: ingest_calls.append(kwargs),
+ )
+ cleaned_staging: list[object] = []
+ monkeypatch.setattr(
+ "xagent.web.api.kb._cleanup_background_ingest_staging_file",
+ lambda path: cleaned_staging.append(path),
+ )
+
+ with pytest.raises(BackgroundJobDeferred) as raised:
+ handle_kb_ingest_document(db, job)
+
+ assert raised.value.delay_seconds == 15
+ # A refused job reruns later: its staged input and reservation stay.
+ assert not cleaned_staging
+ assert not releases
+ assert not ingest_calls
+ assert staged_file.read_text(encoding="utf-8") == "staged content"
+ assert not target_file.exists()
+ db.refresh(job)
+ assert job.payload[_MIGRATION_DEFERRED_AT_KEY]
+ finally:
+ db.close()
+
+
+def test_kb_document_job_gives_up_when_migration_budget_is_spent(tmp_path, monkeypatch):
+ from xagent.web.jobs.exceptions import BackgroundJobHandlerError
+ from xagent.web.jobs.kb_tasks import (
+ _MIGRATION_DEFERRED_AT_KEY,
+ _MIGRATION_GIVE_UP_MESSAGE,
+ handle_kb_ingest_document,
+ )
+ from xagent.web.services.kb_ingest_targets import admit_kb_ingest_target
+
+ monkeypatch.setenv(CELERY_ENABLED, "false")
+ monkeypatch.delenv(CELERY_BROKER_URL, raising=False)
+ _freeze_collection(monkeypatch)
+
+ SessionLocal = _init_test_db(tmp_path / "kb-migration-give-up.db")
+ db = SessionLocal()
+ try:
+ user = _create_user(db, username="kb-migration-give-up")
+ staged_file = tmp_path / "stage" / "doc.txt"
+ target_file = tmp_path / "canonical" / "doc.txt"
+ staged_file.parent.mkdir(parents=True)
+ staged_file.write_text("staged content", encoding="utf-8")
+ file_id = "44444444-4444-4444-8444-444444444444"
+ generation_id = "55555555-5555-4555-8555-555555555555"
+ job = create_background_job(
+ db,
+ user_id=int(user.id),
+ job_type=BackgroundJobType.KB_INGEST_DOCUMENT,
+ payload={
+ "collection": "kb",
+ "source_path": str(staged_file),
+ "target_path": str(target_file),
+ "file_id": file_id,
+ "generation_id": generation_id,
+ "filename": "doc.txt",
+ "mime_type": "text/plain",
+ "file_size": staged_file.stat().st_size,
+ "user_id": int(user.id),
+ "is_admin": False,
+ "ingestion_config": IngestionConfig().model_dump(mode="json"),
+ "collection_existed_before": True,
+ _MIGRATION_DEFERRED_AT_KEY: (
+ datetime.now(timezone.utc) - timedelta(minutes=11)
+ ).isoformat(),
+ },
+ )
+ admit_kb_ingest_target(
+ db,
+ user_id=int(user.id),
+ collection="kb",
+ target_path=str(target_file),
+ file_id=file_id,
+ generation_id=generation_id,
+ job_id=str(job.id),
+ file_sha256="0" * 64,
+ )
+ db.commit()
+
+ releases: list[dict] = []
+ monkeypatch.setattr(
+ "xagent.web.services.kb_ingest_targets.release_kb_ingest_target_generation",
+ lambda *args, **kwargs: releases.append(kwargs) or True,
+ )
+
+ with pytest.raises(BackgroundJobHandlerError) as raised:
+ handle_kb_ingest_document(db, job)
+
+ assert raised.value.retryable is False
+ assert str(raised.value) == _MIGRATION_GIVE_UP_MESSAGE
+ # Giving up follows the terminal path: staged input removed, reservation
+ # released so a later upload can take the target.
+ assert not staged_file.exists()
+ assert [call["generation_id"] for call in releases] == [generation_id]
+ finally:
+ db.close()
+
+
+def test_kb_web_job_defers_while_collection_is_migrating(tmp_path, monkeypatch):
+ from xagent.web.jobs.exceptions import BackgroundJobDeferred
+ from xagent.web.jobs.kb_tasks import handle_kb_ingest_web
+
+ monkeypatch.setenv(CELERY_ENABLED, "false")
+ monkeypatch.delenv(CELERY_BROKER_URL, raising=False)
+ _freeze_collection(monkeypatch)
+
+ SessionLocal = _init_test_db(tmp_path / "kb-web-migration-defer.db")
+ db = SessionLocal()
+ try:
+ user = _create_user(db, username="kb-web-migration-defer")
+ job = create_background_job(
+ db,
+ user_id=int(user.id),
+ job_type=BackgroundJobType.KB_INGEST_WEB,
+ payload={
+ "collection": "web-kb",
+ "crawl_config": WebCrawlConfig(
+ start_url="https://example.com"
+ ).model_dump(mode="json"),
+ "ingestion_config": IngestionConfig().model_dump(mode="json"),
+ "user_id": int(user.id),
+ "is_admin": False,
+ "collection_existed_before": False,
+ },
+ )
+
+ with pytest.raises(BackgroundJobDeferred) as raised:
+ handle_kb_ingest_web(db, job)
+
+ assert raised.value.delay_seconds == 15
+ finally:
+ db.close()
+
+
+def test_execute_background_job_reschedules_a_deferred_job(tmp_path, monkeypatch):
+ from xagent.web.jobs import tasks as tasks_module
+ from xagent.web.jobs.exceptions import BackgroundJobDeferred
+
+ monkeypatch.setenv(CELERY_ENABLED, "false")
+ monkeypatch.delenv(CELERY_BROKER_URL, raising=False)
+ job_type = "test.deferred_registered"
+
+ SessionLocal = _init_test_db(tmp_path / "jobs-deferred.db")
+ db = SessionLocal()
+ try:
+ user = _create_user(db, "registered-deferred")
+ job = create_background_job(
+ db, user_id=int(user.id), job_type=job_type, payload={}, max_attempts=3
+ )
+ job_id = str(job.id)
+ finally:
+ db.close()
+
+ def deferring_handler(_session, _job):
+ raise BackgroundJobDeferred(
+ "Knowledge base is being upgraded", delay_seconds=15
+ )
+
+ tasks_module.register_background_job_handler(job_type, deferring_handler)
+ rescheduled: list[dict] = []
+
+ def fake_apply_async(*, args, queue, countdown):
+ rescheduled.append({"args": args, "queue": queue, "countdown": countdown})
+ return MagicMock(id="deferred-task-id")
+
+ monkeypatch.setattr(
+ tasks_module.execute_background_job, "apply_async", fake_apply_async
+ )
+
+ try:
+ outcome = tasks_module.execute_background_job.apply(args=[job_id], throw=False)
+ finally:
+ tasks_module._EXTRA_HANDLERS.pop(job_type, None)
+
+ assert outcome.result == {"status": "deferred"}
+ assert rescheduled == [{"args": [job_id], "queue": "default", "countdown": 15}]
+
+ verify_db = SessionLocal()
+ try:
+ refreshed = (
+ verify_db.query(BackgroundJob).filter(BackgroundJob.id == job_id).first()
+ )
+ assert refreshed is not None
+ assert refreshed.status == BackgroundJobStatus.ENQUEUED.value
+ # The deferral must not spend the job's retry budget.
+ assert refreshed.attempts == 0
+ assert refreshed.error_message is None
+ assert "upgraded" in refreshed.progress["message"]
+ assert refreshed.celery_task_id == "deferred-task-id"
+ finally:
+ verify_db.close()
+
+
+def test_deferred_job_keeps_its_retry_budget_for_real_failures(tmp_path, monkeypatch):
+ """A deferral returns the attempt, so a later real failure still retries."""
+ from xagent.web.jobs import tasks as tasks_module
+ from xagent.web.jobs.exceptions import (
+ BackgroundJobDeferred,
+ BackgroundJobHandlerError,
+ )
+
+ monkeypatch.setenv(CELERY_ENABLED, "false")
+ monkeypatch.delenv(CELERY_BROKER_URL, raising=False)
+ job_type = "test.deferred_then_failing"
+
+ SessionLocal = _init_test_db(tmp_path / "jobs-deferred-budget.db")
+ db = SessionLocal()
+ try:
+ user = _create_user(db, "deferred-budget")
+ job = create_background_job(
+ db, user_id=int(user.id), job_type=job_type, payload={}, max_attempts=2
+ )
+ job_id = str(job.id)
+ finally:
+ db.close()
+
+ runs: list[int] = []
+
+ def handler(_session, _job):
+ runs.append(len(runs) + 1)
+ if len(runs) == 1:
+ raise BackgroundJobDeferred(
+ "Knowledge base is being upgraded", delay_seconds=15
+ )
+ raise BackgroundJobHandlerError("ingest failed", retryable=True)
+
+ tasks_module.register_background_job_handler(job_type, handler)
+ monkeypatch.setattr(
+ tasks_module.execute_background_job,
+ "apply_async",
+ lambda **_kwargs: MagicMock(id="deferred-task-id"),
+ )
+ retry_calls: list[dict] = []
+
+ def fake_retry(*_args, **kwargs):
+ retry_calls.append(kwargs)
+ return RuntimeError("retry requested")
+
+ monkeypatch.setattr(tasks_module.execute_background_job, "retry", fake_retry)
+
+ try:
+ tasks_module.execute_background_job.apply(args=[job_id], throw=False)
+ tasks_module.execute_background_job.apply(args=[job_id], throw=False)
+ finally:
+ tasks_module._EXTRA_HANDLERS.pop(job_type, None)
+
+ assert len(runs) == 2
+ assert len(retry_calls) == 1
+ assert retry_calls[0]["exc"].retryable is True
+
+ verify_db = SessionLocal()
+ try:
+ refreshed = (
+ verify_db.query(BackgroundJob).filter(BackgroundJob.id == job_id).first()
+ )
+ assert refreshed is not None
+ # The deferral spent nothing: the failed rerun is attempt 1 of 2.
+ assert refreshed.attempts == 1
+ assert refreshed.status == BackgroundJobStatus.ENQUEUED.value
+ finally:
+ verify_db.close()
+
+
+def test_deferred_job_survives_a_failed_reschedule(tmp_path, monkeypatch):
+ from xagent.web.jobs import tasks as tasks_module
+ from xagent.web.jobs.exceptions import BackgroundJobDeferred
+
+ monkeypatch.setenv(CELERY_ENABLED, "false")
+ monkeypatch.delenv(CELERY_BROKER_URL, raising=False)
+ job_type = "test.deferred_reschedule_failure"
+
+ SessionLocal = _init_test_db(tmp_path / "jobs-deferred-broker-down.db")
+ db = SessionLocal()
+ try:
+ user = _create_user(db, "deferred-broker-down")
+ job = create_background_job(
+ db, user_id=int(user.id), job_type=job_type, payload={}, max_attempts=3
+ )
+ job_id = str(job.id)
+ finally:
+ db.close()
+
+ def handler(_session, _job):
+ raise BackgroundJobDeferred(
+ "Knowledge base is being upgraded", delay_seconds=15
+ )
+
+ def failing_apply_async(**_kwargs):
+ raise RuntimeError("broker down")
+
+ tasks_module.register_background_job_handler(job_type, handler)
+ monkeypatch.setattr(
+ tasks_module.execute_background_job, "apply_async", failing_apply_async
+ )
+
+ try:
+ outcome = tasks_module.execute_background_job.apply(args=[job_id], throw=False)
+ finally:
+ tasks_module._EXTRA_HANDLERS.pop(job_type, None)
+
+ assert outcome.result == {"status": "deferred"}
+
+ verify_db = SessionLocal()
+ try:
+ refreshed = (
+ verify_db.query(BackgroundJob).filter(BackgroundJob.id == job_id).first()
+ )
+ assert refreshed is not None
+ # The row stays recoverable for the stale sweep instead of being failed.
+ assert refreshed.status == BackgroundJobStatus.ENQUEUED.value
+ assert refreshed.attempts == 0
+ finally:
+ verify_db.close()
+
+
+def test_admission_clears_the_migration_anchor_so_a_later_freeze_waits(
+ tmp_path, monkeypatch
+):
+ from xagent.core.tools.core.RAG_tools.core.exceptions import (
+ CollectionMigratingError,
+ )
+ from xagent.web.jobs.exceptions import BackgroundJobDeferred
+ from xagent.web.jobs.kb_tasks import (
+ _MIGRATION_DEFERRED_AT_KEY,
+ handle_kb_ingest_document,
+ )
+
+ monkeypatch.setenv(CELERY_ENABLED, "false")
+ monkeypatch.delenv(CELERY_BROKER_URL, raising=False)
+ frozen = {"on": True}
+
+ def guard(_collection: str) -> None:
+ if frozen["on"]:
+ raise CollectionMigratingError("Knowledge base is being upgraded")
+
+ monkeypatch.setattr("xagent.web.jobs.kb_tasks.ensure_collection_writable", guard)
+ monkeypatch.setattr(
+ "xagent.web.jobs.kb_tasks._save_job_collection_config_after_ingest",
+ lambda *args, **kwargs: None,
+ )
+ monkeypatch.setattr(
+ "xagent.web.jobs.kb_tasks.run_document_ingestion",
+ lambda **kwargs: IngestionResult(
+ status="success",
+ doc_id="doc-1",
+ chunk_count=1,
+ message="ok",
+ completed_steps=[
+ {"name": "register_document", "metadata": {"created": True}}
+ ],
+ ),
+ )
+
+ SessionLocal = _init_test_db(tmp_path / "kb-anchor-cleared.db")
+ db = SessionLocal()
+ try:
+ user = _create_user(db, username="kb-anchor-cleared")
+ staged_file = tmp_path / "stage" / "doc.txt"
+ target_file = tmp_path / "canonical" / "doc.txt"
+ staged_file.parent.mkdir(parents=True)
+ staged_file.write_text("staged content", encoding="utf-8")
+ job = create_background_job(
+ db,
+ user_id=int(user.id),
+ job_type=BackgroundJobType.KB_INGEST_DOCUMENT,
+ payload={
+ "collection": "kb",
+ "source_path": str(staged_file),
+ "target_path": str(target_file),
+ "file_id": "66666666-6666-4666-8666-666666666666",
+ "filename": "doc.txt",
+ "mime_type": "text/plain",
+ "file_size": staged_file.stat().st_size,
+ "user_id": int(user.id),
+ "is_admin": False,
+ "ingestion_config": IngestionConfig().model_dump(mode="json"),
+ "collection_existed_before": True,
+ },
+ )
+
+ with pytest.raises(BackgroundJobDeferred):
+ handle_kb_ingest_document(db, job)
+ db.refresh(job)
+ assert job.payload[_MIGRATION_DEFERRED_AT_KEY]
+
+ # The first refusal happened long ago; if the anchor survived admission,
+ # the next freeze would give up instead of waiting.
+ payload = dict(job.payload)
+ payload[_MIGRATION_DEFERRED_AT_KEY] = (
+ datetime.now(timezone.utc) - timedelta(minutes=11)
+ ).isoformat()
+ setattr(job, "payload", payload)
+ db.commit()
+
+ frozen["on"] = False
+ handle_kb_ingest_document(db, job)
+ db.refresh(job)
+ assert _MIGRATION_DEFERRED_AT_KEY not in job.payload
+
+ frozen["on"] = True
+ with pytest.raises(BackgroundJobDeferred) as raised:
+ handle_kb_ingest_document(db, job)
+ assert raised.value.delay_seconds == 15
+ finally:
+ db.close()
+
+
+def test_kb_web_job_clears_the_anchor_once_admitted(tmp_path, monkeypatch):
+ from xagent.web.jobs.kb_tasks import (
+ _MIGRATION_DEFERRED_AT_KEY,
+ handle_kb_ingest_web,
+ )
+
+ monkeypatch.setenv(CELERY_ENABLED, "false")
+ monkeypatch.delenv(CELERY_BROKER_URL, raising=False)
+ monkeypatch.setattr(
+ "xagent.web.jobs.kb_tasks._resolve_job_ingestion_models",
+ lambda *args, **kwargs: (_ for _ in ()).throw(
+ RuntimeError("stop after admission")
+ ),
+ )
+
+ SessionLocal = _init_test_db(tmp_path / "web-anchor-cleared.db")
+ db = SessionLocal()
+ try:
+ user = _create_user(db, username="web-anchor-cleared")
+ job = create_background_job(
+ db,
+ user_id=int(user.id),
+ job_type=BackgroundJobType.KB_INGEST_WEB,
+ payload={
+ "collection": "web-kb",
+ "crawl_config": WebCrawlConfig(
+ start_url="https://example.com"
+ ).model_dump(mode="json"),
+ "ingestion_config": IngestionConfig().model_dump(mode="json"),
+ "user_id": int(user.id),
+ "is_admin": False,
+ "collection_existed_before": False,
+ _MIGRATION_DEFERRED_AT_KEY: (
+ datetime.now(timezone.utc) - timedelta(minutes=11)
+ ).isoformat(),
+ },
+ )
+
+ with pytest.raises(RuntimeError, match="stop after admission"):
+ handle_kb_ingest_web(db, job)
+ db.refresh(job)
+ assert _MIGRATION_DEFERRED_AT_KEY not in job.payload
+ finally:
+ db.close()
+
+
+def test_kb_web_job_gives_up_with_a_terminal_message(tmp_path, monkeypatch):
+ from xagent.web.jobs.exceptions import BackgroundJobHandlerError
+ from xagent.web.jobs.kb_tasks import (
+ _MIGRATION_DEFERRED_AT_KEY,
+ _MIGRATION_GIVE_UP_MESSAGE,
+ handle_kb_ingest_web,
+ )
+
+ monkeypatch.setenv(CELERY_ENABLED, "false")
+ monkeypatch.delenv(CELERY_BROKER_URL, raising=False)
+ _freeze_collection(monkeypatch)
+
+ SessionLocal = _init_test_db(tmp_path / "web-give-up-message.db")
+ db = SessionLocal()
+ try:
+ user = _create_user(db, username="web-give-up-message")
+ job = create_background_job(
+ db,
+ user_id=int(user.id),
+ job_type=BackgroundJobType.KB_INGEST_WEB,
+ payload={
+ "collection": "kb",
+ "crawl_config": WebCrawlConfig(
+ start_url="https://example.com"
+ ).model_dump(mode="json"),
+ "ingestion_config": IngestionConfig().model_dump(mode="json"),
+ "user_id": int(user.id),
+ "is_admin": False,
+ "collection_existed_before": True,
+ _MIGRATION_DEFERRED_AT_KEY: (
+ datetime.now(timezone.utc) - timedelta(minutes=11)
+ ).isoformat(),
+ },
+ )
+
+ with pytest.raises(BackgroundJobHandlerError) as raised:
+ handle_kb_ingest_web(db, job)
+
+ assert raised.value.retryable is False
+ assert str(raised.value) == _MIGRATION_GIVE_UP_MESSAGE
+ finally:
+ db.close()