Enterprise agent platform for building, deploying, and operating AI teammates.
feat(kb): defer ingestion while a collection is frozen (#3757)
蓝鱼bluefish686e2dc73769
diff --git a/example.env b/example.envindex 77b86473c..a287c28b1 100644--- a/example.env+++ b/example.env@@ -426,6 +426,11 @@ XAGENT_REDIS_URL="redis://localhost:6379/0"# XAGENT_BACKGROUND_JOB_STALE_SECONDS="7200"# Scheduler interval for scanning stale background jobs.# XAGENT_BACKGROUND_JOB_SWEEP_INTERVAL_SECONDS="300"+# An ingest job refused because its collection is frozen for an embedding+# migration retries after this delay, until the budget (seconds since the+# first refusal) is spent.+# XAGENT_KB_MIGRATION_DEFER_DELAY_SECONDS="15"+# XAGENT_KB_MIGRATION_DEFER_BUDGET_SECONDS="600"# Milvus only: purge cadence for leftover rows, scheduled through Celery Beat# (no Beat means no purge). Deletes wait twice XAGENT_BACKGROUND_JOB_STALE_SECONDS; ignored by LanceDB.# XAGENT_KB_MILVUS_CLEANUP_INTERVAL_SECONDS="3600"diff --git a/frontend/src/components/kb/knowledge-base-creation-dialog.test.tsx b/frontend/src/components/kb/knowledge-base-creation-dialog.test.tsxindex 60c6e3491..5eb7e3a96 100644--- a/frontend/src/components/kb/knowledge-base-creation-dialog.test.tsx+++ b/frontend/src/components/kb/knowledge-base-creation-dialog.test.tsx@@ -14,6 +14,7 @@ vi.mock("@/contexts/auth-context", () => ({vi.mock("@/lib/api-wrapper", () => ({apiRequest: apiRequestMock,+ uploadWithCollectionMigrationRetry: apiRequestMock,parseApiResponse: async (response: { json: () => Promise<unknown> }) => ({data: await response.json(),text: null,@@ -621,6 +622,8 @@ describe("KnowledgeBaseCreationDialog collection naming", () => {expect(apiRequestMock).toHaveBeenCalledWith("http://api.local/api/kb/ingest-cloud",expect.any(Object),+ expect.any(Function),+ expect.any(Object),)})const cloudCall = apiRequestMock.mock.calls.find(@@ -707,6 +710,56 @@ describe("KnowledgeBaseCreationDialog collection naming", () => {consoleErrorSpy.mockRestore()}})++ it("clears the upgrading notice once a retried web import is accepted", async () => {+ const base = apiRequestMock.getMockImplementation()!+ apiRequestMock.mockImplementation(+ (url: string, options?: RequestInit, onUpgrading?: () => void) => {+ if (url === "http://api.local/api/kb/ingest-web/jobs") {+ onUpgrading?.()+ return Promise.resolve(+ createJsonResponse({+ id: "job-2",+ user_id: 1,+ job_type: "kb.ingest.web",+ queue: "kb",+ status: "running",+ progress: { message: "Crawling", completed: 0, total: 1 },+ result: null,+ error_message: null,+ celery_task_id: "task-2",+ attempts: 1,+ max_attempts: 3,+ })+ )+ }+ if (url === "http://api.local/api/jobs/job-2") {+ return new Promise(() => {})+ }+ return base(url, options)+ }+ )++ const { container } = render(+ <KnowledgeBaseCreationDialog open={true} onOpenChange={vi.fn()} onSuccess={vi.fn()} />+ )++ fireEvent.change(container.querySelector("#collection_name") as HTMLInputElement, {+ target: { value: "web_collection" },+ })++ await goToStep3(container, "web")+ fireEvent.click(screen.getByText("kb.dialog.createButton"))++ await waitFor(() => {+ expect(+ apiRequestMock.mock.calls.some(+ ([url]) => url === "http://api.local/api/jobs/job-2"+ )+ ).toBe(true)+ }, { timeout: 5000 })+ expect(screen.queryByText(/kb\.migration\.upgrading/)).toBeNull()+ })})const RESERVE_URL = "http://api.local/api/knowledge-bases/team-docs/reserve-team"diff --git a/frontend/src/components/kb/knowledge-base-creation-dialog.tsx b/frontend/src/components/kb/knowledge-base-creation-dialog.tsxindex 99e475d7c..2ec2feabc 100644--- a/frontend/src/components/kb/knowledge-base-creation-dialog.tsx+++ b/frontend/src/components/kb/knowledge-base-creation-dialog.tsx@@ -31,7 +31,7 @@ import {import { useI18n } from "@/contexts/i18n-context"import type { TranslationKey } from "@/i18n/translations"import { useAuth } from "@/contexts/auth-context"-import { apiRequest, getUploadErrorMessage, isJsonRecord, parseApiResponse, UPLOAD_ERROR_MESSAGES } from "@/lib/api-wrapper"+import { apiRequest, getUploadErrorMessage, isJsonRecord, parseApiResponse, UPLOAD_ERROR_MESSAGES, uploadWithCollectionMigrationRetry } from "@/lib/api-wrapper"import { Model } from "@/lib/models"import {Upload,@@ -236,6 +236,15 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: Kconst [ingestionResults, setIngestionResults] = useState<IngestionResult[]>([])const [isDragging, setIsDragging] = useState(false)const fileInputRef = useRef<HTMLInputElement>(null)+ const uploadAbortRef = useRef<AbortController | null>(null)+ const mountedRef = useRef(true)+ useEffect(() => {+ mountedRef.current = true+ return () => {+ mountedRef.current = false+ uploadAbortRef.current?.abort()+ }+ }, [])const [currentUploadFileName, setCurrentUploadFileName] = useState<string | null>(null)const [currentUploadCollection, setCurrentUploadCollection] = useState<string | null>(null)const [completedUploadCount, setCompletedUploadCount] = useState(0)@@ -487,10 +496,19 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: Ktoast.warning(t("kb.ownership.releaseFailed", { name: collection }))}+ const beginAbortableUpload = () => {+ uploadAbortRef.current?.abort()+ const controller = new AbortController()+ uploadAbortRef.current = controller+ return controller+ }+const resetState = () => {+ uploadAbortRef.current?.abort()setSelectedFiles([])setOwnership("personal")setUploadProgress(0)+ setUploadProgressDetail(null)setIngestionResults([])setWebIngestionResult(null)setNewCollectionName("")@@ -521,6 +539,7 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: Ktoast.error(t("kb.errors.uploadFileRequired"))return}+ const controller = beginAbortableUpload()setIsUploading(true)// Classify on the status code, not on the backend's English wording.@@ -554,13 +573,17 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: KnormalizeIngestionConfigForFilename(ingestionConfig, file.name))- const response = await apiRequest(+ const response = await uploadWithCollectionMigrationRetry(`${apiUrl}/api/kb/ingest${useBackgroundJobs ? "/jobs" : ""}`,{method: "POST",body: formData- }+ },+ () => setUploadProgressDetail(t("kb.migration.upgrading")),+ controller.signal)+ setUploadProgressDetail(null)+ if (controller.signal.aborted) returnconst parsed = await parseApiResponse(response)@@ -669,10 +692,12 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: KonSuccess?.(successfulCollections)}} finally {- setIsUploading(false)- setCurrentUploadFileName(null)- setCurrentUploadCollection(null)- setUploadProgressDetail(null)+ if (mountedRef.current) {+ setIsUploading(false)+ setCurrentUploadFileName(null)+ setCurrentUploadCollection(null)+ setUploadProgressDetail(null)+ }}}@@ -681,11 +706,13 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: Ktoast.error(t("kb.errors.startUrlRequired"))return}+ const controller = beginAbortableUpload()setIsWebIngesting(true)let failedStatus: number | undefinedsetWebIngestionProgress(0)setWebIngestionResult(null)+ setUploadProgressDetail(null)const collectionName = trimmedCollectionNameconst teamClaimed = { current: false }@@ -722,13 +749,17 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: KsetWebIngestionProgress(10)- const response = await apiRequest(+ const response = await uploadWithCollectionMigrationRetry(`${apiUrl}/api/kb/ingest-web${useBackgroundJobs ? "/jobs" : ""}`,{method: "POST",body: formData- }+ },+ () => setUploadProgressDetail(t("kb.migration.upgrading")),+ controller.signal)+ setUploadProgressDetail(null)+ if (controller.signal.aborted) returnconst parsed = await parseApiResponse(response)@@ -802,17 +833,21 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: K})if (teamClaimed.current) void releaseTeamName(collectionName)} finally {- setIsWebIngesting(false)- setWebIngestionProgress(0)+ if (mountedRef.current) {+ setIsWebIngesting(false)+ setWebIngestionProgress(0)+ }}}const handleCloudIngest = async () => {if (totalCloudFiles === 0) return+ const controller = beginAbortableUpload()setIsCloudConnecting(true)let failedStatus: number | undefinedsetIngestionResults([])+ setUploadProgressDetail(null)const collectionName = trimmedCollectionNameconst teamClaimed = { current: false }@@ -859,13 +894,15 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: Kretry_delay: ingestionConfig.retry_delay}- const response = await apiRequest(`${getApiUrl()}/api/kb/ingest-cloud`, {+ const response = await uploadWithCollectionMigrationRetry(`${getApiUrl()}/api/kb/ingest-cloud`, {method: "POST",headers: {"Content-Type": "application/json"},body: JSON.stringify(requestBody)- })+ }, () => setUploadProgressDetail(t("kb.migration.upgrading")), controller.signal)+ setUploadProgressDetail(null)+ if (controller.signal.aborted) returnconst parsed = await parseApiResponse(response)@@ -935,7 +972,7 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: Kvoid releaseTeamName(collectionName)}} finally {- setIsCloudConnecting(false)+ if (mountedRef.current) setIsCloudConnecting(false)}}diff --git a/frontend/src/components/kb/knowledge-base-detail.component.test.tsx b/frontend/src/components/kb/knowledge-base-detail.component.test.tsxindex c2a0ec349..c147d9edd 100644--- a/frontend/src/components/kb/knowledge-base-detail.component.test.tsx+++ b/frontend/src/components/kb/knowledge-base-detail.component.test.tsx@@ -7,6 +7,7 @@ const toastErrorMock = vi.hoisted(() => vi.fn())vi.mock("@/lib/api-wrapper", () => ({apiRequest: apiRequestMock,+ uploadWithCollectionMigrationRetry: apiRequestMock,}))vi.mock("@/lib/utils", () => ({diff --git a/frontend/src/components/kb/knowledge-base-detail.tsx b/frontend/src/components/kb/knowledge-base-detail.tsxindex a9c0eeb32..89915aae3 100644--- a/frontend/src/components/kb/knowledge-base-detail.tsx+++ b/frontend/src/components/kb/knowledge-base-detail.tsx@@ -13,7 +13,7 @@ import { ScrollArea } from "@/components/ui/scroll-area"import { Select } from "@/components/ui/select"import { Badge } from "@/components/ui/badge"import { ConfirmDialog } from "@/components/ui/confirm-dialog"-import { apiRequest, getUploadErrorMessage, isJsonRecord, parseApiResponse, UPLOAD_ERROR_MESSAGES } from "@/lib/api-wrapper"+import { apiRequest, getUploadErrorMessage, isJsonRecord, parseApiResponse, UPLOAD_ERROR_MESSAGES, uploadWithCollectionMigrationRetry } from "@/lib/api-wrapper"import { getApiUrl } from "@/lib/utils"import {getBackgroundJobFailureMessage,@@ -162,6 +162,15 @@ export function KnowledgeBaseDetailContent({ collectionName }: { collectionName:const [ingestionResults, setIngestionResults] = useState<IngestionResult[]>([])const [currentUploadFileName, setCurrentUploadFileName] = useState<string | null>(null)const [completedUploadCount, setCompletedUploadCount] = useState(0)+ const uploadAbortRef = useRef<AbortController | null>(null)+ const mountedRef = useRef(true)+ useEffect(() => {+ mountedRef.current = true+ return () => {+ mountedRef.current = false+ uploadAbortRef.current?.abort()+ }+ }, [])const [isAddSourceOpen, setIsAddSourceOpen] = useState(false)const [activeAddSourceMode, setActiveAddSourceMode] = useState<"web" | "file" | null>(null)const [isDragging, setIsDragging] = useState(false)@@ -214,6 +223,7 @@ export function KnowledgeBaseDetailContent({ collectionName }: { collectionName:const [isWebIngesting, setIsWebIngesting] = useState(false)const [webIngestionProgress, setWebIngestionProgress] = useState(0)const [webIngestionResult, setWebIngestionResult] = useState<WebIngestionResult | null>(null)+ const [webIngestionNotice, setWebIngestionNotice] = useState<string | null>(null)const [webIngestionConfig, setWebIngestionConfig] = useState({start_url: "",max_pages: 100,@@ -405,8 +415,16 @@ export function KnowledgeBaseDetailContent({ collectionName }: { collectionName:setSelectedFiles(prev => prev.filter((_, i) => i !== index))}+ const beginAbortableUpload = () => {+ uploadAbortRef.current?.abort()+ const controller = new AbortController()+ uploadAbortRef.current = controller+ return controller+ }+const doUpload = async () => {if (selectedFiles.length === 0) return+ const controller = beginAbortableUpload()setIsUploading(true)setUploadProgress(0)@@ -430,13 +448,17 @@ export function KnowledgeBaseDetailContent({ collectionName }: { collectionName:normalizeIngestionConfigForFilename(ingestionConfig, file.name))- const response = await apiRequest(+ const response = await uploadWithCollectionMigrationRetry(`${apiUrl}/api/kb/ingest${useBackgroundJobs ? "/jobs" : ""}`,{method: "POST",body: formData- }+ },+ () => setUploadProgressDetail(t("kb.migration.upgrading")),+ controller.signal)+ setUploadProgressDetail(null)+ if (controller.signal.aborted) returnconst parsed = await parseApiResponse(response)@@ -533,9 +555,11 @@ export function KnowledgeBaseDetailContent({ collectionName }: { collectionName:description: toastContent.description,})} finally {- setIsUploading(false)- setCurrentUploadFileName(null)- setUploadProgressDetail(null)+ if (mountedRef.current) {+ setIsUploading(false)+ setCurrentUploadFileName(null)+ setUploadProgressDetail(null)+ }}}@@ -599,6 +623,8 @@ export function KnowledgeBaseDetailContent({ collectionName }: { collectionName:setIsWebIngesting(true)setWebIngestionProgress(0)setWebIngestionResult(null)+ setWebIngestionNotice(null)+ const controller = beginAbortableUpload()try {const apiUrl = getApiUrl()@@ -632,13 +658,17 @@ export function KnowledgeBaseDetailContent({ collectionName }: { collectionName:setWebIngestionProgress(10)- const response = await apiRequest(+ const response = await uploadWithCollectionMigrationRetry(`${apiUrl}/api/kb/ingest-web${useBackgroundJobs ? "/jobs" : ""}`,{method: "POST",body: formData- }+ },+ () => setWebIngestionNotice(t("kb.migration.upgrading")),+ controller.signal)+ setWebIngestionNotice(null)+ if (controller.signal.aborted) returnconst parsed = await parseApiResponse(response)@@ -730,8 +760,10 @@ export function KnowledgeBaseDetailContent({ collectionName }: { collectionName:description: toastContent.description,})} finally {- setIsWebIngesting(false)- setWebIngestionProgress(0)+ if (mountedRef.current) {+ setIsWebIngesting(false)+ setWebIngestionProgress(0)+ }}}@@ -1568,7 +1600,7 @@ export function KnowledgeBaseDetailContent({ collectionName }: { collectionName:{isWebIngesting ? (<div className="flex items-center gap-2"><Loader2 className="h-4 w-4 animate-spin" />- {t("kb.dialog.webImport.status.crawling")} ({Math.round(webIngestionProgress)}%)+ {webIngestionNotice || t("kb.dialog.webImport.status.crawling")} ({Math.round(webIngestionProgress)}%)</div>) : (t("kb.index.startImport")diff --git a/frontend/src/components/kb/knowledge-base-detail.web-ingest.test.tsx b/frontend/src/components/kb/knowledge-base-detail.web-ingest.test.tsxindex a1e448d84..7421a998c 100644--- a/frontend/src/components/kb/knowledge-base-detail.web-ingest.test.tsx+++ b/frontend/src/components/kb/knowledge-base-detail.web-ingest.test.tsx@@ -7,6 +7,7 @@ const toastErrorMock = vi.hoisted(() => vi.fn())vi.mock("@/lib/api-wrapper", () => ({apiRequest: apiRequestMock,+ uploadWithCollectionMigrationRetry: apiRequestMock,parseApiResponse: async (response: { json: () => Promise<unknown> }) => ({data: await response.json(),text: null,@@ -290,6 +291,94 @@ describe("KnowledgeBaseDetailContent web ingest", () => {expect.anything())})++ it("clears the upgrading notice once a retried web import is accepted", async () => {+ const previous = apiRequestMock.getMockImplementation()!+ apiRequestMock.mockImplementation(+ (url: string, options?: RequestInit, onUpgrading?: () => void) => {+ if (url === "http://api.local/api/kb/ingest-web/jobs") {+ onUpgrading?.()+ return Promise.resolve(+ createJsonResponse({+ id: "job-2",+ user_id: 1,+ job_type: "kb.ingest.web",+ queue: "kb",+ status: "running",+ progress: { message: "Crawling", completed: 0, total: 1 },+ result: null,+ error_message: null,+ celery_task_id: "task-2",+ attempts: 1,+ max_attempts: 3,+ })+ )+ }+ if (url === "http://api.local/api/jobs/job-2") {+ return new Promise(() => {})+ }+ return previous(url, options)+ }+ )++ render(<KnowledgeBaseDetailContent collectionName="demo" />)++ await waitFor(() => {+ expect(screen.getByText("kb.detail.files.addSource")).toBeInTheDocument()+ })++ fireEvent.click(screen.getByText("kb.detail.files.addSource"))+ fireEvent.click(screen.getByText("kb.dialog.tabs.web"))+ fireEvent.change(screen.getByLabelText("kb.dialog.webImport.basic.startUrl *"), {+ target: { value: "https://example.com/docs" },+ })+ fireEvent.click(screen.getByText("kb.index.startImport"))++ await waitFor(() => {+ expect(+ apiRequestMock.mock.calls.some(+ ([url]) => url === "http://api.local/api/jobs/job-2"+ )+ ).toBe(true)+ }, { timeout: 5000 })+ expect(screen.queryByText(/kb\.migration\.upgrading/)).toBeNull()+ })++ it("aborts the pending upload retry when the page unmounts", async () => {+ const previous = apiRequestMock.getMockImplementation()!+ let signal: AbortSignal | undefined+ apiRequestMock.mockImplementation(+ (+ url: string,+ options?: RequestInit,+ _onUpgrading?: () => void,+ captured?: AbortSignal+ ) => {+ if (url === "http://api.local/api/kb/ingest-web/jobs") {+ signal = captured+ return new Promise(() => {})+ }+ return previous(url, options)+ }+ )++ const { unmount } = render(<KnowledgeBaseDetailContent collectionName="demo" />)++ await waitFor(() => {+ expect(screen.getByText("kb.detail.files.addSource")).toBeInTheDocument()+ })++ fireEvent.click(screen.getByText("kb.detail.files.addSource"))+ fireEvent.click(screen.getByText("kb.dialog.tabs.web"))+ fireEvent.change(screen.getByLabelText("kb.dialog.webImport.basic.startUrl *"), {+ target: { value: "https://example.com/docs" },+ })+ fireEvent.click(screen.getByText("kb.index.startImport"))++ await waitFor(() => expect(signal).toBeDefined())+ unmount()+ expect(signal?.aborted).toBe(true)+ })})describe("KnowledgeBaseDetailContent config save", () => {diff --git a/frontend/src/i18n/locales/en.ts b/frontend/src/i18n/locales/en.tsindex b0be4f09b..42ff6f352 100644--- a/frontend/src/i18n/locales/en.ts+++ b/frontend/src/i18n/locales/en.ts@@ -2509,6 +2509,9 @@ Build when you need.`,},},kb: {+ migration: {+ upgrading: "Knowledge base is upgrading. The upload retries automatically.",+ },ownership: {label: "Ownership",personal: "Personal",diff --git a/frontend/src/i18n/locales/zh.ts b/frontend/src/i18n/locales/zh.tsindex 3ad06b35c..9f70dba94 100644--- a/frontend/src/i18n/locales/zh.ts+++ b/frontend/src/i18n/locales/zh.ts@@ -2504,6 +2504,9 @@ const zh = {},},kb: {+ migration: {+ upgrading: "知识库升级中,上传将自动重试。",+ },ownership: {label: "归属",personal: "个人",diff --git a/frontend/src/lib/api-wrapper.test.ts b/frontend/src/lib/api-wrapper.test.tsindex e3f4c782b..67a4c1a85 100644--- a/frontend/src/lib/api-wrapper.test.ts+++ b/frontend/src/lib/api-wrapper.test.ts@@ -5,8 +5,12 @@ import {classifyUploadError,getApiErrorMessage,getUploadErrorMessage,+ KB_COLLECTION_MIGRATING_CODE,+ KB_MIGRATION_RETRY_DELAY_MS,+ KB_MIGRATION_RETRY_LIMIT,parseApiResponse,refreshStoredAccessToken,+ uploadWithCollectionMigrationRetry,} from "@/lib/api-wrapper"import {AUTH_CACHE_KEY,@@ -260,6 +264,283 @@ describe("api-wrapper API error helpers", () => {})})+describe("api-wrapper collection migration retry", () => {+ function migratingResponse(retryAfterSeconds?: number): Response {+ const headers: Record<string, string> = { "Content-Type": "application/json" }+ if (retryAfterSeconds !== undefined) headers["Retry-After"] = String(retryAfterSeconds)+ return new Response(JSON.stringify({+ detail: {+ code: KB_COLLECTION_MIGRATING_CODE,+ message: "Knowledge base is being upgraded",+ },+ }), { status: 409, headers })+ }++ function successResponse(): Response {+ return new Response(JSON.stringify({ status: "success" }), {+ status: 200,+ headers: { "Content-Type": "application/json" },+ })+ }++ it("reads the structured detail message for uploads", async () => {+ const response = migratingResponse()+ const parsed = await parseApiResponse(response)++ expect(getUploadErrorMessage(response, parsed, MESSAGES)).toBe(+ "Knowledge base is being upgraded",+ )+ })++ it("retries an upload refused while the collection is frozen", async () => {+ vi.useFakeTimers()+ try {+ const fetchMock = vi.spyOn(globalThis, "fetch")+ .mockResolvedValueOnce(migratingResponse(5))+ .mockResolvedValueOnce(successResponse())+ const onUpgrading = vi.fn()++ const options = { method: "POST", body: new FormData() }+ const pending = uploadWithCollectionMigrationRetry(+ "/api/kb/ingest/jobs",+ options,+ onUpgrading,+ )+ await vi.advanceTimersByTimeAsync(0)+ expect(fetchMock).toHaveBeenCalledTimes(1)+ // The server's Retry-After paces the retry; the default gap is not used.+ await vi.advanceTimersByTimeAsync(4_999)+ expect(fetchMock).toHaveBeenCalledTimes(1)+ await vi.advanceTimersByTimeAsync(1)++ const response = await pending+ expect(response.status).toBe(200)+ expect(fetchMock).toHaveBeenCalledTimes(2)+ expect(onUpgrading).toHaveBeenCalledTimes(1)+ const [retriedUrl, retriedOptions] = fetchMock.mock.calls[1]+ expect(retriedUrl).toBe("/api/kb/ingest/jobs")+ expect(retriedOptions?.method).toBe("POST")+ expect(retriedOptions?.body).toBe(options.body)+ } finally {+ vi.useRealTimers()+ vi.restoreAllMocks()+ }+ })++ it("gives up after the retry limit and returns the refusal", async () => {+ vi.useFakeTimers()+ try {+ const fetchMock = vi.spyOn(globalThis, "fetch")+ for (let i = 0; i < KB_MIGRATION_RETRY_LIMIT + 1; i++) {+ fetchMock.mockResolvedValueOnce(migratingResponse())+ }+ const onUpgrading = vi.fn()++ const pending = uploadWithCollectionMigrationRetry(+ "/api/kb/ingest/jobs",+ { method: "POST", body: new FormData() },+ onUpgrading,+ )+ for (let i = 0; i < KB_MIGRATION_RETRY_LIMIT; i++) {+ await vi.advanceTimersByTimeAsync(KB_MIGRATION_RETRY_DELAY_MS)+ }++ const response = await pending+ expect(response.status).toBe(409)+ expect(fetchMock).toHaveBeenCalledTimes(KB_MIGRATION_RETRY_LIMIT + 1)+ expect(onUpgrading).toHaveBeenCalledTimes(KB_MIGRATION_RETRY_LIMIT)+ } finally {+ vi.useRealTimers()+ vi.restoreAllMocks()+ }+ })++ it("stops retrying when another sign-in replaces the initiating session", async () => {+ vi.useFakeTimers()+ try {+ writeAuthCache({ id: "1", username: "alice" }, "token-a")+ const fetchMock = vi.spyOn(globalThis, "fetch")+ .mockResolvedValueOnce(migratingResponse(5))+ const onUpgrading = vi.fn()++ const pending = uploadWithCollectionMigrationRetry(+ "/api/kb/ingest/jobs",+ { method: "POST", body: new FormData() },+ onUpgrading,+ )+ await vi.advanceTimersByTimeAsync(0)+ expect(fetchMock).toHaveBeenCalledTimes(1)++ writeAuthCache({ id: "2", username: "bob" }, "token-b")+ await vi.advanceTimersByTimeAsync(5_000)++ const response = await pending+ expect(response.status).toBe(409)+ // A's body must never be resent with B's session.+ expect(fetchMock).toHaveBeenCalledTimes(1)+ expect(onUpgrading).toHaveBeenCalledTimes(1)+ } finally {+ vi.useRealTimers()+ vi.restoreAllMocks()+ }+ })++ it("stops retrying when the signal aborts during the wait", async () => {+ vi.useFakeTimers()+ try {+ writeAuthCache({ id: "1", username: "alice" }, "token-a")+ const controller = new AbortController()+ const fetchMock = vi+ .spyOn(globalThis, "fetch")+ .mockResolvedValueOnce(migratingResponse(15))+ const onUpgrading = vi.fn()++ const pending = uploadWithCollectionMigrationRetry(+ "/api/kb/ingest/jobs",+ { method: "POST", body: new FormData() },+ onUpgrading,+ controller.signal,+ )+ await vi.advanceTimersByTimeAsync(0)+ expect(fetchMock).toHaveBeenCalledTimes(1)++ controller.abort()+ await vi.advanceTimersByTimeAsync(0)++ const response = await pending+ expect(response.status).toBe(409)+ expect(fetchMock).toHaveBeenCalledTimes(1)+ expect(onUpgrading).toHaveBeenCalledTimes(1)+ } finally {+ vi.useRealTimers()+ vi.restoreAllMocks()+ }+ })++ it("sends no request at all when the signal is already aborted", async () => {+ const controller = new AbortController()+ controller.abort()+ const fetchMock = vi.spyOn(globalThis, "fetch")++ const response = await uploadWithCollectionMigrationRetry(+ "/api/kb/ingest/jobs",+ { method: "POST", body: new FormData() },+ undefined,+ controller.signal,+ )++ expect(response.status).toBe(409)+ expect(fetchMock).not.toHaveBeenCalled()+ vi.restoreAllMocks()+ })++ it("sends the credential from the snapshot it checked, not a later storage read", async () => {+ writeAuthCache({ id: "1", username: "alice" }, "token-a")+ const cacheA = localStorage.getItem(AUTH_CACHE_KEY)+ writeAuthCache({ id: "2", username: "bob" }, "token-b")+ const cacheB = localStorage.getItem(AUTH_CACHE_KEY)+ if (!cacheA || !cacheB) throw new Error("auth cache missing")+ localStorage.setItem(AUTH_CACHE_KEY, cacheA)++ const originalGetItem = window.localStorage.getItem.bind(window.localStorage)+ let authReads = 0+ vi.spyOn(window.localStorage, "getItem").mockImplementation((key: string) => {+ if (key === AUTH_CACHE_KEY) {+ authReads += 1+ // The storage probe plus the snapshot read are one logical read; any+ // further read would pick up the other tab's session.+ return authReads <= 2 ? cacheA : cacheB+ }+ return originalGetItem(key)+ })+ const fetchMock = vi+ .spyOn(globalThis, "fetch")+ .mockResolvedValueOnce(successResponse())++ const response = await uploadWithCollectionMigrationRetry(+ "/api/kb/ingest/jobs",+ { method: "POST", body: new FormData() },+ )++ expect(response.status).toBe(200)+ expect(authReads).toBe(2)+ const headers = fetchMock.mock.calls[0][1]?.headers as Record<string, string>+ expect(headers.Authorization).toBe("Bearer token-a")+ vi.restoreAllMocks()+ })++ it("keeps retrying after a credential refresh within the initiating session", async () => {+ vi.useFakeTimers()+ try {+ writeAuthCache({ id: "1", username: "alice" }, "token-a")+ const cached = readAuthCache()+ if (!cached) throw new Error("auth cache missing")+ const fetchMock = vi.spyOn(globalThis, "fetch")+ .mockResolvedValueOnce(migratingResponse(5))+ .mockResolvedValueOnce(successResponse())++ const pending = uploadWithCollectionMigrationRetry(+ "/api/kb/ingest/jobs",+ { method: "POST", body: new FormData() },+ )+ await vi.advanceTimersByTimeAsync(0)+ localStorage.setItem(AUTH_CACHE_KEY, JSON.stringify({+ ...cached,+ credentialRevision: cached.credentialRevision + 1,+ token: "token-a-refreshed",+ }))+ await vi.advanceTimersByTimeAsync(5_000)++ const response = await pending+ expect(response.status).toBe(200)+ expect(fetchMock).toHaveBeenCalledTimes(2)+ } finally {+ vi.useRealTimers()+ vi.restoreAllMocks()+ }+ })++ it("does not retry a 409 that is not a migration refusal", async () => {+ const fetchMock = vi.spyOn(globalThis, "fetch")+ .mockResolvedValueOnce(new Response(JSON.stringify({+ detail: { code: "kb_admission_conflict", message: "Another upload is in progress" },+ }), { status: 409, headers: { "Content-Type": "application/json" } }))+ const onUpgrading = vi.fn()++ const response = await uploadWithCollectionMigrationRetry(+ "/api/kb/ingest/jobs",+ { method: "POST", body: new FormData() },+ onUpgrading,+ )++ expect(response.status).toBe(409)+ expect(fetchMock).toHaveBeenCalledTimes(1)+ expect(onUpgrading).not.toHaveBeenCalled()+ vi.restoreAllMocks()+ })++ it("does not retry a 409 whose body is not parseable JSON", async () => {+ const fetchMock = vi.spyOn(globalThis, "fetch").mockResolvedValueOnce(+ new Response("<html>upstream</html>", {+ status: 409,+ headers: { "Content-Type": "text/html" },+ }),+ )+ const onUpgrading = vi.fn()++ const response = await uploadWithCollectionMigrationRetry(+ "/api/kb/ingest/jobs",+ { method: "POST", body: new FormData() },+ onUpgrading,+ )++ expect(response.status).toBe(409)+ expect(fetchMock).toHaveBeenCalledTimes(1)+ expect(onUpgrading).not.toHaveBeenCalled()+ vi.restoreAllMocks()+ })+})+describe("api-wrapper auth refresh", () => {const user = { id: "1", username: "alice", email: null, is_admin: false }diff --git a/frontend/src/lib/api-wrapper.ts b/frontend/src/lib/api-wrapper.tsindex 4b81f4774..7cfaf7f16 100644--- a/frontend/src/lib/api-wrapper.ts+++ b/frontend/src/lib/api-wrapper.ts@@ -11,6 +11,7 @@ import {type AuthMutationUnavailableReason,clearAuthSessionIfCurrent,compareAuthSession, compareCredentialSession,+ compareAuthSessionLineage,commitAuthSessionRefresh,getAuthStorageAvailability,isSafeAuthExpirySeconds,@@ -148,8 +149,12 @@ function withBearer(options: RequestInit, token: string): RequestInit {return { ...options, headers: { ...options.headers, Authorization: `Bearer ${token}` } }}/** A request has at most one post-401 replay, bound to an exact immutable credential snapshot. */-export async function apiRequest(url: string, options: RequestInit = {}): Promise<Response> {- const session = readAuthSessionSnapshot()+export async function apiRequest(+ url: string,+ options: RequestInit = {},+ capturedSession?: AuthSessionSnapshot,+): Promise<Response> {+ const session = capturedSession ?? readAuthSessionSnapshot()if (!session.accessToken) return fetch(url, options)const response = await fetchWithRetry(url, withBearer(options, session.accessToken))if (response.status !== 401 || shouldSkipRefresh(url)) return response@@ -208,11 +213,69 @@ export function getUploadErrorMessage(response: Response, parsed: ParsedApiRespo.filter(Boolean)if (validationMessages.length) return validationMessages.join("; ")}+ if (isJsonRecord(parsed.data) && isJsonRecord(parsed.data.detail) && typeof parsed.data.detail.message === "string" && parsed.data.detail.message.trim()) return parsed.data.detail.messageif (isJsonRecord(parsed.data) && typeof parsed.data.message === "string" && parsed.data.message.trim()) return parsed.data.messageif (response.status === 413) return messages.tooLargeif (parsed.isHtml) return messages.proxyreturn parsed.text?.trim() ? truncateUploadMessage(parsed.text) : messages.generic}+export const KB_COLLECTION_MIGRATING_CODE = "kb_collection_migrating"+export const KB_MIGRATION_RETRY_LIMIT = 8+export const KB_MIGRATION_RETRY_DELAY_MS = 10_000+function isCollectionMigratingResponse(parsed: ParsedApiResponse): boolean {+ return isJsonRecord(parsed.data) && isJsonRecord(parsed.data.detail)+ && parsed.data.detail.code === KB_COLLECTION_MIGRATING_CODE+}+function isSameSessionLineage(captured: AuthSessionSnapshot, current: AuthSessionSnapshot): boolean {+ return compareAuthSessionLineage(captured, current) === "same"+}+function waitForRetry(ms: number, signal?: AbortSignal): Promise<void> {+ return new Promise(resolve => {+ if (signal?.aborted) { resolve(); return }+ let timer: ReturnType<typeof setTimeout>+ const done = () => {+ clearTimeout(timer)+ signal?.removeEventListener("abort", done)+ resolve()+ }+ timer = setTimeout(done, ms)+ signal?.addEventListener("abort", done, { once: true })+ })+}+/** Retries an upload refused while its collection is frozen, staying with the initiating user. */+export async function uploadWithCollectionMigrationRetry(+ url: string,+ options: RequestInit,+ onUpgrading?: () => void,+ signal?: AbortSignal,+): Promise<Response> {+ let initiatingSession: AuthSessionSnapshot | null = null+ let boundToSession = false+ let refusal = new Response(null, { status: 409 })+ for (let attempt = 0; ; attempt += 1) {+ if (signal?.aborted) return refusal+ // One read per attempt: the checked snapshot is the one whose credential is sent.+ const attemptSession = readAuthSessionSnapshot()+ if (initiatingSession === null) {+ initiatingSession = attemptSession+ boundToSession = Boolean(attemptSession.sessionId)+ } else if (boundToSession && !isSameSessionLineage(initiatingSession, attemptSession)) {+ // Another tab may have signed in as someone else while this one waited.+ return refusal+ }+ const response = await apiRequest(url, options, attemptSession)+ if (response.status !== 409 || attempt >= KB_MIGRATION_RETRY_LIMIT) return response+ if (!isCollectionMigratingResponse(await parseApiResponse(response.clone()))) return response+ refusal = response+ const retryAfterSeconds = Number.parseInt(response.headers.get("Retry-After") ?? "", 10)+ const delayMs = Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0+ ? Math.min(retryAfterSeconds, 60) * 1000+ : KB_MIGRATION_RETRY_DELAY_MS+ if (signal?.aborted) return refusal+ onUpgrading?.()+ await waitForRetry(delayMs, signal)+ }+}export interface ClassifiedUploadError {errorCode: ClientErrorCodemessage: stringdiff --git a/frontend/src/lib/auth-cache.ts b/frontend/src/lib/auth-cache.tsindex 3719dffc9..049c9bad3 100644--- a/frontend/src/lib/auth-cache.ts+++ b/frontend/src/lib/auth-cache.ts@@ -307,19 +307,30 @@ export function readAuthSessionSnapshot(): AuthSessionSnapshot {const inspection = inspectAuthSession()return inspection.status === "valid" ? inspection.projection.snapshot : EMPTY_SNAPSHOT}-export function compareAuthSession(captured: AuthSessionSnapshot, inspection = inspectAuthSession()): AuthSessionComparison {- if (inspection.status !== "valid") return { status: inspection.status, projection: null }- const { cache, snapshot } = inspection.projection- if (!captured.sessionId || cache.sessionId !== captured.sessionId) return { status: "replaced", projection: null }- if (cache.user.id !== captured.userId || cache.credentialRevision < (captured.credentialRevision ?? -1) || cache.profileRevision < (captured.profileRevision ?? -1)) {- return { status: "invalid", projection: null }+export type AuthSessionLineage = "same" | "replaced" | "invalid"+/** Compare two snapshots without touching shared storage. */+export function compareAuthSessionLineage(captured: AuthSessionSnapshot, current: AuthSessionSnapshot): AuthSessionLineage {+ if (!captured.sessionId || current.sessionId !== captured.sessionId) return "replaced"+ if (current.userId !== captured.userId || current.credentialRevision === null || current.profileRevision === null) {+ return "invalid"+ }+ if (current.credentialRevision < (captured.credentialRevision ?? -1) || current.profileRevision < (captured.profileRevision ?? -1)) {+ return "invalid"}- if (cache.credentialRevision === captured.credentialRevision && (cache.token !== captured.accessToken || cache.refreshToken !== captured.refreshToken)) {- return { status: "invalid", projection: null }+ if (current.credentialRevision === captured.credentialRevision && (current.accessToken !== captured.accessToken || current.refreshToken !== captured.refreshToken)) {+ return "invalid"}- if (cache.profileRevision === captured.profileRevision && snapshot.profileFingerprint !== captured.profileFingerprint) {- return { status: "invalid", projection: null }+ if (current.profileRevision === captured.profileRevision && current.profileFingerprint !== captured.profileFingerprint) {+ return "invalid"}+ return "same"+}+export function compareAuthSession(captured: AuthSessionSnapshot, inspection = inspectAuthSession()): AuthSessionComparison {+ if (inspection.status !== "valid") return { status: inspection.status, projection: null }+ const { cache, snapshot } = inspection.projection+ const lineage = compareAuthSessionLineage(captured, snapshot)+ if (lineage === "replaced") return { status: "replaced", projection: null }+ if (lineage === "invalid") return { status: "invalid", projection: null }const credentialExact = cache.credentialRevision === captured.credentialRevision && cache.token === captured.accessToken && cache.refreshToken === captured.refreshTokenconst profileExact = cache.profileRevision === captured.profileRevisionif (credentialExact && profileExact) return { status: "exact", projection: inspection.projection }diff --git a/src/xagent/config.py b/src/xagent/config.pyindex df64b631a..80afd15d6 100644--- a/src/xagent/config.py+++ b/src/xagent/config.py@@ -250,6 +250,8 @@ BACKGROUND_JOB_VISIBILITY_TIMEOUT_SECONDS = (BACKGROUND_JOB_MAX_RETRIES = "XAGENT_BACKGROUND_JOB_MAX_RETRIES"BACKGROUND_JOB_STALE_SECONDS = "XAGENT_BACKGROUND_JOB_STALE_SECONDS"BACKGROUND_JOB_SWEEP_INTERVAL_SECONDS = "XAGENT_BACKGROUND_JOB_SWEEP_INTERVAL_SECONDS"+KB_MIGRATION_DEFER_DELAY_SECONDS = "XAGENT_KB_MIGRATION_DEFER_DELAY_SECONDS"+KB_MIGRATION_DEFER_BUDGET_SECONDS = "XAGENT_KB_MIGRATION_DEFER_BUDGET_SECONDS"KB_MILVUS_CLEANUP_INTERVAL_SECONDS = "XAGENT_KB_MILVUS_CLEANUP_INTERVAL_SECONDS"KB_MILVUS_CLEANUP_ENABLED = "XAGENT_KB_MILVUS_CLEANUP_ENABLED"TASKLESS_UPLOAD_TTL_SECONDS = "XAGENT_TASKLESS_UPLOAD_TTL_SECONDS"@@ -1868,6 +1870,16 @@ def get_background_job_sweep_interval_seconds() -> int:)+def get_kb_migration_defer_delay_seconds() -> int:+ """Return how long an ingest job waits before retrying a frozen collection."""+ return _get_positive_int_env(KB_MIGRATION_DEFER_DELAY_SECONDS, 15)+++def get_kb_migration_defer_budget_seconds() -> int:+ """Return the longest an ingest job defers for a collection migration."""+ return _get_positive_int_env(KB_MIGRATION_DEFER_BUDGET_SECONDS, 10 * 60)++def get_kb_milvus_cleanup_interval_seconds() -> int:"""Return how often the leftover Milvus rows of a Milvus deployment are purged."""return _get_positive_int_env(diff --git a/src/xagent/web/api/kb.py b/src/xagent/web/api/kb.pyindex b28609dd2..8d587114f 100644--- a/src/xagent/web/api/kb.py+++ b/src/xagent/web/api/kb.py@@ -2690,6 +2690,7 @@ def handle_kb_exceptions(func: T) -> T:"code": "kb_collection_migrating","message": e.message,},+ headers={"Retry-After": "15"},) from eexcept RollbackFailureError as e:logger.error("KB rollback failure in %s: %s", func.__name__, e)diff --git a/src/xagent/web/app.py b/src/xagent/web/app.pyindex 2d66f35b6..eeff30d7c 100644--- a/src/xagent/web/app.py+++ b/src/xagent/web/app.py@@ -1630,6 +1630,8 @@ app.add_middleware(allow_credentials=True,allow_methods=["*"],allow_headers=["*"],+ # The upload retry loop paces itself by this hint across origins.+ expose_headers=["Retry-After"],)app.add_middleware(PublicWidgetCORSMiddleware)app.add_middleware(SessionMiddleware, secret_key=get_session_secret(), same_site="lax")diff --git a/src/xagent/web/jobs/exceptions.py b/src/xagent/web/jobs/exceptions.pyindex 20223dbda..38f466897 100644--- a/src/xagent/web/jobs/exceptions.py+++ b/src/xagent/web/jobs/exceptions.py@@ -14,3 +14,11 @@ class BackgroundJobHandlerError(RuntimeError):super().__init__(message)self.result = resultself.retryable = retryable+++class BackgroundJobDeferred(RuntimeError):+ """Run the same job again later without spending its retry budget."""++ def __init__(self, message: str, *, delay_seconds: int) -> None:+ super().__init__(message)+ self.delay_seconds = max(1, int(delay_seconds))diff --git a/src/xagent/web/jobs/kb_tasks.py b/src/xagent/web/jobs/kb_tasks.pyindex 92981465f..d9abb78f9 100644--- a/src/xagent/web/jobs/kb_tasks.py+++ b/src/xagent/web/jobs/kb_tasks.py@@ -4,11 +4,16 @@ import asyncioimport hashlibimport loggingimport shutil+from datetime import datetime, timezonefrom pathlib import Pathfrom typing import Any, Literalfrom sqlalchemy.orm import Session+from ...config import (+ get_kb_migration_defer_budget_seconds,+ get_kb_migration_defer_delay_seconds,+)from ...core.tools.core.RAG_tools.core.exceptions import CollectionMigratingErrorfrom ...core.tools.core.RAG_tools.core.schemas import (IngestionConfig,@@ -46,13 +51,20 @@ from ..services.kb_ingest_targets import is_latest_kb_ingest_generationfrom ..services.kb_reference_protection import coordinate_ingest_jobfrom ..services.quota_hooks import get_standalone_gatefrom ..tracking.standalone_usage import StandaloneUsageRefused, usage_scope-from .exceptions import BackgroundJobHandlerError+from .exceptions import BackgroundJobDeferred, BackgroundJobHandlerErrorfrom .progress import BackgroundJobProgressManagerlogger = logging.getLogger(__name__)_SUPERSEDED_STAGED_INGEST_MESSAGE = "KB ingest job superseded by a newer upload"+# created_at also covers queue wait and stale requeues, so it cannot anchor+# the wait budget; a deferred job carries its first refusal here instead.+_MIGRATION_DEFERRED_AT_KEY = "_migration_deferred_at"+_MIGRATION_GIVE_UP_MESSAGE = (+ "Gave up waiting for the knowledge base upgrade; please re-submit."+)+class StagedDocumentIngestSuperseded(RuntimeError):pass@@ -67,6 +79,43 @@ def _payload_collection(db: Session, job: BackgroundJob) -> str | None:return str(collection) if collection is not None else None+def _defer_collection_migration(+ db: Session, job: BackgroundJob, exc: CollectionMigratingError+) -> BackgroundJobDeferred | None:+ now = datetime.now(timezone.utc)+ payload = dict(job.payload or {})+ first_deferral = payload.get(_MIGRATION_DEFERRED_AT_KEY)+ if first_deferral is None:+ first_at = now+ payload[_MIGRATION_DEFERRED_AT_KEY] = first_at.isoformat()+ setattr(job, "payload", payload)+ db.add(job)+ db.commit()+ else:+ first_at = datetime.fromisoformat(str(first_deferral))+ if first_at.tzinfo is None:+ first_at = first_at.replace(tzinfo=timezone.utc)+ remaining = get_kb_migration_defer_budget_seconds() - int(+ (now - first_at).total_seconds()+ )+ if remaining <= 0:+ return None+ return BackgroundJobDeferred(+ str(exc),+ delay_seconds=min(get_kb_migration_defer_delay_seconds(), remaining),+ )+++def _clear_migration_deferral(db: Session, job: BackgroundJob) -> None:+ payload = dict(job.payload or {})+ if _MIGRATION_DEFERRED_AT_KEY not in payload:+ return+ payload.pop(_MIGRATION_DEFERRED_AT_KEY)+ setattr(job, "payload", payload)+ db.add(job)+ db.commit()++def _collection_existed_before(payload: dict[str, Any]) -> bool:"""Read the flag stamped when the job was submitted.@@ -252,13 +301,18 @@ def handle_kb_ingest_document(db: Session, job: BackgroundJob) -> dict[str, Any]raise StagedDocumentIngestSuperseded(_SUPERSEDED_STAGED_INGEST_MESSAGE)# Before model resolution, which rewrites the collection's metadata row, and- # outside the try: a job refused here was never admitted and must not compensate.+ # outside the try: a refused job never compensates, only a give-up cleans up.try:ensure_collection_writable(str(payload["collection"]))- except CollectionMigratingError:- if int(job.attempts or 0) >= int(job.max_attempts or 1):- _cleanup_staged_document_input(payload)- raise+ except CollectionMigratingError as exc:+ deferred = _defer_collection_migration(db, job, exc)+ if deferred is not None:+ raise deferred from exc+ _cleanup_staged_document_input(payload)+ raise BackgroundJobHandlerError(+ _MIGRATION_GIVE_UP_MESSAGE, retryable=False+ ) from exc+ _clear_migration_deferral(db, job)try:model_pin, model_ids = _resolve_job_ingestion_models(payload, ingestion_config)# Billed to the member who submitted the job (the job's owner), not@@ -447,7 +501,16 @@ def handle_kb_ingest_web(db: Session, job: BackgroundJob) -> dict[str, Any]:db_session.close()update_job_progress(db, job, message="Crawling website")- ensure_collection_writable(collection)+ try:+ ensure_collection_writable(collection)+ except CollectionMigratingError as exc:+ deferred = _defer_collection_migration(db, job, exc)+ if deferred is not None:+ raise deferred from exc+ raise BackgroundJobHandlerError(+ _MIGRATION_GIVE_UP_MESSAGE, retryable=False+ ) from exc+ _clear_migration_deferral(db, job)try:model_pin, model_ids = _resolve_job_ingestion_models(payload, ingestion_config)with (diff --git a/src/xagent/web/jobs/tasks.py b/src/xagent/web/jobs/tasks.pyindex 80d48d9eb..7c28916fa 100644--- a/src/xagent/web/jobs/tasks.py+++ b/src/xagent/web/jobs/tasks.py@@ -13,12 +13,13 @@ from ..models.background_job import ()from ..models.database import get_session_local, init_dbfrom ..services.background_jobs import (+ QUEUE_DEFAULT,mark_job_failed,mark_job_running,mark_job_succeeded,)from .celery_app import celery_app-from .exceptions import BackgroundJobHandlerError+from .exceptions import BackgroundJobDeferred, BackgroundJobHandlerErrorlogger = logging.getLogger(__name__)@@ -129,6 +130,36 @@ def execute_background_job(self: Any, job_id: str) -> dict[str, Any]:mark_job_running(db, job)try:result = _execute_job_handler(db, job)+ except BackgroundJobDeferred as exc:+ db.refresh(job)+ # A deferral is not a failed attempt; give the attempt back so the+ # job's real retry budget survives a migration window.+ setattr(job, "attempts", max(0, int(job.attempts or 0) - 1))+ setattr(job, "status", BackgroundJobStatus.ENQUEUED.value)+ setattr(job, "error_message", None)+ setattr(+ job,+ "progress",+ {"message": str(exc), "completed": 0, "total": 1},+ )+ db.add(job)+ db.commit()+ db.refresh(job)+ if not celery_app.conf.task_always_eager:+ try:+ async_result = execute_background_job.apply_async(+ args=[job.id],+ queue=str(job.queue or QUEUE_DEFAULT),+ countdown=exc.delay_seconds,+ )+ except Exception: # noqa: BLE001+ # The stale sweep is the safety net; the row stays recoverable.+ logger.exception("Failed to reschedule deferred job %s", job.id)+ else:+ setattr(job, "celery_task_id", async_result.id)+ db.add(job)+ db.commit()+ return {"status": "deferred"}except BackgroundJobHandlerError as exc:db.refresh(job)if exc.retryable and int(job.attempts or 0) < int(job.max_attempts or 1):diff --git a/src/xagent/web/services/kb_reference_protection.py b/src/xagent/web/services/kb_reference_protection.pyindex 72f88c1f0..417f7e784 100644--- a/src/xagent/web/services/kb_reference_protection.py+++ b/src/xagent/web/services/kb_reference_protection.py@@ -364,7 +364,7 @@ def guard_upload_compensation(operation: _Operation) -> _Operation:def coordinate_ingest_job(operation: _Operation) -> _Operation:@wraps(operation)def ingest(db: Session, job: Any) -> Any:- from ..jobs.exceptions import BackgroundJobHandlerError+ from ..jobs.exceptions import BackgroundJobDeferred, BackgroundJobHandlerErrorfrom .kb_ingest_targets import release_kb_ingest_target_generationpayload = dict(job.payload or {})@@ -382,6 +382,9 @@ def coordinate_ingest_job(operation: _Operation) -> _Operation:result = operation(db, job)release = Truereturn result+ except BackgroundJobDeferred:+ # The job will run again: do not release its reservation.+ raiseexcept BackgroundJobHandlerError as exc:release = final_attempt or not exc.retryableraisediff --git a/tests/core/test_config.py b/tests/core/test_config.pyindex 3760a8d3c..cd5352529 100644--- a/tests/core/test_config.py+++ b/tests/core/test_config.py@@ -59,6 +59,8 @@ from xagent.config import (HOT_PATH_CACHE_TTL_SECONDS,HOT_PATH_TASK_CACHE_TTL_SECONDS,KB_COLLECTIONS_TIMEOUT_SECONDS,+ KB_MIGRATION_DEFER_BUDGET_SECONDS,+ KB_MIGRATION_DEFER_DELAY_SECONDS,KB_MILVUS_CLEANUP_ENABLED,KB_MILVUS_CLEANUP_INTERVAL_SECONDS,KB_SEARCH_TIMEOUT_SECONDS,@@ -175,6 +177,8 @@ from xagent.config import (get_hot_path_cache_ttl_seconds,get_hot_path_task_cache_ttl_seconds,get_kb_collections_timeout_seconds,+ get_kb_migration_defer_budget_seconds,+ get_kb_migration_defer_delay_seconds,get_kb_milvus_cleanup_enabled,get_kb_milvus_cleanup_interval_seconds,get_kb_search_timeout_seconds,@@ -523,6 +527,14 @@ class TestEnvironmentVariableConstants:BACKGROUND_JOB_SWEEP_INTERVAL_SECONDS== "XAGENT_BACKGROUND_JOB_SWEEP_INTERVAL_SECONDS")+ assert (+ KB_MIGRATION_DEFER_DELAY_SECONDS+ == "XAGENT_KB_MIGRATION_DEFER_DELAY_SECONDS"+ )+ assert (+ KB_MIGRATION_DEFER_BUDGET_SECONDS+ == "XAGENT_KB_MIGRATION_DEFER_BUDGET_SECONDS"+ )assert TRIGGER_DISPATCHER_ENABLED == "XAGENT_TRIGGER_DISPATCHER_ENABLED"assert (TRIGGER_DISPATCHER_INTERVAL_SECONDS@@ -881,6 +893,18 @@ class TestCeleryBackgroundJobConfig:assert get_background_job_stale_seconds() == 7200assert get_background_job_sweep_interval_seconds() == 300+ def test_kb_migration_defer_defaults(self, monkeypatch):+ monkeypatch.delenv(KB_MIGRATION_DEFER_DELAY_SECONDS, raising=False)+ monkeypatch.delenv(KB_MIGRATION_DEFER_BUDGET_SECONDS, raising=False)+ assert get_kb_migration_defer_delay_seconds() == 15+ assert get_kb_migration_defer_budget_seconds() == 600++ def test_kb_migration_defer_env_overrides(self, monkeypatch):+ monkeypatch.setenv(KB_MIGRATION_DEFER_DELAY_SECONDS, "20")+ monkeypatch.setenv(KB_MIGRATION_DEFER_BUDGET_SECONDS, "1200")+ assert get_kb_migration_defer_delay_seconds() == 20+ assert get_kb_migration_defer_budget_seconds() == 1200+def test_kb_milvus_cleanup_interval_default(self, monkeypatch):monkeypatch.delenv(KB_MILVUS_CLEANUP_INTERVAL_SECONDS, raising=False)assert get_kb_milvus_cleanup_interval_seconds() == 3600diff --git a/tests/web/api/test_kb_cloud_rollback_contract.py b/tests/web/api/test_kb_cloud_rollback_contract.pyindex dfb97b2e4..12aebd09f 100644--- a/tests/web/api/test_kb_cloud_rollback_contract.py+++ b/tests/web/api/test_kb_cloud_rollback_contract.py@@ -10,6 +10,7 @@ from __future__ import annotationsimport asyncioimport inspectfrom contextlib import ExitStack+from datetime import datetime, timedelta, timezonefrom pathlib import Pathfrom types import SimpleNamespacefrom typing import Any, Optional@@ -577,20 +578,27 @@ def test_staged_job_rollback_runs_inside_admission_after_mid_job_freeze(assert str(err) == "ingestion failed"[email protected]("job_type", ["document", "web"])+ ("job_type", "give_up"),+ [("document", False), ("document", True), ("web", False), ("web", True)],+)def test_frozen_job_refuses_before_model_resolution(- test_env, tmp_path, monkeypatch, job_type+ test_env, tmp_path, monkeypatch, job_type, give_up) -> None:- from xagent.core.tools.core.RAG_tools.core.exceptions import (- CollectionMigratingError,- )from xagent.core.tools.core.RAG_tools.management.migration_state import (FROZEN,CollectionMigrationState,clear_migration_state,write_migration_state,)- from xagent.web.jobs.kb_tasks import handle_kb_ingest_web+ from xagent.web.jobs.exceptions import (+ BackgroundJobDeferred,+ BackgroundJobHandlerError,+ )+ from xagent.web.jobs.kb_tasks import (+ _MIGRATION_DEFERRED_AT_KEY,+ handle_kb_ingest_web,+ )resolved: list[str] = []monkeypatch.setattr(@@ -598,6 +606,9 @@ def test_frozen_job_refuses_before_model_resolution(lambda *a, **k: resolved.append("called"),)_, _, user, session_local = test_env+ staged = tmp_path / "stage" / "doc.txt"+ staged.parent.mkdir(parents=True)+ staged.write_text("staged", encoding="utf-8")payload: dict[str, Any] = {"collection": "existing-kb","user_id": int(user.id),@@ -605,8 +616,13 @@ def test_frozen_job_refuses_before_model_resolution("ingestion_config": IngestionConfig().model_dump(mode="json"),"collection_existed_before": True,}+ if give_up:+ payload[_MIGRATION_DEFERRED_AT_KEY] = (+ datetime.now(timezone.utc) - timedelta(minutes=11)+ ).isoformat()if job_type == "document":handler, kind = handle_kb_ingest_document, BackgroundJobType.KB_INGEST_DOCUMENT+ payload["source_path"] = str(staged)payload["target_path"] = str(tmp_path / "doc.txt")else:handler, kind = handle_kb_ingest_web, BackgroundJobType.KB_INGEST_WEB@@ -617,24 +633,27 @@ def test_frozen_job_refuses_before_model_resolution(job = create_background_job(db, user_id=int(user.id), job_type=kind, payload=payload)- with pytest.raises(CollectionMigratingError):+ expected_error = BackgroundJobHandlerError if give_up else BackgroundJobDeferred+ with pytest.raises(expected_error) as raised:handler(db, job)finally:clear_migration_state("existing-kb")db.close()assert resolved == []+ if give_up:+ assert raised.value.retryable is False+ if job_type == "document":+ # Deferred keeps the staged copy; only giving up removes it.+ assert staged.exists() is not give_up@pytest.mark.parametrize(- ("frozen", "final"), [(True, True), (True, False), (False, True)]+ ("frozen", "give_up"), [(True, False), (True, True), (False, False)])def test_final_attempt_refusal_does_not_compensate_an_existing_document(- test_env, tmp_path, monkeypatch, frozen, final+ test_env, tmp_path, monkeypatch, frozen, give_up) -> None:- from xagent.core.tools.core.RAG_tools.core.exceptions import (- CollectionMigratingError,- )from xagent.core.tools.core.RAG_tools.management.migration_state import (FROZEN,CollectionMigrationState,@@ -648,6 +667,14 @@ def test_final_attempt_refusal_does_not_compensate_an_existing_document(from xagent.core.tools.core.RAG_tools.utils.string_utils import (generate_deterministic_doc_id,)+ from xagent.web.jobs.exceptions import (+ BackgroundJobDeferred,+ BackgroundJobHandlerError,+ )+ from xagent.web.jobs.kb_tasks import (+ _MIGRATION_DEFERRED_AT_KEY,+ _MIGRATION_GIVE_UP_MESSAGE,+ )_, _, user, session_local = test_envfile_id = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"@@ -668,42 +695,58 @@ def test_final_attempt_refusal_does_not_compensate_an_existing_document(db = session_local()if frozen:write_migration_state("existing-kb", CollectionMigrationState(state=FROZEN))+ payload: dict[str, Any] = {+ "collection": "existing-kb",+ "source_path": str(staged),+ "target_path": str(tmp_path / "canonical" / "doc.txt"),+ "file_id": file_id,+ "filename": "doc.txt",+ "user_id": int(user.id),+ "is_admin": False,+ "ingestion_config": IngestionConfig().model_dump(mode="json"),+ "collection_existed_before": True,+ "document_existed_before": True,+ }+ if give_up:+ payload[_MIGRATION_DEFERRED_AT_KEY] = (+ datetime.now(timezone.utc) - timedelta(minutes=11)+ ).isoformat()try:job = create_background_job(db,user_id=int(user.id),job_type=BackgroundJobType.KB_INGEST_DOCUMENT,- payload={- "collection": "existing-kb",- "source_path": str(staged),- "target_path": str(tmp_path / "canonical" / "doc.txt"),- "file_id": file_id,- "filename": "doc.txt",- "user_id": int(user.id),- "is_admin": False,- "ingestion_config": IngestionConfig().model_dump(mode="json"),- "collection_existed_before": True,- "document_existed_before": True,- },+ payload=payload,max_attempts=2,)- job.attempts = 2 if final else 1+ # Even the last allowed attempt is deferred while the budget holds: a+ # freeze is not a failed attempt.+ job.attempts = 2+ if frozen and not give_up:+ expected_error: type[Exception] = BackgroundJobDeferred+ elif frozen:+ expected_error = BackgroundJobHandlerError+ else:+ expected_error = RuntimeErrorwith (patch("xagent.web.api.kb._rollback_failed_cloud_ingestion",side_effect=_rollback,),- pytest.raises(CollectionMigratingError if frozen else RuntimeError),+ pytest.raises(expected_error) as raised,):handle_kb_ingest_document(db, job)finally:clear_migration_state("existing-kb")db.close()+ if frozen and give_up:+ assert raised.value.retryable is False+ assert str(raised.value) == _MIGRATION_GIVE_UP_MESSAGE# The unfrozen run is the positive control: this payload does reach compensation.assert compensated == ([] if frozen else ["existing-kb"])- # A retry still needs the staged copy; the last attempt must not leak it.- assert staged.exists() is not final+ # A deferred job still needs the staged copy; giving up or failing does not.+ assert staged.exists() is (frozen and not give_up)if frozen:rows = load_ingestion_status("existing-kb", doc_id, user_id=user.id)assert [r["status"] for r in rows] == ["success"]diff --git a/tests/web/api/test_kb_exception_decorator.py b/tests/web/api/test_kb_exception_decorator.pyindex 889348b54..fe5adae0f 100644--- a/tests/web/api/test_kb_exception_decorator.py+++ b/tests/web/api/test_kb_exception_decorator.py@@ -419,6 +419,7 @@ async def test_handle_kb_exceptions_maps_migration_freeze_to_409() -> None:assert exc_info.value.status_code == 409assert exc_info.value.detail == _MIGRATION_DETAIL+ assert exc_info.value.headers == {"Retry-After": "15"}@pytest.mark.asyncio@@ -454,6 +455,7 @@ async def test_ingest_route_maps_migration_freeze_to_409() -> None:assert exc_info.value.status_code == 409assert exc_info.value.detail == _MIGRATION_DETAIL+ assert exc_info.value.headers == {"Retry-After": "15"}@pytest.mark.asyncio@@ -484,6 +486,7 @@ async def test_ingest_cloud_route_maps_migration_freeze_to_409() -> None:assert exc_info.value.status_code == 409assert exc_info.value.detail == _MIGRATION_DETAIL+ assert exc_info.value.headers == {"Retry-After": "15"}@pytest.mark.asyncio@@ -514,6 +517,7 @@ async def test_ingest_web_route_maps_migration_freeze_to_409() -> None:assert exc_info.value.status_code == 409assert exc_info.value.detail == _MIGRATION_DETAIL+ assert exc_info.value.headers == {"Retry-After": "15"}@pytest.mark.asyncio@@ -553,6 +557,7 @@ async def test_create_ingest_job_maps_migration_freeze_to_409() -> None:assert exc_info.value.status_code == 409assert exc_info.value.detail == _MIGRATION_DETAIL+ assert exc_info.value.headers == {"Retry-After": "15"}class _FakeRollbackFacade:diff --git a/tests/web/test_api_cors.py b/tests/web/test_api_cors.pynew file mode 100644index 000000000..ca8308ba7--- /dev/null+++ b/tests/web/test_api_cors.py@@ -0,0 +1,16 @@+"""Cross-origin clients must be able to read the retry hint."""++from fastapi.testclient import TestClient++from xagent.config import get_cors_origins+from xagent.web.app import app+++def test_allowed_origin_exposes_retry_after() -> None:+ origin = get_cors_origins()[0]++ response = TestClient(app).get("/api/deployment-config", headers={"Origin": origin})++ assert response.status_code == 200+ assert response.headers["access-control-allow-origin"] == origin+ assert "Retry-After" in response.headers["access-control-expose-headers"]diff --git a/tests/web/test_background_jobs.py b/tests/web/test_background_jobs.pyindex 94027cd35..ef40d4d6e 100644--- a/tests/web/test_background_jobs.py+++ b/tests/web/test_background_jobs.py@@ -2120,3 +2120,590 @@ def test_kb_document_job_publishes_the_config_end_to_end(tmp_path, monkeypatch):assert '"chunk_size":1234' in saved_configfinally:db.close()+++def _freeze_collection(monkeypatch) -> None:+ from xagent.core.tools.core.RAG_tools.core.exceptions import (+ CollectionMigratingError,+ )++ def refuse(_collection: str) -> None:+ raise CollectionMigratingError("Knowledge base 'kb' is being upgraded")++ monkeypatch.setattr("xagent.web.jobs.kb_tasks.ensure_collection_writable", refuse)+++def _backdate_created_at(db, job, *, seconds: int) -> None:+ db.query(BackgroundJob).filter(BackgroundJob.id == job.id).update(+ {"created_at": datetime.now(timezone.utc) - timedelta(seconds=seconds)},+ synchronize_session=False,+ )+ db.commit()+ db.expire_all()+++def test_kb_document_job_defers_while_collection_is_migrating(tmp_path, monkeypatch):+ from xagent.web.jobs.exceptions import BackgroundJobDeferred+ from xagent.web.jobs.kb_tasks import (+ _MIGRATION_DEFERRED_AT_KEY,+ handle_kb_ingest_document,+ )+ from xagent.web.services.kb_ingest_targets import admit_kb_ingest_target++ monkeypatch.setenv(CELERY_ENABLED, "false")+ monkeypatch.delenv(CELERY_BROKER_URL, raising=False)+ _freeze_collection(monkeypatch)++ SessionLocal = _init_test_db(tmp_path / "kb-migration-defer.db")+ db = SessionLocal()+ try:+ user = _create_user(db, username="kb-migration-defer")+ staged_file = tmp_path / "stage" / "doc.txt"+ target_file = tmp_path / "canonical" / "doc.txt"+ staged_file.parent.mkdir(parents=True)+ staged_file.write_text("staged content", encoding="utf-8")+ file_id = "22222222-2222-4222-8222-222222222222"+ generation_id = "33333333-3333-4333-8333-333333333333"+ job = create_background_job(+ db,+ user_id=int(user.id),+ job_type=BackgroundJobType.KB_INGEST_DOCUMENT,+ payload={+ "collection": "kb",+ "source_path": str(staged_file),+ "target_path": str(target_file),+ "file_id": file_id,+ "generation_id": generation_id,+ "filename": "doc.txt",+ "mime_type": "text/plain",+ "file_size": staged_file.stat().st_size,+ "user_id": int(user.id),+ "is_admin": False,+ "ingestion_config": IngestionConfig().model_dump(mode="json"),+ "collection_existed_before": True,+ },+ )+ admit_kb_ingest_target(+ db,+ user_id=int(user.id),+ collection="kb",+ target_path=str(target_file),+ file_id=file_id,+ generation_id=generation_id,+ job_id=str(job.id),+ file_sha256="0" * 64,+ )+ # Even on the last allowed attempt a deferral keeps the reservation:+ # the job will run again, and the freeze is not the job's failure.+ setattr(job, "attempts", int(job.max_attempts or 3))+ db.commit()+ db.refresh(job)+ # A stale requeue can run a job hours after it was created; the wait+ # budget must still start when the freeze is first hit, not at creation.+ _backdate_created_at(db, job, seconds=2 * 3600)++ releases: list[dict] = []+ monkeypatch.setattr(+ "xagent.web.services.kb_ingest_targets.release_kb_ingest_target_generation",+ lambda *args, **kwargs: releases.append(kwargs) or True,+ )+ ingest_calls: list[dict] = []+ monkeypatch.setattr(+ "xagent.web.jobs.kb_tasks.run_document_ingestion",+ lambda **kwargs: ingest_calls.append(kwargs),+ )+ cleaned_staging: list[object] = []+ monkeypatch.setattr(+ "xagent.web.api.kb._cleanup_background_ingest_staging_file",+ lambda path: cleaned_staging.append(path),+ )++ with pytest.raises(BackgroundJobDeferred) as raised:+ handle_kb_ingest_document(db, job)++ assert raised.value.delay_seconds == 15+ # A refused job reruns later: its staged input and reservation stay.+ assert not cleaned_staging+ assert not releases+ assert not ingest_calls+ assert staged_file.read_text(encoding="utf-8") == "staged content"+ assert not target_file.exists()+ db.refresh(job)+ assert job.payload[_MIGRATION_DEFERRED_AT_KEY]+ finally:+ db.close()+++def test_kb_document_job_gives_up_when_migration_budget_is_spent(tmp_path, monkeypatch):+ from xagent.web.jobs.exceptions import BackgroundJobHandlerError+ from xagent.web.jobs.kb_tasks import (+ _MIGRATION_DEFERRED_AT_KEY,+ _MIGRATION_GIVE_UP_MESSAGE,+ handle_kb_ingest_document,+ )+ from xagent.web.services.kb_ingest_targets import admit_kb_ingest_target++ monkeypatch.setenv(CELERY_ENABLED, "false")+ monkeypatch.delenv(CELERY_BROKER_URL, raising=False)+ _freeze_collection(monkeypatch)++ SessionLocal = _init_test_db(tmp_path / "kb-migration-give-up.db")+ db = SessionLocal()+ try:+ user = _create_user(db, username="kb-migration-give-up")+ staged_file = tmp_path / "stage" / "doc.txt"+ target_file = tmp_path / "canonical" / "doc.txt"+ staged_file.parent.mkdir(parents=True)+ staged_file.write_text("staged content", encoding="utf-8")+ file_id = "44444444-4444-4444-8444-444444444444"+ generation_id = "55555555-5555-4555-8555-555555555555"+ job = create_background_job(+ db,+ user_id=int(user.id),+ job_type=BackgroundJobType.KB_INGEST_DOCUMENT,+ payload={+ "collection": "kb",+ "source_path": str(staged_file),+ "target_path": str(target_file),+ "file_id": file_id,+ "generation_id": generation_id,+ "filename": "doc.txt",+ "mime_type": "text/plain",+ "file_size": staged_file.stat().st_size,+ "user_id": int(user.id),+ "is_admin": False,+ "ingestion_config": IngestionConfig().model_dump(mode="json"),+ "collection_existed_before": True,+ _MIGRATION_DEFERRED_AT_KEY: (+ datetime.now(timezone.utc) - timedelta(minutes=11)+ ).isoformat(),+ },+ )+ admit_kb_ingest_target(+ db,+ user_id=int(user.id),+ collection="kb",+ target_path=str(target_file),+ file_id=file_id,+ generation_id=generation_id,+ job_id=str(job.id),+ file_sha256="0" * 64,+ )+ db.commit()++ releases: list[dict] = []+ monkeypatch.setattr(+ "xagent.web.services.kb_ingest_targets.release_kb_ingest_target_generation",+ lambda *args, **kwargs: releases.append(kwargs) or True,+ )++ with pytest.raises(BackgroundJobHandlerError) as raised:+ handle_kb_ingest_document(db, job)++ assert raised.value.retryable is False+ assert str(raised.value) == _MIGRATION_GIVE_UP_MESSAGE+ # Giving up follows the terminal path: staged input removed, reservation+ # released so a later upload can take the target.+ assert not staged_file.exists()+ assert [call["generation_id"] for call in releases] == [generation_id]+ finally:+ db.close()+++def test_kb_web_job_defers_while_collection_is_migrating(tmp_path, monkeypatch):+ from xagent.web.jobs.exceptions import BackgroundJobDeferred+ from xagent.web.jobs.kb_tasks import handle_kb_ingest_web++ monkeypatch.setenv(CELERY_ENABLED, "false")+ monkeypatch.delenv(CELERY_BROKER_URL, raising=False)+ _freeze_collection(monkeypatch)++ SessionLocal = _init_test_db(tmp_path / "kb-web-migration-defer.db")+ db = SessionLocal()+ try:+ user = _create_user(db, username="kb-web-migration-defer")+ job = create_background_job(+ db,+ user_id=int(user.id),+ job_type=BackgroundJobType.KB_INGEST_WEB,+ payload={+ "collection": "web-kb",+ "crawl_config": WebCrawlConfig(+ start_url="https://example.com"+ ).model_dump(mode="json"),+ "ingestion_config": IngestionConfig().model_dump(mode="json"),+ "user_id": int(user.id),+ "is_admin": False,+ "collection_existed_before": False,+ },+ )++ with pytest.raises(BackgroundJobDeferred) as raised:+ handle_kb_ingest_web(db, job)++ assert raised.value.delay_seconds == 15+ finally:+ db.close()+++def test_execute_background_job_reschedules_a_deferred_job(tmp_path, monkeypatch):+ from xagent.web.jobs import tasks as tasks_module+ from xagent.web.jobs.exceptions import BackgroundJobDeferred++ monkeypatch.setenv(CELERY_ENABLED, "false")+ monkeypatch.delenv(CELERY_BROKER_URL, raising=False)+ job_type = "test.deferred_registered"++ SessionLocal = _init_test_db(tmp_path / "jobs-deferred.db")+ db = SessionLocal()+ try:+ user = _create_user(db, "registered-deferred")+ job = create_background_job(+ db, user_id=int(user.id), job_type=job_type, payload={}, max_attempts=3+ )+ job_id = str(job.id)+ finally:+ db.close()++ def deferring_handler(_session, _job):+ raise BackgroundJobDeferred(+ "Knowledge base is being upgraded", delay_seconds=15+ )++ tasks_module.register_background_job_handler(job_type, deferring_handler)+ rescheduled: list[dict] = []++ def fake_apply_async(*, args, queue, countdown):+ rescheduled.append({"args": args, "queue": queue, "countdown": countdown})+ return MagicMock(id="deferred-task-id")++ monkeypatch.setattr(+ tasks_module.execute_background_job, "apply_async", fake_apply_async+ )++ try:+ outcome = tasks_module.execute_background_job.apply(args=[job_id], throw=False)+ finally:+ tasks_module._EXTRA_HANDLERS.pop(job_type, None)++ assert outcome.result == {"status": "deferred"}+ assert rescheduled == [{"args": [job_id], "queue": "default", "countdown": 15}]++ verify_db = SessionLocal()+ try:+ refreshed = (+ verify_db.query(BackgroundJob).filter(BackgroundJob.id == job_id).first()+ )+ assert refreshed is not None+ assert refreshed.status == BackgroundJobStatus.ENQUEUED.value+ # The deferral must not spend the job's retry budget.+ assert refreshed.attempts == 0+ assert refreshed.error_message is None+ assert "upgraded" in refreshed.progress["message"]+ assert refreshed.celery_task_id == "deferred-task-id"+ finally:+ verify_db.close()+++def test_deferred_job_keeps_its_retry_budget_for_real_failures(tmp_path, monkeypatch):+ """A deferral returns the attempt, so a later real failure still retries."""+ from xagent.web.jobs import tasks as tasks_module+ from xagent.web.jobs.exceptions import (+ BackgroundJobDeferred,+ BackgroundJobHandlerError,+ )++ monkeypatch.setenv(CELERY_ENABLED, "false")+ monkeypatch.delenv(CELERY_BROKER_URL, raising=False)+ job_type = "test.deferred_then_failing"++ SessionLocal = _init_test_db(tmp_path / "jobs-deferred-budget.db")+ db = SessionLocal()+ try:+ user = _create_user(db, "deferred-budget")+ job = create_background_job(+ db, user_id=int(user.id), job_type=job_type, payload={}, max_attempts=2+ )+ job_id = str(job.id)+ finally:+ db.close()++ runs: list[int] = []++ def handler(_session, _job):+ runs.append(len(runs) + 1)+ if len(runs) == 1:+ raise BackgroundJobDeferred(+ "Knowledge base is being upgraded", delay_seconds=15+ )+ raise BackgroundJobHandlerError("ingest failed", retryable=True)++ tasks_module.register_background_job_handler(job_type, handler)+ monkeypatch.setattr(+ tasks_module.execute_background_job,+ "apply_async",+ lambda **_kwargs: MagicMock(id="deferred-task-id"),+ )+ retry_calls: list[dict] = []++ def fake_retry(*_args, **kwargs):+ retry_calls.append(kwargs)+ return RuntimeError("retry requested")++ monkeypatch.setattr(tasks_module.execute_background_job, "retry", fake_retry)++ try:+ tasks_module.execute_background_job.apply(args=[job_id], throw=False)+ tasks_module.execute_background_job.apply(args=[job_id], throw=False)+ finally:+ tasks_module._EXTRA_HANDLERS.pop(job_type, None)++ assert len(runs) == 2+ assert len(retry_calls) == 1+ assert retry_calls[0]["exc"].retryable is True++ verify_db = SessionLocal()+ try:+ refreshed = (+ verify_db.query(BackgroundJob).filter(BackgroundJob.id == job_id).first()+ )+ assert refreshed is not None+ # The deferral spent nothing: the failed rerun is attempt 1 of 2.+ assert refreshed.attempts == 1+ assert refreshed.status == BackgroundJobStatus.ENQUEUED.value+ finally:+ verify_db.close()+++def test_deferred_job_survives_a_failed_reschedule(tmp_path, monkeypatch):+ from xagent.web.jobs import tasks as tasks_module+ from xagent.web.jobs.exceptions import BackgroundJobDeferred++ monkeypatch.setenv(CELERY_ENABLED, "false")+ monkeypatch.delenv(CELERY_BROKER_URL, raising=False)+ job_type = "test.deferred_reschedule_failure"++ SessionLocal = _init_test_db(tmp_path / "jobs-deferred-broker-down.db")+ db = SessionLocal()+ try:+ user = _create_user(db, "deferred-broker-down")+ job = create_background_job(+ db, user_id=int(user.id), job_type=job_type, payload={}, max_attempts=3+ )+ job_id = str(job.id)+ finally:+ db.close()++ def handler(_session, _job):+ raise BackgroundJobDeferred(+ "Knowledge base is being upgraded", delay_seconds=15+ )++ def failing_apply_async(**_kwargs):+ raise RuntimeError("broker down")++ tasks_module.register_background_job_handler(job_type, handler)+ monkeypatch.setattr(+ tasks_module.execute_background_job, "apply_async", failing_apply_async+ )++ try:+ outcome = tasks_module.execute_background_job.apply(args=[job_id], throw=False)+ finally:+ tasks_module._EXTRA_HANDLERS.pop(job_type, None)++ assert outcome.result == {"status": "deferred"}++ verify_db = SessionLocal()+ try:+ refreshed = (+ verify_db.query(BackgroundJob).filter(BackgroundJob.id == job_id).first()+ )+ assert refreshed is not None+ # The row stays recoverable for the stale sweep instead of being failed.+ assert refreshed.status == BackgroundJobStatus.ENQUEUED.value+ assert refreshed.attempts == 0+ finally:+ verify_db.close()+++def test_admission_clears_the_migration_anchor_so_a_later_freeze_waits(+ tmp_path, monkeypatch+):+ from xagent.core.tools.core.RAG_tools.core.exceptions import (+ CollectionMigratingError,+ )+ from xagent.web.jobs.exceptions import BackgroundJobDeferred+ from xagent.web.jobs.kb_tasks import (+ _MIGRATION_DEFERRED_AT_KEY,+ handle_kb_ingest_document,+ )++ monkeypatch.setenv(CELERY_ENABLED, "false")+ monkeypatch.delenv(CELERY_BROKER_URL, raising=False)+ frozen = {"on": True}++ def guard(_collection: str) -> None:+ if frozen["on"]:+ raise CollectionMigratingError("Knowledge base is being upgraded")++ monkeypatch.setattr("xagent.web.jobs.kb_tasks.ensure_collection_writable", guard)+ monkeypatch.setattr(+ "xagent.web.jobs.kb_tasks._save_job_collection_config_after_ingest",+ lambda *args, **kwargs: None,+ )+ monkeypatch.setattr(+ "xagent.web.jobs.kb_tasks.run_document_ingestion",+ lambda **kwargs: IngestionResult(+ status="success",+ doc_id="doc-1",+ chunk_count=1,+ message="ok",+ completed_steps=[+ {"name": "register_document", "metadata": {"created": True}}+ ],+ ),+ )++ SessionLocal = _init_test_db(tmp_path / "kb-anchor-cleared.db")+ db = SessionLocal()+ try:+ user = _create_user(db, username="kb-anchor-cleared")+ staged_file = tmp_path / "stage" / "doc.txt"+ target_file = tmp_path / "canonical" / "doc.txt"+ staged_file.parent.mkdir(parents=True)+ staged_file.write_text("staged content", encoding="utf-8")+ job = create_background_job(+ db,+ user_id=int(user.id),+ job_type=BackgroundJobType.KB_INGEST_DOCUMENT,+ payload={+ "collection": "kb",+ "source_path": str(staged_file),+ "target_path": str(target_file),+ "file_id": "66666666-6666-4666-8666-666666666666",+ "filename": "doc.txt",+ "mime_type": "text/plain",+ "file_size": staged_file.stat().st_size,+ "user_id": int(user.id),+ "is_admin": False,+ "ingestion_config": IngestionConfig().model_dump(mode="json"),+ "collection_existed_before": True,+ },+ )++ with pytest.raises(BackgroundJobDeferred):+ handle_kb_ingest_document(db, job)+ db.refresh(job)+ assert job.payload[_MIGRATION_DEFERRED_AT_KEY]++ # The first refusal happened long ago; if the anchor survived admission,+ # the next freeze would give up instead of waiting.+ payload = dict(job.payload)+ payload[_MIGRATION_DEFERRED_AT_KEY] = (+ datetime.now(timezone.utc) - timedelta(minutes=11)+ ).isoformat()+ setattr(job, "payload", payload)+ db.commit()++ frozen["on"] = False+ handle_kb_ingest_document(db, job)+ db.refresh(job)+ assert _MIGRATION_DEFERRED_AT_KEY not in job.payload++ frozen["on"] = True+ with pytest.raises(BackgroundJobDeferred) as raised:+ handle_kb_ingest_document(db, job)+ assert raised.value.delay_seconds == 15+ finally:+ db.close()+++def test_kb_web_job_clears_the_anchor_once_admitted(tmp_path, monkeypatch):+ from xagent.web.jobs.kb_tasks import (+ _MIGRATION_DEFERRED_AT_KEY,+ handle_kb_ingest_web,+ )++ monkeypatch.setenv(CELERY_ENABLED, "false")+ monkeypatch.delenv(CELERY_BROKER_URL, raising=False)+ monkeypatch.setattr(+ "xagent.web.jobs.kb_tasks._resolve_job_ingestion_models",+ lambda *args, **kwargs: (_ for _ in ()).throw(+ RuntimeError("stop after admission")+ ),+ )++ SessionLocal = _init_test_db(tmp_path / "web-anchor-cleared.db")+ db = SessionLocal()+ try:+ user = _create_user(db, username="web-anchor-cleared")+ job = create_background_job(+ db,+ user_id=int(user.id),+ job_type=BackgroundJobType.KB_INGEST_WEB,+ payload={+ "collection": "web-kb",+ "crawl_config": WebCrawlConfig(+ start_url="https://example.com"+ ).model_dump(mode="json"),+ "ingestion_config": IngestionConfig().model_dump(mode="json"),+ "user_id": int(user.id),+ "is_admin": False,+ "collection_existed_before": False,+ _MIGRATION_DEFERRED_AT_KEY: (+ datetime.now(timezone.utc) - timedelta(minutes=11)+ ).isoformat(),+ },+ )++ with pytest.raises(RuntimeError, match="stop after admission"):+ handle_kb_ingest_web(db, job)+ db.refresh(job)+ assert _MIGRATION_DEFERRED_AT_KEY not in job.payload+ finally:+ db.close()+++def test_kb_web_job_gives_up_with_a_terminal_message(tmp_path, monkeypatch):+ from xagent.web.jobs.exceptions import BackgroundJobHandlerError+ from xagent.web.jobs.kb_tasks import (+ _MIGRATION_DEFERRED_AT_KEY,+ _MIGRATION_GIVE_UP_MESSAGE,+ handle_kb_ingest_web,+ )++ monkeypatch.setenv(CELERY_ENABLED, "false")+ monkeypatch.delenv(CELERY_BROKER_URL, raising=False)+ _freeze_collection(monkeypatch)++ SessionLocal = _init_test_db(tmp_path / "web-give-up-message.db")+ db = SessionLocal()+ try:+ user = _create_user(db, username="web-give-up-message")+ job = create_background_job(+ db,+ user_id=int(user.id),+ job_type=BackgroundJobType.KB_INGEST_WEB,+ payload={+ "collection": "kb",+ "crawl_config": WebCrawlConfig(+ start_url="https://example.com"+ ).model_dump(mode="json"),+ "ingestion_config": IngestionConfig().model_dump(mode="json"),+ "user_id": int(user.id),+ "is_admin": False,+ "collection_existed_before": True,+ _MIGRATION_DEFERRED_AT_KEY: (+ datetime.now(timezone.utc) - timedelta(minutes=11)+ ).isoformat(),+ },+ )++ with pytest.raises(BackgroundJobHandlerError) as raised:+ handle_kb_ingest_web(db, job)++ assert raised.value.retryable is False+ assert str(raised.value) == _MIGRATION_GIVE_UP_MESSAGE+ finally:+ db.close()