OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
feat: improve ingest tracking events (#1933)
Mike Fortman08fe04eac504
diff --git a/frontend/app/chat/page.tsx b/frontend/app/chat/page.tsxindex af9f89857..3a0e43d62 100644--- a/frontend/app/chat/page.tsx+++ b/frontend/app/chat/page.tsx@@ -201,7 +201,7 @@ function ChatPage() {);if (result.type === "task") {- addTask(result.taskId);+ addTask(result.taskId, { source: "file" });return { type: "task-queued" as const };}diff --git a/frontend/app/connectors/page.tsx b/frontend/app/connectors/page.tsxindex 6f79f6d7d..63991bb43 100644--- a/frontend/app/connectors/page.tsx+++ b/frontend/app/connectors/page.tsx@@ -55,7 +55,10 @@ export default function ConnectorsPage() {if (response.status === 201) {const taskId = result.task_id;if (taskId) {- addTask(taskId, { connectorType: connector.type });+ addTask(taskId, {+ connectorType: connector.type,+ source: "connector",+ });setSyncResult({processed: 0,total: selectedFiles.length,diff --git a/frontend/app/upload/[provider]/page.tsx b/frontend/app/upload/[provider]/page.tsxindex 8306192eb..d908a9fac 100644--- a/frontend/app/upload/[provider]/page.tsx+++ b/frontend/app/upload/[provider]/page.tsx@@ -18,6 +18,7 @@ import {} from "@/components/ui/tooltip";import { useTask } from "@/contexts/task-context";import { useSessionIngestSettings } from "@/hooks/useSessionIngestSettings";+import { trackProcessFailure, trackStartProcess } from "@/lib/analytics";import { getConnectorDescriptor } from "@/lib/connectors/registry";// CloudFile interface is now imported from the unified cloud picker@@ -91,6 +92,14 @@ export default function UploadProviderPage() {files: CloudFile[],replaceDuplicates: boolean,) => {+ trackStartProcess({+ processType: "Ingestion",+ process: "Document Upload",+ category: "Knowledge",+ source: "connector",+ connector_type: connector.type,+ total_files: files.length,+ });syncMutation.mutate({connectorType: connector.type,@@ -112,11 +121,22 @@ export default function UploadProviderPage() {onSuccess: (result) => {const taskIds = result.task_ids;if (taskIds && taskIds.length > 0) {- addTask(taskIds[0], { connectorType: connector.type });+ addTask(taskIds[0], {+ connectorType: connector.type,+ source: "connector",+ });router.push("/knowledge");}},onError: (err) => {+ trackProcessFailure({+ processType: "Ingestion",+ process: "Document Upload",+ category: "Knowledge",+ source: "connector",+ connector_type: connector.type,+ resultValue: err instanceof Error ? err.message : "Sync failed",+ });toast.error(err instanceof Error ? err.message : "Sync failed");},},diff --git a/frontend/components/connectors/shared-bucket-view.tsx b/frontend/components/connectors/shared-bucket-view.tsxindex 3cd5d89b9..e6b91b94e 100644--- a/frontend/components/connectors/shared-bucket-view.tsx+++ b/frontend/components/connectors/shared-bucket-view.tsx@@ -12,6 +12,7 @@ import { FileBrowserDialog } from "@/components/file-browser-dialog";import { Button } from "@/components/ui/button";import { useAuth } from "@/contexts/auth-context";import { useSessionIngestSettings } from "@/hooks/useSessionIngestSettings";+import { trackProcessFailure, trackStartProcess } from "@/lib/analytics";export interface SharedBucketViewProps {connector: any;@@ -21,7 +22,10 @@ export interface SharedBucketViewProps {onRefetch: () => void;invalidateQueryKey: readonly unknown[];syncMutation: ReturnType<typeof useSyncConnector>;- addTask: (id: string, options?: { connectorType?: string }) => void;+ addTask: (+ id: string,+ options?: { connectorType?: string; source?: string },+ ) => void;onBack: () => void;onDone: () => void;}@@ -77,6 +81,14 @@ export function SharedBucketView({toast.error("Could not start ingest", { description: chunkErr });return;}+ trackStartProcess({+ processType: "Ingestion",+ process: "Document Upload",+ category: "Knowledge",+ source: "connector",+ connector_type: connector.type,+ total_buckets: selectedBuckets.size,+ });syncMutation.mutate({connectorType: connector.type,@@ -91,13 +103,24 @@ export function SharedBucketView({onSuccess: (result) => {invalidate();if (result.task_ids?.length) {- addTask(result.task_ids[0], { connectorType: connector.type });+ addTask(result.task_ids[0], {+ connectorType: connector.type,+ source: "connector",+ });onDone();} else {toast.info("No files found in the selected buckets.");}},onError: (err) => {+ trackProcessFailure({+ processType: "Ingestion",+ process: "Document Upload",+ category: "Knowledge",+ source: "connector",+ connector_type: connector.type,+ resultValue: err instanceof Error ? err.message : "Sync failed",+ });toast.error(err instanceof Error ? err.message : "Sync failed");},},diff --git a/frontend/components/knowledge-dropdown.tsx b/frontend/components/knowledge-dropdown.tsxindex c8338e8de..321996fcc 100644--- a/frontend/components/knowledge-dropdown.tsx+++ b/frontend/components/knowledge-dropdown.tsx@@ -35,6 +35,7 @@ import { useAuth } from "@/contexts/auth-context";import { useIsCloudBrand } from "@/contexts/brand-context";import { useTask } from "@/contexts/task-context";import { usePermissions } from "@/hooks/use-permissions";+import { trackProcessFailure, trackStartProcess } from "@/lib/analytics";import {getConnectorDescriptor,getConnectorDescriptors,@@ -333,11 +334,25 @@ export function KnowledgeDropdown() {const uploadFile = async (file: File, replace: boolean) => {setFileUploading(true);+ trackStartProcess({+ processType: "Ingestion",+ process: "Document Upload",+ category: "Knowledge",+ source: "file",+ total_files: 1,+ });try {await uploadFileUtil(file, replace);refetchTasks();} catch (error) {+ trackProcessFailure({+ processType: "Ingestion",+ process: "Document Upload",+ category: "Knowledge",+ source: "file",+ resultValue: error instanceof Error ? error.message : "Unknown error",+ });// Dispatch event that chat context can listen to// This avoids circular dependency issuesif (typeof window !== "undefined") {@@ -359,6 +374,14 @@ export function KnowledgeDropdown() {filesToUpload: File[],replace: boolean,) => {+ trackStartProcess({+ processType: "Ingestion",+ process: "Document Upload",+ category: "Knowledge",+ source: "folder",+ total_files: filesToUpload.length,+ });+const batches: File[][] = [];for (let i = 0; i < filesToUpload.length; i += uploadBatchSize) {batches.push(filesToUpload.slice(i, i + uploadBatchSize));@@ -371,8 +394,15 @@ export function KnowledgeDropdown() {for (const batch of batches) {try {const result = await uploadFiles(batch, replace);- addTask(result.taskId);+ addTask(result.taskId, { source: "folder" });} catch (error) {+ trackProcessFailure({+ processType: "Ingestion",+ process: "Document Upload",+ category: "Knowledge",+ source: "folder",+ resultValue: error instanceof Error ? error.message : "Unknown error",+ });console.error("[Folder Upload] Batch upload failed:", error);toast.error("Batch upload failed", {description: error instanceof Error ? error.message : "Unknown error",@@ -577,6 +607,12 @@ export function KnowledgeDropdown() {setFolderLoading(true);setShowFolderDialog(false);+ trackStartProcess({+ processType: "Ingestion",+ process: "Document Upload",+ category: "Knowledge",+ source: "path",+ });try {const response = await fetch("/api/upload_path", {@@ -596,7 +632,7 @@ export function KnowledgeDropdown() {throw new Error("No task ID received from server");}- addTask(taskId);+ addTask(taskId, { source: "path" });setFolderPath("");// Refetch tasks to show the new taskrefetchTasks();@@ -613,6 +649,13 @@ export function KnowledgeDropdown() {}}} catch (error) {+ trackProcessFailure({+ processType: "Ingestion",+ process: "Document Upload",+ category: "Knowledge",+ source: "path",+ resultValue: error instanceof Error ? error.message : "Unknown error",+ });console.error("Folder upload error:", error);} finally {setFolderLoading(false);diff --git a/frontend/contexts/task-context.tsx b/frontend/contexts/task-context.tsxindex 8471e3743..e462b89fb 100644--- a/frontend/contexts/task-context.tsx+++ b/frontend/contexts/task-context.tsx@@ -61,7 +61,10 @@ export interface TaskFile {interface TaskContextType {tasks: Task[];files: TaskFile[];- addTask: (taskId: string, options?: { connectorType?: string }) => void;+ addTask: (+ taskId: string,+ options?: { connectorType?: string; source?: string },+ ) => void;addFiles: (files: Partial<TaskFile>[], taskId: string) => void;/** Mark knowledge-table overlays as processing when a retry starts. */markTaskFilesProcessing: (taskId: string, sourceUrls: string[]) => void;@@ -100,17 +103,20 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {const [isTaskDialogOpen, setIsTaskDialogOpen] = useState(false);const previousTasksRef = useRef<Task[]>([]);const taskConnectorTypesRef = useRef<Map<string, string>>(new Map());+ const taskSourcesRef = useRef<Map<string, string>>(new Map());- const clearTaskConnectorType = useCallback((taskId: string) => {+ const clearTaskMetadata = useCallback((taskId: string) => {taskConnectorTypesRef.current.delete(taskId);+ taskSourcesRef.current.delete(taskId);}, []);- const clearTaskConnectorTypesWithoutOverlays = useCallback(+ const clearTaskMetadataWithoutOverlays = useCallback((prevFiles: TaskFile[], nextFiles: TaskFile[]) => {const nextTaskIds = new Set(nextFiles.map((file) => file.task_id));for (const file of prevFiles) {if (!nextTaskIds.has(file.task_id)) {taskConnectorTypesRef.current.delete(file.task_id);+ taskSourcesRef.current.delete(file.task_id);}}},@@ -157,7 +163,7 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {},);- clearTaskConnectorType(variables.taskId);+ clearTaskMetadata(variables.taskId);// Update file to display as cancelledsetFiles((prevFiles) =>@@ -248,7 +254,7 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {const currentTaskIds = new Set(tasks.map((task) => task.task_id));for (const previousTask of previousTasksRef.current) {if (!currentTaskIds.has(previousTask.task_id)) {- clearTaskConnectorType(previousTask.task_id);+ clearTaskMetadata(previousTask.task_id);}}@@ -428,6 +434,16 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {const failedFiles = getFailedFileCount(currentTask);const isTotalFailure = failedFiles > 0 && successfulFiles === 0;+ const firstFile = currentTask.files+ ? Object.values(currentTask.files)[0]+ : undefined;+ const embeddingModel = firstFile?.embedding_model;+ const connectorType =+ taskConnectorTypesRef.current.get(currentTask.task_id) || "local";+ const source =+ taskSourcesRef.current.get(currentTask.task_id) ||+ (connectorType === "local" ? "file" : "connector");+if (isTotalFailure) {trackProcessFailure({processType: "Ingestion",@@ -437,6 +453,9 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {total_files: currentTask.total_files,failed_files: failedFiles,duration_seconds: currentTask.duration_seconds,+ embedding_model: embeddingModel,+ connector_type: connectorType,+ source,});} else {trackProcessSuccess({@@ -448,6 +467,9 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {successful_files: successfulFiles,failed_files: failedFiles,duration_seconds: currentTask.duration_seconds,+ embedding_model: embeddingModel,+ connector_type: connectorType,+ source,});}@@ -541,7 +563,7 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {}}- clearTaskConnectorType(currentTask.task_id);+ clearTaskMetadata(currentTask.task_id);setFiles((prevFiles) =>prevFiles.filter((file) => {@@ -578,7 +600,7 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {}void refetchKnowledgeAfterTaskCompletion();} else if (taskJustReachedTerminal) {- clearTaskConnectorType(currentTask.task_id);+ clearTaskMetadata(currentTask.task_id);}if (@@ -632,16 +654,20 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {tasks,refetchSearch,isOnboardingActive,- clearTaskConnectorType,+ clearTaskMetadata,queryClient,]);const addTask = useCallback(- (taskId: string, options?: { connectorType?: string }) => {+ (taskId: string, options?: { connectorType?: string; source?: string }) => {const connectorType = options?.connectorType?.trim();if (connectorType) {taskConnectorTypesRef.current.set(taskId, connectorType);}+ const source = options?.source?.trim();+ if (source) {+ taskSourcesRef.current.set(taskId, source);+ }// React Query will automatically handle polling when tasks are active// Just trigger a refetch to get the latest datasetTimeout(() => {@@ -656,11 +682,11 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {const nextFiles = prevFiles.filter((file) => file.status !== "active" && file.status !== "failed",);- clearTaskConnectorTypesWithoutOverlays(prevFiles, nextFiles);+ clearTaskMetadataWithoutOverlays(prevFiles, nextFiles);return nextFiles;});await refetchTasks();- }, [refetchTasks, clearTaskConnectorTypesWithoutOverlays]);+ }, [refetchTasks, clearTaskMetadataWithoutOverlays]);const cancelTask = useCallback(async (taskId: string) => {diff --git a/frontend/lib/analytics.ts b/frontend/lib/analytics.tsindex 5851e88cf..64a2866d0 100644--- a/frontend/lib/analytics.ts+++ b/frontend/lib/analytics.ts@@ -70,6 +70,16 @@ export const trackButton = <T = Record<string, unknown>>({}: T & ButtonEventParams): void =>track("Button Clicked", { action, ...rest } as Record<string, unknown>);+interface StartProcessParams {+ processType: string;+ process?: string;+ category?: string;+}++export const trackStartProcess = <T = Record<string, unknown>>(+ props: T & StartProcessParams,+): void => track("Started Process", props as Record<string, unknown>);+interface EndProcessParams {processType: string;process?: string;