Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat: improve ingest tracking events (#1933)

Mike Fortman08fe04eac504
diff --git a/frontend/app/chat/page.tsx b/frontend/app/chat/page.tsx
index af9f89857..3a0e43d62 100644
--- a/frontend/app/chat/page.tsx
+++ b/frontend/app/chat/page.tsx
@@ -201,7 +201,7 @@ function ChatPage() {
);
if (result.type === "task") {
- addTask(result.taskId);
+ addTask(result.taskId, { source: "file" });
return { type: "task-queued" as const };
}
diff --git a/frontend/app/connectors/page.tsx b/frontend/app/connectors/page.tsx
index 6f79f6d7d..63991bb43 100644
--- a/frontend/app/connectors/page.tsx
+++ b/frontend/app/connectors/page.tsx
@@ -55,7 +55,10 @@ export default function ConnectorsPage() {
if (response.status === 201) {
const taskId = result.task_id;
if (taskId) {
- addTask(taskId, { connectorType: connector.type });
+ addTask(taskId, {
+ connectorType: connector.type,
+ source: "connector",
+ });
setSyncResult({
processed: 0,
total: selectedFiles.length,
diff --git a/frontend/app/upload/[provider]/page.tsx b/frontend/app/upload/[provider]/page.tsx
index 8306192eb..d908a9fac 100644
--- a/frontend/app/upload/[provider]/page.tsx
+++ b/frontend/app/upload/[provider]/page.tsx
@@ -18,6 +18,7 @@ import {
} from "@/components/ui/tooltip";
import { useTask } from "@/contexts/task-context";
import { useSessionIngestSettings } from "@/hooks/useSessionIngestSettings";
+import { trackProcessFailure, trackStartProcess } from "@/lib/analytics";
import { getConnectorDescriptor } from "@/lib/connectors/registry";
// CloudFile interface is now imported from the unified cloud picker
@@ -91,6 +92,14 @@ export default function UploadProviderPage() {
files: CloudFile[],
replaceDuplicates: boolean,
) => {
+ trackStartProcess({
+ processType: "Ingestion",
+ process: "Document Upload",
+ category: "Knowledge",
+ source: "connector",
+ connector_type: connector.type,
+ total_files: files.length,
+ });
syncMutation.mutate(
{
connectorType: connector.type,
@@ -112,11 +121,22 @@ export default function UploadProviderPage() {
onSuccess: (result) => {
const taskIds = result.task_ids;
if (taskIds && taskIds.length > 0) {
- addTask(taskIds[0], { connectorType: connector.type });
+ addTask(taskIds[0], {
+ connectorType: connector.type,
+ source: "connector",
+ });
router.push("/knowledge");
}
},
onError: (err) => {
+ trackProcessFailure({
+ processType: "Ingestion",
+ process: "Document Upload",
+ category: "Knowledge",
+ source: "connector",
+ connector_type: connector.type,
+ resultValue: err instanceof Error ? err.message : "Sync failed",
+ });
toast.error(err instanceof Error ? err.message : "Sync failed");
},
},
diff --git a/frontend/components/connectors/shared-bucket-view.tsx b/frontend/components/connectors/shared-bucket-view.tsx
index 3cd5d89b9..e6b91b94e 100644
--- a/frontend/components/connectors/shared-bucket-view.tsx
+++ b/frontend/components/connectors/shared-bucket-view.tsx
@@ -12,6 +12,7 @@ import { FileBrowserDialog } from "@/components/file-browser-dialog";
import { Button } from "@/components/ui/button";
import { useAuth } from "@/contexts/auth-context";
import { useSessionIngestSettings } from "@/hooks/useSessionIngestSettings";
+import { trackProcessFailure, trackStartProcess } from "@/lib/analytics";
export interface SharedBucketViewProps {
connector: any;
@@ -21,7 +22,10 @@ export interface SharedBucketViewProps {
onRefetch: () => void;
invalidateQueryKey: readonly unknown[];
syncMutation: ReturnType<typeof useSyncConnector>;
- addTask: (id: string, options?: { connectorType?: string }) => void;
+ addTask: (
+ id: string,
+ options?: { connectorType?: string; source?: string },
+ ) => void;
onBack: () => void;
onDone: () => void;
}
@@ -77,6 +81,14 @@ export function SharedBucketView({
toast.error("Could not start ingest", { description: chunkErr });
return;
}
+ trackStartProcess({
+ processType: "Ingestion",
+ process: "Document Upload",
+ category: "Knowledge",
+ source: "connector",
+ connector_type: connector.type,
+ total_buckets: selectedBuckets.size,
+ });
syncMutation.mutate(
{
connectorType: connector.type,
@@ -91,13 +103,24 @@ export function SharedBucketView({
onSuccess: (result) => {
invalidate();
if (result.task_ids?.length) {
- addTask(result.task_ids[0], { connectorType: connector.type });
+ addTask(result.task_ids[0], {
+ connectorType: connector.type,
+ source: "connector",
+ });
onDone();
} else {
toast.info("No files found in the selected buckets.");
}
},
onError: (err) => {
+ trackProcessFailure({
+ processType: "Ingestion",
+ process: "Document Upload",
+ category: "Knowledge",
+ source: "connector",
+ connector_type: connector.type,
+ resultValue: err instanceof Error ? err.message : "Sync failed",
+ });
toast.error(err instanceof Error ? err.message : "Sync failed");
},
},
diff --git a/frontend/components/knowledge-dropdown.tsx b/frontend/components/knowledge-dropdown.tsx
index c8338e8de..321996fcc 100644
--- a/frontend/components/knowledge-dropdown.tsx
+++ b/frontend/components/knowledge-dropdown.tsx
@@ -35,6 +35,7 @@ import { useAuth } from "@/contexts/auth-context";
import { useIsCloudBrand } from "@/contexts/brand-context";
import { useTask } from "@/contexts/task-context";
import { usePermissions } from "@/hooks/use-permissions";
+import { trackProcessFailure, trackStartProcess } from "@/lib/analytics";
import {
getConnectorDescriptor,
getConnectorDescriptors,
@@ -333,11 +334,25 @@ export function KnowledgeDropdown() {
const uploadFile = async (file: File, replace: boolean) => {
setFileUploading(true);
+ trackStartProcess({
+ processType: "Ingestion",
+ process: "Document Upload",
+ category: "Knowledge",
+ source: "file",
+ total_files: 1,
+ });
try {
await uploadFileUtil(file, replace);
refetchTasks();
} catch (error) {
+ trackProcessFailure({
+ processType: "Ingestion",
+ process: "Document Upload",
+ category: "Knowledge",
+ source: "file",
+ resultValue: error instanceof Error ? error.message : "Unknown error",
+ });
// Dispatch event that chat context can listen to
// This avoids circular dependency issues
if (typeof window !== "undefined") {
@@ -359,6 +374,14 @@ export function KnowledgeDropdown() {
filesToUpload: File[],
replace: boolean,
) => {
+ trackStartProcess({
+ processType: "Ingestion",
+ process: "Document Upload",
+ category: "Knowledge",
+ source: "folder",
+ total_files: filesToUpload.length,
+ });
+
const batches: File[][] = [];
for (let i = 0; i < filesToUpload.length; i += uploadBatchSize) {
batches.push(filesToUpload.slice(i, i + uploadBatchSize));
@@ -371,8 +394,15 @@ export function KnowledgeDropdown() {
for (const batch of batches) {
try {
const result = await uploadFiles(batch, replace);
- addTask(result.taskId);
+ addTask(result.taskId, { source: "folder" });
} catch (error) {
+ trackProcessFailure({
+ processType: "Ingestion",
+ process: "Document Upload",
+ category: "Knowledge",
+ source: "folder",
+ resultValue: error instanceof Error ? error.message : "Unknown error",
+ });
console.error("[Folder Upload] Batch upload failed:", error);
toast.error("Batch upload failed", {
description: error instanceof Error ? error.message : "Unknown error",
@@ -577,6 +607,12 @@ export function KnowledgeDropdown() {
setFolderLoading(true);
setShowFolderDialog(false);
+ trackStartProcess({
+ processType: "Ingestion",
+ process: "Document Upload",
+ category: "Knowledge",
+ source: "path",
+ });
try {
const response = await fetch("/api/upload_path", {
@@ -596,7 +632,7 @@ export function KnowledgeDropdown() {
throw new Error("No task ID received from server");
}
- addTask(taskId);
+ addTask(taskId, { source: "path" });
setFolderPath("");
// Refetch tasks to show the new task
refetchTasks();
@@ -613,6 +649,13 @@ export function KnowledgeDropdown() {
}
}
} catch (error) {
+ trackProcessFailure({
+ processType: "Ingestion",
+ process: "Document Upload",
+ category: "Knowledge",
+ source: "path",
+ resultValue: error instanceof Error ? error.message : "Unknown error",
+ });
console.error("Folder upload error:", error);
} finally {
setFolderLoading(false);
diff --git a/frontend/contexts/task-context.tsx b/frontend/contexts/task-context.tsx
index 8471e3743..e462b89fb 100644
--- a/frontend/contexts/task-context.tsx
+++ b/frontend/contexts/task-context.tsx
@@ -61,7 +61,10 @@ export interface TaskFile {
interface TaskContextType {
tasks: Task[];
files: TaskFile[];
- addTask: (taskId: string, options?: { connectorType?: string }) => void;
+ addTask: (
+ taskId: string,
+ options?: { connectorType?: string; source?: string },
+ ) => void;
addFiles: (files: Partial<TaskFile>[], taskId: string) => void;
/** Mark knowledge-table overlays as processing when a retry starts. */
markTaskFilesProcessing: (taskId: string, sourceUrls: string[]) => void;
@@ -100,17 +103,20 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
const [isTaskDialogOpen, setIsTaskDialogOpen] = useState(false);
const previousTasksRef = useRef<Task[]>([]);
const taskConnectorTypesRef = useRef<Map<string, string>>(new Map());
+ const taskSourcesRef = useRef<Map<string, string>>(new Map());
- const clearTaskConnectorType = useCallback((taskId: string) => {
+ const clearTaskMetadata = useCallback((taskId: string) => {
taskConnectorTypesRef.current.delete(taskId);
+ taskSourcesRef.current.delete(taskId);
}, []);
- const clearTaskConnectorTypesWithoutOverlays = useCallback(
+ const clearTaskMetadataWithoutOverlays = useCallback(
(prevFiles: TaskFile[], nextFiles: TaskFile[]) => {
const nextTaskIds = new Set(nextFiles.map((file) => file.task_id));
for (const file of prevFiles) {
if (!nextTaskIds.has(file.task_id)) {
taskConnectorTypesRef.current.delete(file.task_id);
+ taskSourcesRef.current.delete(file.task_id);
}
}
},
@@ -157,7 +163,7 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
},
);
- clearTaskConnectorType(variables.taskId);
+ clearTaskMetadata(variables.taskId);
// Update file to display as cancelled
setFiles((prevFiles) =>
@@ -248,7 +254,7 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
const currentTaskIds = new Set(tasks.map((task) => task.task_id));
for (const previousTask of previousTasksRef.current) {
if (!currentTaskIds.has(previousTask.task_id)) {
- clearTaskConnectorType(previousTask.task_id);
+ clearTaskMetadata(previousTask.task_id);
}
}
@@ -428,6 +434,16 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
const failedFiles = getFailedFileCount(currentTask);
const isTotalFailure = failedFiles > 0 && successfulFiles === 0;
+ const firstFile = currentTask.files
+ ? Object.values(currentTask.files)[0]
+ : undefined;
+ const embeddingModel = firstFile?.embedding_model;
+ const connectorType =
+ taskConnectorTypesRef.current.get(currentTask.task_id) || "local";
+ const source =
+ taskSourcesRef.current.get(currentTask.task_id) ||
+ (connectorType === "local" ? "file" : "connector");
+
if (isTotalFailure) {
trackProcessFailure({
processType: "Ingestion",
@@ -437,6 +453,9 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
total_files: currentTask.total_files,
failed_files: failedFiles,
duration_seconds: currentTask.duration_seconds,
+ embedding_model: embeddingModel,
+ connector_type: connectorType,
+ source,
});
} else {
trackProcessSuccess({
@@ -448,6 +467,9 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
successful_files: successfulFiles,
failed_files: failedFiles,
duration_seconds: currentTask.duration_seconds,
+ embedding_model: embeddingModel,
+ connector_type: connectorType,
+ source,
});
}
@@ -541,7 +563,7 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
}
}
- clearTaskConnectorType(currentTask.task_id);
+ clearTaskMetadata(currentTask.task_id);
setFiles((prevFiles) =>
prevFiles.filter((file) => {
@@ -578,7 +600,7 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
}
void refetchKnowledgeAfterTaskCompletion();
} else if (taskJustReachedTerminal) {
- clearTaskConnectorType(currentTask.task_id);
+ clearTaskMetadata(currentTask.task_id);
}
if (
@@ -632,16 +654,20 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
tasks,
refetchSearch,
isOnboardingActive,
- clearTaskConnectorType,
+ clearTaskMetadata,
queryClient,
]);
const addTask = useCallback(
- (taskId: string, options?: { connectorType?: string }) => {
+ (taskId: string, options?: { connectorType?: string; source?: string }) => {
const connectorType = options?.connectorType?.trim();
if (connectorType) {
taskConnectorTypesRef.current.set(taskId, connectorType);
}
+ const source = options?.source?.trim();
+ if (source) {
+ taskSourcesRef.current.set(taskId, source);
+ }
// React Query will automatically handle polling when tasks are active
// Just trigger a refetch to get the latest data
setTimeout(() => {
@@ -656,11 +682,11 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
const nextFiles = prevFiles.filter(
(file) => file.status !== "active" && file.status !== "failed",
);
- clearTaskConnectorTypesWithoutOverlays(prevFiles, nextFiles);
+ clearTaskMetadataWithoutOverlays(prevFiles, nextFiles);
return nextFiles;
});
await refetchTasks();
- }, [refetchTasks, clearTaskConnectorTypesWithoutOverlays]);
+ }, [refetchTasks, clearTaskMetadataWithoutOverlays]);
const cancelTask = useCallback(
async (taskId: string) => {
diff --git a/frontend/lib/analytics.ts b/frontend/lib/analytics.ts
index 5851e88cf..64a2866d0 100644
--- a/frontend/lib/analytics.ts
+++ b/frontend/lib/analytics.ts
@@ -70,6 +70,16 @@ export const trackButton = <T = Record<string, unknown>>({
}: T & ButtonEventParams): void =>
track("Button Clicked", { action, ...rest } as Record<string, unknown>);
+interface StartProcessParams {
+ processType: string;
+ process?: string;
+ category?: string;
+}
+
+export const trackStartProcess = <T = Record<string, unknown>>(
+ props: T & StartProcessParams,
+): void => track("Started Process", props as Record<string, unknown>);
+
interface EndProcessParams {
processType: string;
process?: string;