OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
import type { Task, TaskFileEntry } from "@/app/api/queries/useGetTasksQuery";
import {
buildRowStatusLabel,
isFileCancelled,
normalizeFailurePhase,
} from "@/lib/task-error-display";
export const ALL_TASK_FILE_TYPES = "__all__";
export const ALL_TASK_STATUS_CATEGORIES = "__all__";
export type TaskFileStatusCategory =
| "completed"
| "warning"
| "system_error"
| "cancelled"
| "indexing";
export type TaskFileNameSort = "asc" | "desc";
export type TaskFileFilterOptions = {
search?: string;
fileType?: string | typeof ALL_TASK_FILE_TYPES;
statusCategory?: TaskFileStatusCategory | typeof ALL_TASK_STATUS_CATEGORIES;
task?: Task;
};
function getFileResultReason(fileInfo: TaskFileEntry): string | undefined {
const result = fileInfo.result;
if (!result || typeof result !== "object" || Array.isArray(result)) {
return undefined;
}
const reason = (result as { reason?: unknown }).reason;
return typeof reason === "string" ? reason : undefined;
}
export function isDeletedAtSourceFile(fileInfo: TaskFileEntry): boolean {
return getFileResultReason(fileInfo) === "deleted_at_source";
}
export function getDeletedAtSourceMessage(
fileInfo: TaskFileEntry,
): string | undefined {
if (!isDeletedAtSourceFile(fileInfo)) {
return undefined;
}
const message = (fileInfo.result as { message?: unknown }).message;
return typeof message === "string" && message.trim()
? message.trim()
: undefined;
}
function isTaskFileCompleted(fileInfo: TaskFileEntry): boolean {
return fileInfo.status === "completed" || isDeletedAtSourceFile(fileInfo);
}
export function isTaskFileFailed(fileInfo: TaskFileEntry): boolean {
return fileInfo.status === "failed" || fileInfo.status === "error";
}
export function isTaskFileWarning(fileInfo: TaskFileEntry): boolean {
// Source-deleted files are successful cleanup, not a skip warning.
return fileInfo.status === "skipped" && !isDeletedAtSourceFile(fileInfo);
}
export function getTaskFileDialogStatusLabel(
fileInfo: TaskFileEntry,
taskError?: string,
): string {
if (isTaskFileFailed(fileInfo)) {
// Use centralized cancellation detection
if (isFileCancelled(fileInfo)) {
return "Cancelled";
}
const failurePhase = normalizeFailurePhase(fileInfo.failure_phase);
if (failurePhase) {
return buildRowStatusLabel(failurePhase);
}
if (fileInfo.user_facing_message?.trim()) {
return "Failed";
}
return buildRowStatusLabel("unknown");
}
if (isDeletedAtSourceFile(fileInfo)) {
return "Removed";
}
if (isTaskFileWarning(fileInfo)) {
return "Warning";
}
if (isTaskFileCompleted(fileInfo)) {
return "Complete";
}
return "Processing";
}
export function getTaskFileName(
filePath: string,
fileInfo: TaskFileEntry,
): string {
return fileInfo.filename || filePath.split("/").pop() || filePath;
}
/** Lowercase extension without dot, or empty string when none. */
function getFileExtensionFromName(filename: string): string {
const trimmed = filename.trim();
const dotIndex = trimmed.lastIndexOf(".");
if (dotIndex <= 0 || dotIndex === trimmed.length - 1) {
return "";
}
return trimmed.slice(dotIndex + 1).toLowerCase();
}
function getTaskFileTypeKey(filePath: string, fileInfo: TaskFileEntry): string {
const extension = getFileExtensionFromName(
getTaskFileName(filePath, fileInfo),
);
return extension || "unknown";
}
export function getTaskFileEntries(task: Task): Array<[string, TaskFileEntry]> {
return Object.entries(task.files || {});
}
export function getTaskFileTypes(task: Task): string[] {
const types = new Set(
getTaskFileEntries(task).map(([path, entry]) =>
getTaskFileTypeKey(path, entry),
),
);
return Array.from(types).sort((a, b) => a.localeCompare(b));
}
export function formatTaskFileTypeLabel(fileType: string): string {
if (fileType === "unknown") {
return "Unknown";
}
return fileType.toUpperCase();
}
export function isTaskFileRetryable(fileInfo: TaskFileEntry): boolean {
return fileInfo.actionable_by === "RETRYABLE";
}
export function getRetryableFileEntries(
task: Task,
): Array<[string, TaskFileEntry]> {
return getTaskFileEntries(task).filter(([, fileInfo]) =>
isTaskFileRetryable(fileInfo),
);
}
export function getRetryableFilePaths(
entries: Array<[string, TaskFileEntry]>,
): string[] {
return entries
.filter(([, fileInfo]) => isTaskFileRetryable(fileInfo))
.map(([filePath]) => filePath);
}
export function countRetryIngestionFiles(task: Task): number {
return getTaskFileEntries(task).filter(([, fileInfo]) =>
isTaskFileRetryable(fileInfo),
).length;
}
/** Maps a file to a dialog filter chip bucket (aligned with per-file status labels). */
function getTaskFileStatusCategory(
fileInfo: TaskFileEntry,
): TaskFileStatusCategory {
if (isTaskFileFailed(fileInfo)) {
// Use centralized cancellation detection
if (isFileCancelled(fileInfo)) {
return "cancelled";
}
return "system_error";
}
if (isTaskFileWarning(fileInfo)) {
return "warning";
}
const status = fileInfo.status ?? "pending";
if (status === "pending" || status === "running" || status === "processing") {
return "indexing";
}
if (isTaskFileCompleted(fileInfo)) {
return "completed";
}
return "indexing";
}
export function countTaskFileEntriesByCategory(
entries: Array<[string, TaskFileEntry]>,
): Record<TaskFileStatusCategory, number> {
const counts: Record<TaskFileStatusCategory, number> = {
completed: 0,
warning: 0,
system_error: 0,
cancelled: 0,
indexing: 0,
};
for (const [, fileInfo] of entries) {
const category = getTaskFileStatusCategory(fileInfo);
counts[category] += 1;
}
return counts;
}
function _countTaskFilesByCategory(
task: Task,
): Record<TaskFileStatusCategory, number> {
return countTaskFileEntriesByCategory(getTaskFileEntries(task));
}
export function sortTaskFileEntries(
entries: Array<[string, TaskFileEntry]>,
direction: TaskFileNameSort = "asc",
): Array<[string, TaskFileEntry]> {
const sorted = entries.toSorted(([pathA, infoA], [pathB, infoB]) =>
getTaskFileName(pathA, infoA).localeCompare(
getTaskFileName(pathB, infoB),
undefined,
{ sensitivity: "base" },
),
);
return direction === "asc" ? sorted : sorted.reverse();
}
export function filterTaskFileEntries(
entries: Array<[string, TaskFileEntry]>,
options: TaskFileFilterOptions,
): Array<[string, TaskFileEntry]> {
const query = options.search?.trim().toLowerCase() ?? "";
const fileType = options.fileType ?? ALL_TASK_FILE_TYPES;
const statusCategory = options.statusCategory ?? ALL_TASK_STATUS_CATEGORIES;
return entries.filter(([filePath, fileInfo]) => {
if (fileType !== ALL_TASK_FILE_TYPES) {
const typeKey = getTaskFileTypeKey(filePath, fileInfo);
if (typeKey !== fileType) {
return false;
}
}
if (
statusCategory !== ALL_TASK_STATUS_CATEGORIES &&
getTaskFileStatusCategory(fileInfo) !== statusCategory
) {
return false;
}
if (query) {
const name = getTaskFileName(filePath, fileInfo);
if (!name.toLowerCase().includes(query)) {
return false;
}
}
return true;
});
}
function getFailedFileEntries(task: Task): Array<[string, TaskFileEntry]> {
return Object.entries(task.files || {}).filter(([, fileInfo]) =>
isTaskFileFailed(fileInfo),
);
}
export function getWarningFileEntries(
task: Task,
): Array<[string, TaskFileEntry]> {
return Object.entries(task.files || {}).filter(([, fileInfo]) =>
isTaskFileWarning(fileInfo),
);
}
export function getTaskIssueFileEntries(
task: Task,
): Array<[string, TaskFileEntry]> {
return Object.entries(task.files || {}).filter(
([, fileInfo]) => isTaskFileFailed(fileInfo) || isTaskFileWarning(fileInfo),
);
}
export function hasFailedFileEntries(task: Task): boolean {
if ((task.failed_files ?? 0) > 0) {
return true;
}
return getFailedFileEntries(task).length > 0;
}
export function hasIssueFileEntries(task: Task): boolean {
if ((task.failed_files ?? 0) > 0) {
return true;
}
return getTaskIssueFileEntries(task).length > 0;
}
export function isTerminalFailedTask(task: Task): boolean {
return task.status === "failed" || task.status === "error";
}
function isCompletedWithFailures(task: Task): boolean {
return task.status === "completed" && hasFailedFileEntries(task);
}
export function getSuccessfulFileCount(task: Task): number {
if (typeof task.successful_files === "number") {
// The backend counts skipped (duplicate) files as successful_files too.
// Subtract them so the toast only reports truly-indexed files as uploaded.
const skipped = getSkippedFileCount(task);
return Math.max(0, task.successful_files - skipped);
}
return Object.values(task.files || {}).filter((fileInfo) =>
fileInfo ? isTaskFileCompleted(fileInfo) : false,
).length;
}
export function getSkippedFileCount(task: Task): number {
return Object.values(task.files || {}).filter(
(fileInfo) => fileInfo?.status === "skipped",
).length;
}
/**
* Count only files skipped because of duplicate content (reason = "duplicate_content").
* Excludes other skip reasons such as "deleted_at_source" (connector cleanup).
*/
export function getDuplicateContentFileCount(task: Task): number {
return Object.values(task.files || {}).filter((fileInfo) => {
if (fileInfo?.status !== "skipped") return false;
const reason = (fileInfo.result as Record<string, unknown> | undefined)
?.reason;
return reason === "duplicate_content";
}).length;
}
export function getFailedFileCount(task: Task): number {
if (typeof task.failed_files === "number") {
return task.failed_files;
}
return getFailedFileEntries(task).length;
}
/** Completed task with failures and no successful files — treat as failed, not partial success. */
export function isCompletedTotalFailure(task: Task): boolean {
return isCompletedWithFailures(task) && getSuccessfulFileCount(task) === 0;
}
export function isFailureLikeTask(task: Task): boolean {
return isTerminalFailedTask(task) || isCompletedWithFailures(task);
}
export function isTaskInProgressStatus(status: Task["status"]): boolean {
return (
status === "pending" || status === "running" || status === "processing"
);
}
export function isTerminalTaskStatus(status: Task["status"]): boolean {
return (
status === "completed" ||
status === "failed" ||
status === "cancelled" ||
status === "error" ||
status === "skipped"
);
}
/** True when a task has just entered a terminal status. */
export function didTaskReachTerminalState(
previousTask: Task | undefined,
currentTask: Task,
): boolean {
return (
!!previousTask &&
!isTerminalTaskStatus(previousTask.status) &&
isTerminalTaskStatus(currentTask.status)
);
}
/** True when a task has just transitioned to completed. */
export function didTaskReachCompleted(
previousTask: Task | undefined,
currentTask: Task,
): boolean {
return (
!!previousTask &&
previousTask.status !== "completed" &&
currentTask.status === "completed"
);
}
/**
* File paths present on the previous enhanced-list payload but omitted now.
* The enhanced list drops completed files from `files` while a task is still running.
*/
/** Stable overlay key before the backend temp path is known. */
function pendingTaskFileSourceUrl(taskId: string, filename: string): string {
return `pending:${taskId}:${filename}`;
}
function isPendingTaskFileSourceUrl(sourceUrl: string): boolean {
return sourceUrl.startsWith("pending:");
}
export function findTaskFileOverlayIndex(
files: Array<{ task_id: string; source_url: string; filename: string }>,
taskId: string,
filePath: string,
fileName: string,
): number {
const pendingUrl = pendingTaskFileSourceUrl(taskId, fileName);
return files.findIndex(
(f) =>
f.task_id === taskId &&
(f.source_url === filePath ||
f.source_url === pendingUrl ||
(f.filename === fileName && isPendingTaskFileSourceUrl(f.source_url))),
);
}
export function getEnhancedListDisappearedFilePaths(
currentTask: Task,
previousTask: Task,
): string[] {
const currentKeys = new Set(Object.keys(currentTask.files ?? {}));
return Object.keys(previousTask.files ?? {}).filter(
(filePath) => !currentKeys.has(filePath),
);
}
interface ProcessingFileOverlay {
task_id: string;
source_url: string;
status: "active" | "failed" | "processing" | "cancelled" | "skipped";
error?: string;
}
/**
* Promote local processing overlays when the enhanced list omits completed files.
* Pass `disappearedPaths` while the task is in progress; omit it when the task completes
* to finalize every remaining processing file for that task.
*
* Source-deleted cleanups are completed-but-visible: drop their overlay instead
* of promoting it to `active`, or the knowledge table would show a ghost ingest.
*/
export function finalizeProcessingOverlaysForEnhancedTask<
T extends ProcessingFileOverlay,
>(prevFiles: T[], currentTask: Task, disappearedPaths?: string[]): T[] {
const pathsFilter =
disappearedPaths === undefined ? null : new Set(disappearedPaths);
let changed = false;
const updated: T[] = [];
for (const file of prevFiles) {
if (file.task_id !== currentTask.task_id) {
updated.push(file);
continue;
}
if (pathsFilter !== null && !pathsFilter.has(file.source_url)) {
updated.push(file);
continue;
}
// Overlays can still be "failed" until the list poll sees a retry as running.
if (file.status !== "processing" && file.status !== "failed") {
updated.push(file);
continue;
}
const entry = currentTask.files?.[file.source_url];
if (entry && isDeletedAtSourceFile(entry)) {
changed = true;
continue;
}
if (entry && isTaskFileFailed(entry)) {
if (file.status === "failed") {
const error =
typeof entry.error === "string" && entry.error.trim().length > 0
? entry.error.trim()
: file.error;
if (error === file.error) {
updated.push(file);
continue;
}
}
changed = true;
const error =
typeof entry.error === "string" && entry.error.trim().length > 0
? entry.error.trim()
: file.error;
updated.push({ ...file, status: "failed" as const, error });
continue;
}
// Left the enhanced list (completed files are omitted) or task finished.
changed = true;
updated.push({ ...file, status: "active" as const, error: undefined });
}
return changed ? updated : prevFiles;
}