LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
import { format, parseISO } from "date-fns";
import moment from "moment";
import momentTz from "moment-timezone";
import { v4 as uuidv4 } from "uuid";
let cloudHomePagePath;
try {
const mod = await import("../plugins/unstract-subscription/helper/constants");
cloudHomePagePath = mod.cloudHomePagePath;
} catch {
// Ignore if plugin not available
}
const THEME = {
DARK: "dark",
LIGHT: "light",
};
const calculateDivHeight = (offset) => {
const windowHeight = window.innerHeight;
const calculatedHeight = windowHeight - offset;
return calculatedHeight;
};
const publicRoutes = ["/landing"];
/*
This function will return the url without the path/routes
For ex,
If the url is http://localhost:3000/home,
then it will return http://localhost:3000/
*/
const getBaseUrl = () => {
const location = window.location.href;
const url = new URL(location).origin;
return url;
};
const getOrgNameFromPathname = (pathname, isLlmWhisperer) => {
if (!pathname) {
return null;
}
if (publicRoutes.includes(pathname)) {
return null;
}
if (isLlmWhisperer) {
return pathname?.split("/")[2];
}
return pathname?.split("/")[1];
};
const replaceFirstRoute = (url, newRoute) => {
// Regular expression to match the first route segment
const regex = /^(\/?[^/]+)/;
const match = url.match(regex);
// If there is a match, replace the first route with the new route
if (match) {
const oldRoute = match[1];
const replacedUrl = url.replace(oldRoute, newRoute);
return "/" + replacedUrl;
}
// Return the home page with the new route
return "/" + newRoute;
};
const formatBytes = (bytes, decimals = 1) => {
if (!+bytes) {
return "0 B";
}
const k = 1000;
const dm = decimals < 0 ? 0 : decimals;
const sizes = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return sizes[i]
? `${parseFloat((bytes / k ** i).toFixed(dm))} ${sizes[i]}`
: `${bytes / k ** (i - 1)} ${sizes.at(-1)}`;
};
const O_AUTH_PROVIDERS = {
GOOGLE: "google-oauth2",
MICROSOFT: "azuread-tenant-oauth2",
};
const CONNECTOR_TYPE_MAP = {
input: "Source",
output: "Destination",
};
const workflowStatus = {
yet_to_start: "YET TO START",
in_progress: "IN PROGRESS",
generated: "GENERATED",
};
const setInitialWorkflowInstance = (promptName, prompt) => {
return {
promptName,
prompt,
status: workflowStatus.yet_to_start,
details: {},
};
};
const wfExecutionTypes = ["START", "NEXT", "STOP", "CONTINUE", "RUN_WORKFLOW"];
const toolIdeOutput = [
{
company: "Apple Inc.",
stockholders_equity: "USD 33,400",
},
{
company: "Apple Inc.",
stockholders_equity: "USD 33,400",
},
{
company: "Apple Inc.",
stockholders_equity: "USD 33,400",
},
{
company: "Apple Inc.",
stockholders_equity: "USD 33,400",
},
{
company: "Apple Inc.",
stockholders_equity: "USD",
},
];
// TODO: Remove this once the BE API for it is ready.
const listOfAppDeployments = [
{
id: "08a47c30-e2ec-48b3-be45-5b1f06639031",
pipeline_name: "Financial document Q&A",
app_id: null,
active: true,
scheduled: false,
pipeline_type: "ETL",
run_count: 6,
last_run_time: "2023-07-28T08:09:31.045244Z",
last_run_status: "SUCCESS",
workflow: "73b57446-fafc-445e-96fe-f5d072044dcd",
cron: null,
workflow_name: "demo",
source_name: "MinioFS/S3",
source_icon: "/icons/connector-icons/S3.png",
destination_name: "Unstract Cloud Storage",
destination_icon: "/icons/connector-icons/Pandora%20Storage.png",
goto: "https://finance-qa.pandora-demo.zipstack.io/",
},
{
id: "08a47c30-e2ec-48b3-be45-5b1f06639032",
pipeline_name: "Legal document Q&A",
app_id: null,
active: true,
scheduled: false,
pipeline_type: "ETL",
run_count: 6,
last_run_time: "2023-07-28T08:09:31.045244Z",
last_run_status: "SUCCESS",
workflow: "73b57446-fafc-445e-96fe-f5d072044dcd",
cron: null,
workflow_name: "demo",
source_name: "MinioFS/S3",
source_icon: "/icons/connector-icons/S3.png",
destination_name: "Unstract Cloud Storage",
destination_icon: "/icons/connector-icons/Pandora%20Storage.png",
goto: "https://legal-qa.pandora-demo.zipstack.io/",
},
];
const getReadableDateAndTime = (timestamp, includeTime = true) => {
const currentDate = new Date(timestamp);
if (isNaN(currentDate)) {
return "Invalid date";
}
// Options for formatting the date
const dateOptions = { year: "numeric", month: "long", day: "numeric" };
const formattedDate = currentDate.toLocaleDateString("en-US", dateOptions);
if (!includeTime) {
return formattedDate;
}
// Options for formatting the time
const timeOptions = {
hour: "2-digit",
minute: "2-digit",
second: "2-digit",
timeZoneName: "short",
};
const formattedTime = currentDate.toLocaleTimeString("en-US", timeOptions);
return formattedDate + ", " + formattedTime;
};
const promptType = {
prompt: "PROMPT",
notes: "NOTES",
};
const sourceTypes = {
connectors: ["input", "output"],
adapters: ["llm", "vector_db", "embedding"],
};
const getSequenceNumber = (listOfPrompts) => {
let maxSequenceNumber = 0;
listOfPrompts.forEach((item) => {
if (item?.sequence_number > maxSequenceNumber) {
maxSequenceNumber = item?.sequence_number;
}
});
return maxSequenceNumber + 1;
};
const deploymentTypes = {
etl: "etl",
task: "task",
api: "api",
app: "app",
};
const deploymentApiTypes = {
api: "api",
pipeline: "pipeline",
};
const deploymentsStaticContent = {
etl: {
title: "Unstructured to Structured ETL Pipelines",
modalTitle: "Deploy ETL Pipeline",
addBtn: "ETL Pipeline",
},
task: {
title: "Unstructured to Structured Task Pipelines",
modalTitle: "Deploy Task Pipeline",
addBtn: "Task Pipeline",
},
api: {
title: "API Deployments",
addBtn: "API Deployment",
},
app: {
title: "App Deployments",
addBtn: "App Deployment",
},
};
const endpointType = {
input: "SOURCE",
output: "DESTINATION",
};
const promptStudioUpdateStatus = {
isUpdating: "IS_UPDATING",
done: "DONE",
validationError: "VALIDATION_ERROR",
};
const getTimeForLogs = () => {
const timestamp = Date.now();
const date = new Date(timestamp);
const hours = ("0" + date.getHours()).slice(-2);
const minutes = ("0" + date.getMinutes()).slice(-2);
const seconds = ("0" + date.getSeconds()).slice(-2);
const formattedDate = `${hours}:${minutes}:${seconds}`;
return formattedDate;
};
const getDateTimeString = (timestamp) => {
// Check if the timestamp is a valid number
if (typeof timestamp !== "number" || isNaN(timestamp) || timestamp <= 0) {
return timestamp;
}
const timestampInMilliseconds = timestamp * 1000;
const date = new Date(timestampInMilliseconds);
if (isNaN(date.getTime())) {
return timestamp;
}
const year = date.getFullYear();
const month = (date.getMonth() + 1).toString().padStart(2, "0"); // Months are zero-indexed
const day = date.getDate().toString().padStart(2, "0");
const hours = date.getHours().toString().padStart(2, "0");
const minutes = date.getMinutes().toString().padStart(2, "0");
const seconds = date.getSeconds().toString().padStart(2, "0");
const milliseconds = date.getMilliseconds().toString().padStart(3, "0");
// Formatted date-time string
return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}.${milliseconds}`;
};
const base64toBlob = (data, mimeType) => {
const bytes = atob(data);
let length = bytes.length;
const out = new Uint8Array(length);
while (length--) {
out[length] = bytes.charCodeAt(length);
}
return new Blob([out], { type: mimeType || "application/pdf" });
};
const removeFileExtension = (fileName) => {
if (!fileName) {
return "";
}
const fileNameSplit = fileName.split(".");
const fileNameSplitLength = fileNameSplit.length;
const modFileName = fileNameSplit.slice(0, fileNameSplitLength - 1);
return modFileName.join(".");
};
const isJson = (text) => {
try {
if (typeof text === "object") {
return true;
}
if (typeof text === "string") {
const json = JSON.parse(text);
return typeof json === "object";
}
return false;
} catch (_err) {
return false;
}
};
const displayPromptResult = (
output,
isFormat = false,
isHighlightEnabled = false,
) => {
/*
output: The data to be displayed or parsed
isFormat: A flag indicating whether the output should be formatted
*/
let i = 0;
let parsedData = output;
while (i < 3) {
i++;
try {
parsedData = JSON.parse(parsedData);
} catch {
// Break the loop if parsing fails
break;
}
}
if (!isFormat) {
// If formatting is not requested, return the parsed data directly
return parsedData;
}
// Check if the parsed data is an array or object and formatting is requested
if (Array.isArray(parsedData) || typeof parsedData === "object") {
// If formatting is requested, return the JSON string with indentation
if (isHighlightEnabled) {
return parsedData;
}
return JSON.stringify(parsedData, null, 4);
}
return String(parsedData);
};
// Deprecated adapters stay listed so users can see what to migrate off, but
// they cannot be selected or counted as configured.
const usableAdapters = (adapters) =>
(adapters || []).filter((adapter) => !adapter?.is_deprecated);
const onboardCompleted = (adaptersList) => {
if (!Array.isArray(adaptersList)) {
return false;
}
const MANDATORY_ADAPTERS = ["llm", "vector_db", "embedding", "x2text"];
adaptersList = adaptersList.map((element) => element.toLowerCase());
return MANDATORY_ADAPTERS.every((value) => adaptersList.includes(value));
};
// Input: ISOdateTime format
// Output: Mar 10, 2023 7:33 PM IST
const formattedDateTime = (ISOdateTime) => {
if (ISOdateTime) {
const validIsoDate = moment.utc(ISOdateTime).toISOString();
const zone = Intl.DateTimeFormat().resolvedOptions().timeZone;
return momentTz.tz(validIsoDate, zone).format("lll z");
} else {
return "";
}
};
/**
* Relative time for a backend timestamp ("3 days ago").
*
* Strict ISO parsing: malformed input returns "" (callers should hide the
* label) instead of moment's literal "Invalid date". Clamped to now so a
* slightly-fast server clock never renders "in a few seconds".
* @param {string} ISOdateTime ISO-8601 timestamp
* @returns {string} relative time, or "" when absent/malformed
*/
const timeAgo = (ISOdateTime) => {
const parsed = moment(ISOdateTime, moment.ISO_8601, true);
if (!parsed.isValid()) {
return "";
}
return moment.min(parsed, moment()).fromNow();
};
const getBackendErrorDetail = (attr, backendErrors) => {
if (backendErrors) {
// Handle structured error response with errors array
if (backendErrors?.errors && Array.isArray(backendErrors.errors)) {
const error = backendErrors.errors.find((error) => error?.attr === attr);
return error ? error?.detail : null;
}
// Handle direct field errors (e.g., { api_name: ["error message"] })
if (backendErrors[attr]) {
return Array.isArray(backendErrors[attr])
? backendErrors[attr].join(", ")
: backendErrors[attr];
}
// Handle nested errors in non_field_errors
if (backendErrors?.non_field_errors) {
return Array.isArray(backendErrors.non_field_errors)
? backendErrors.non_field_errors.join(", ")
: backendErrors.non_field_errors;
}
}
return null;
};
const titleCase = (str) => {
if (str === null || str.length === 0) {
return "";
}
const words = str.toLowerCase().split(" ");
for (let i = 0; i < words.length; i++) {
words[i] = words[i][0].toUpperCase() + words[i].slice(1);
}
return words.join(" ");
};
const getMenuItem = (label, key, icon, children, type, isDisabled) => {
return {
key,
icon,
children,
label,
type,
isDisabled,
};
};
const docIndexStatus = {
yet_to_start: "YET_TO_START",
indexing: "INDEXING",
done: "DONE",
};
const isNonNegativeNumber = (value) => {
return typeof value === "number" && !isNaN(value) && value >= 0;
};
// Icons are image URLs; an unresolved adapter falls back to an emoji.
// Detect the URL - compound (ZWJ) emoji break length heuristics.
const isImageUrl = (value) =>
typeof value === "string" && /^(https?:\/\/|\/|data:image\/)/.test(value);
// Default token usage object with all counts initialized to 0
const defaultTokenUsage = {
embedding_tokens: 0,
prompt_tokens: 0,
completion_tokens: 0,
total_tokens: 0,
};
// Generate a UUID
const generateUUID = () => {
const uuid = uuidv4();
return uuid;
};
function getFormattedTotalCost(tokenUsageDetails) {
const value = tokenUsageDetails?.cost_in_dollars ?? 0;
// Format the value to 5 decimal places or return "0" if the value is zero
return value === 0 ? 0 : value.toFixed(5);
}
const pollForCompletion = (
startTime,
requestOptions,
maxWaitTime,
pollingInterval,
makeApiRequest,
) => {
const elapsedTime = Date.now() - startTime;
if (elapsedTime >= maxWaitTime) {
return Promise.reject(
new Error(
"Unable to fetch results since there's an ongoing extraction, please try again later",
),
);
}
const recursivePoll = () => {
return makeApiRequest(requestOptions)
.then((response) => {
if (response?.data?.status === "pending") {
return new Promise((resolve) =>
setTimeout(resolve, pollingInterval),
).then(recursivePoll);
} else {
return response;
}
})
.catch((err) => {
throw err;
});
};
return recursivePoll();
};
function getDocIdFromKey(key) {
// Split the key by '__'
const parts = key.split("__");
// Return the docId part, which is the second element in the array
if (parts.length === 3) {
return parts[1];
} else {
return null;
}
}
const displayURL = (text) => {
return getBaseUrl() + "/" + text;
};
// Helper to strip trailing slashes without regex (avoids ReDoS concerns)
const stripTrailingSlashes = (str) => {
let end = str.length;
while (end > 0 && str[end - 1] === "/") {
end--;
}
return str.slice(0, end);
};
const shortenApiEndpoint = (url) => {
if (typeof url !== "string" || url.trim() === "") {
return "";
}
try {
// Parse URL to strip query/hash and get clean pathname
const parsed = new URL(url, globalThis.location.origin);
const path = stripTrailingSlashes(parsed.pathname);
const parts = path.split("/").filter(Boolean);
const suffix = parts[parts.length - 1] || "";
return suffix ? `.../${suffix}` : ".../";
} catch {
// Fallback for invalid URLs - strip query/hash manually
const path = stripTrailingSlashes(url.split("?")[0].split("#")[0]);
const parts = path.split("/").filter(Boolean);
const suffix = parts[parts.length - 1] || "";
return suffix ? `.../${suffix}` : ".../";
}
};
const formatNumberWithCommas = (number) => {
if (!number && number !== 0) {
return null;
}
// Convert the number to a string and split into integer and decimal parts.
const [integerPart, decimalPart] = number.toString().split(".");
// Add commas to the integer part.
const formattedIntegerPart = Number(integerPart).toLocaleString();
// Reassemble the formatted number, including the decimal part if it exists.
return decimalPart
? `${formattedIntegerPart}.${decimalPart}`
: formattedIntegerPart;
};
const isValidJsonKey = (key) => {
// Check for Prompt-Key
// Allowed case, contains alphanumeric characters and underscores,
// and doesn't start with a number.
const regex = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
return regex.test(key);
};
const PROMPT_RUN_TYPES = {
RUN_ONE_PROMPT_ONE_LLM_ONE_DOC: "RUN_ONE_PROMPT_ONE_LLM_ONE_DOC",
RUN_ONE_PROMPT_ONE_LLM_ALL_DOCS: "RUN_ONE_PROMPT_ONE_LLM_ALL_DOCS",
RUN_ONE_PROMPT_ALL_LLMS_ONE_DOC: "RUN_ONE_PROMPT_ALL_LLMS_ONE_DOC",
RUN_ONE_PROMPT_ALL_LLMS_ALL_DOCS: "RUN_ONE_PROMPT_ALL_LLMS_ALL_DOCS",
RUN_ALL_PROMPTS_ALL_LLMS_ONE_DOC: "RUN_ALL_PROMPTS_ALL_LLMS_ONE_DOC",
RUN_ALL_PROMPTS_ALL_LLMS_ALL_DOCS: "RUN_ALL_PROMPTS_ALL_LLMS_ALL_DOCS",
};
const PROMPT_RUN_API_STATUSES = {
RUNNING: "RUNNING",
COMPLETED: "COMPLETED",
};
const generateApiRunStatusId = (docId, profileId) => {
return `${docId}__${profileId}`;
};
const base64toBlobWithMime = (data, mimeType) => {
// Converts a base64 encoded string to a Blob object with the specified MIME type.
const byteCharacters = atob(data?.data); // Decode base64
const byteArrays = [];
for (let offset = 0; offset < byteCharacters.length; offset += 512) {
const slice = byteCharacters.slice(offset, offset + 512);
const byteNumbers = new Array(slice.length);
for (let i = 0; i < slice.length; i++) {
byteNumbers[i] = slice.charCodeAt(i);
}
const byteArray = new Uint8Array(byteNumbers);
byteArrays.push(byteArray);
}
return new Blob(byteArrays, { type: mimeType });
};
const generateCoverageKey = (promptId, profileId) => {
return `coverage_${promptId}_${profileId}`;
};
function formatSecondsToHMS(seconds) {
if (isNaN(seconds) || seconds < 0) {
return "00:00:00";
}
const hrs = Math.floor(seconds / 3600);
const mins = Math.floor((seconds % 3600) / 60);
const secs = Math.floor(seconds % 60);
return [hrs, mins, secs]
.map((unit) => String(unit).padStart(2, "0"))
.join(":");
}
const formattedDateTimeWithSeconds = (ISOdateTime) => {
if (ISOdateTime) {
const zone = Intl.DateTimeFormat().resolvedOptions().timeZone;
const zonedDate = parseISO(ISOdateTime);
return format(zonedDate, "MMM d, yyyy h:mm:ss a zzz", { timeZone: zone });
} else {
return "";
}
};
const TRIAL_PLAN = "TRIAL";
const homePagePath = cloudHomePagePath || "tools";
const convertTimestampToHHMMSS = (timestamp) => {
// Convert the timestamp to milliseconds
const date = new Date(timestamp * 1000);
// Extract hours, minutes, and seconds
const [hours, minutes, seconds] = [
date.getUTCHours(),
date.getUTCMinutes(),
date.getUTCSeconds(),
].map((unit) => unit.toString().padStart(2, "0"));
// Return the formatted time string
return `${hours}:${minutes}:${seconds}`;
};
const formatTimeDisplay = (seconds) => {
// Format time display for TTL or duration display
if (seconds <= 0) {
return "Expired";
}
const days = Math.floor(seconds / 86400);
const hours = Math.floor((seconds % 86400) / 3600);
const minutes = Math.floor((seconds % 3600) / 60);
const secs = Math.floor(seconds % 60);
const parts = [];
if (days > 0) {
parts.push(`${days}d`);
}
if (hours > 0) {
parts.push(`${hours}h`);
}
if (minutes > 0) {
parts.push(`${minutes}m`);
}
if (secs > 0 || parts.length === 0) {
parts.push(`${secs}s`);
}
return parts.join(" ");
};
const copyToClipboard = async (text) => {
try {
await navigator.clipboard.writeText(text);
return true;
} catch {
// Clipboard API may fail in insecure contexts or unsupported browsers
return false;
}
};
const UNSTRACT_ADMIN = "unstract_admin";
const logsStaticContent = {
ETL: {
addBtn: "ETL Pipeline",
route: "etl",
},
TASK: {
addBtn: "Task Pipeline",
route: "task",
},
API: {
addBtn: "API Deployment",
route: "api",
},
WF: {
addBtn: "Workflow",
route: "workflows",
},
};
export {
base64toBlob,
base64toBlobWithMime,
CONNECTOR_TYPE_MAP,
calculateDivHeight,
convertTimestampToHHMMSS,
copyToClipboard,
defaultTokenUsage,
deploymentApiTypes,
deploymentsStaticContent,
deploymentTypes,
displayPromptResult,
displayURL,
docIndexStatus,
endpointType,
formatBytes,
formatNumberWithCommas,
formatSecondsToHMS,
formatTimeDisplay,
formattedDateTime,
formattedDateTimeWithSeconds,
generateApiRunStatusId,
generateCoverageKey,
generateUUID,
getBackendErrorDetail,
getBaseUrl,
getDateTimeString,
getDocIdFromKey,
getFormattedTotalCost,
getMenuItem,
getOrgNameFromPathname,
getReadableDateAndTime,
getSequenceNumber,
getTimeForLogs,
homePagePath,
isImageUrl,
isJson,
isNonNegativeNumber,
isValidJsonKey,
listOfAppDeployments,
logsStaticContent,
O_AUTH_PROVIDERS,
onboardCompleted,
PROMPT_RUN_API_STATUSES,
PROMPT_RUN_TYPES,
pollForCompletion,
promptStudioUpdateStatus,
promptType,
publicRoutes,
removeFileExtension,
replaceFirstRoute,
setInitialWorkflowInstance,
shortenApiEndpoint,
sourceTypes,
THEME,
TRIAL_PLAN,
timeAgo,
titleCase,
toolIdeOutput,
UNSTRACT_ADMIN,
usableAdapters,
wfExecutionTypes,
workflowStatus,
};