LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Fix/remove duplicate input options (#476)
Rahul Johny0af6784167be
diff --git a/frontend/package-lock.json b/frontend/package-lock.jsonindex 018771100..2c7eec1d6 100644--- a/frontend/package-lock.json+++ b/frontend/package-lock.json@@ -25,11 +25,13 @@"cronstrue": "^2.48.0","emoji-picker-react": "^4.8.0","emoji-regex": "^10.3.0",+ "file-saver": "^2.0.5","framer-motion": "^11.2.10","handlebars": "^4.7.8","http-proxy-middleware": "^2.0.6","js-cookie": "^3.0.5","js-yaml": "^4.1.0",+ "json-2-csv": "^5.5.4","markdown-to-jsx": "^7.2.1","moment": "^2.29.4","moment-timezone": "^0.5.45",@@ -7520,6 +7522,14 @@"resolved": "https://registry.npmjs.org/dedent/-/dedent-0.7.0.tgz","integrity": "sha512-Q6fKUPqnAHAyhiUgFU7BUzLiv0kd8saH9al7tnu5Q/okj6dnupxyTgFIBjVzJATdfIAm9NAsvXNzjaKa+bxVyA=="},+ "node_modules/deeks": {+ "version": "3.1.0",+ "resolved": "https://registry.npmjs.org/deeks/-/deeks-3.1.0.tgz",+ "integrity": "sha512-e7oWH1LzIdv/prMQ7pmlDlaVoL64glqzvNgkgQNgyec9ORPHrT2jaOqMtRyqJuwWjtfb6v+2rk9pmaHj+F137A==",+ "engines": {+ "node": ">= 16"+ }+ },"node_modules/deep-equal": {"version": "2.2.1","resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-2.2.1.tgz",@@ -7748,6 +7758,14 @@"node": ">=6"}},+ "node_modules/doc-path": {+ "version": "4.1.1",+ "resolved": "https://registry.npmjs.org/doc-path/-/doc-path-4.1.1.tgz",+ "integrity": "sha512-h1ErTglQAVv2gCnOpD3sFS6uolDbOKHDU1BZq+Kl3npPqroU3dYL42lUgMfd5UimlwtRgp7C9dLGwqQ5D2HYgQ==",+ "engines": {+ "node": ">=16"+ }+ },"node_modules/doctrine": {"version": "3.0.0","resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz",@@ -9045,6 +9063,11 @@"webpack": "^4.0.0 || ^5.0.0"}},+ "node_modules/file-saver": {+ "version": "2.0.5",+ "resolved": "https://registry.npmjs.org/file-saver/-/file-saver-2.0.5.tgz",+ "integrity": "sha512-P9bmyZ3h/PRG+Nzga+rbdI4OEpNDzAVyy74uVO9ATgzLK6VtAsYybF/+TOCvrc0MO793d6+42lLyZTw7/ArVzA=="+ },"node_modules/filelist": {"version": "1.0.4","resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz",@@ -12749,6 +12772,18 @@"node": ">=4"}},+ "node_modules/json-2-csv": {+ "version": "5.5.4",+ "resolved": "https://registry.npmjs.org/json-2-csv/-/json-2-csv-5.5.4.tgz",+ "integrity": "sha512-gB24IF5SvZn7QhEh6kp9QwFhRnI3FVEEXAGyq0xtPxqOQ4odYU3PU9pFKRoR1SGABxunQlBP6VFv0c8EnLbsLQ==",+ "dependencies": {+ "deeks": "3.1.0",+ "doc-path": "4.1.1"+ },+ "engines": {+ "node": ">= 16"+ }+ },"node_modules/json-parse-even-better-errors": {"version": "2.3.1","resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz",@@ -25878,6 +25913,11 @@"resolved": "https://registry.npmjs.org/dedent/-/dedent-0.7.0.tgz","integrity": "sha512-Q6fKUPqnAHAyhiUgFU7BUzLiv0kd8saH9al7tnu5Q/okj6dnupxyTgFIBjVzJATdfIAm9NAsvXNzjaKa+bxVyA=="},+ "deeks": {+ "version": "3.1.0",+ "resolved": "https://registry.npmjs.org/deeks/-/deeks-3.1.0.tgz",+ "integrity": "sha512-e7oWH1LzIdv/prMQ7pmlDlaVoL64glqzvNgkgQNgyec9ORPHrT2jaOqMtRyqJuwWjtfb6v+2rk9pmaHj+F137A=="+ },"deep-equal": {"version": "2.2.1","resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-2.2.1.tgz",@@ -26052,6 +26092,11 @@"@leichtgewicht/ip-codec": "^2.0.1"}},+ "doc-path": {+ "version": "4.1.1",+ "resolved": "https://registry.npmjs.org/doc-path/-/doc-path-4.1.1.tgz",+ "integrity": "sha512-h1ErTglQAVv2gCnOpD3sFS6uolDbOKHDU1BZq+Kl3npPqroU3dYL42lUgMfd5UimlwtRgp7C9dLGwqQ5D2HYgQ=="+ },"doctrine": {"version": "3.0.0","resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz",@@ -27018,6 +27063,11 @@"schema-utils": "^3.0.0"}},+ "file-saver": {+ "version": "2.0.5",+ "resolved": "https://registry.npmjs.org/file-saver/-/file-saver-2.0.5.tgz",+ "integrity": "sha512-P9bmyZ3h/PRG+Nzga+rbdI4OEpNDzAVyy74uVO9ATgzLK6VtAsYybF/+TOCvrc0MO793d6+42lLyZTw7/ArVzA=="+ },"filelist": {"version": "1.0.4","resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz",@@ -29794,6 +29844,15 @@"resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz","integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA=="},+ "json-2-csv": {+ "version": "5.5.4",+ "resolved": "https://registry.npmjs.org/json-2-csv/-/json-2-csv-5.5.4.tgz",+ "integrity": "sha512-gB24IF5SvZn7QhEh6kp9QwFhRnI3FVEEXAGyq0xtPxqOQ4odYU3PU9pFKRoR1SGABxunQlBP6VFv0c8EnLbsLQ==",+ "requires": {+ "deeks": "3.1.0",+ "doc-path": "4.1.1"+ }+ },"json-parse-even-better-errors": {"version": "2.3.1","resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz",diff --git a/frontend/package.json b/frontend/package.jsonindex 674ef6689..9a9bd21ae 100644--- a/frontend/package.json+++ b/frontend/package.json@@ -20,11 +20,13 @@"cronstrue": "^2.48.0","emoji-picker-react": "^4.8.0","emoji-regex": "^10.3.0",+ "file-saver": "^2.0.5","framer-motion": "^11.2.10","handlebars": "^4.7.8","http-proxy-middleware": "^2.0.6","js-cookie": "^3.0.5","js-yaml": "^4.1.0",+ "json-2-csv": "^5.5.4","markdown-to-jsx": "^7.2.1","moment": "^2.29.4","moment-timezone": "^0.5.45",diff --git a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsxindex 9681a96c8..fdc752d03 100644--- a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx+++ b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx@@ -89,21 +89,26 @@ function DsSettingsCard({ type, endpointDetails, message }) {input: <ImportOutlined className="ds-set-icon-size" />,output: <ExportOutlined className="ds-set-icon-size" />,};++ const setUpdatedInputoptions = (inputOption) => {+ setInputOptions((prevInputOptions) => {+ // Check if inputOption already exists in prevInputOptions+ if (prevInputOptions.some((opt) => opt.value === inputOption.value)) {+ return prevInputOptions; // Return previous state unchanged+ } else {+ // Create a new array with the existing options and the new option+ const updatedInputOptions = [...prevInputOptions, inputOption];+ return updatedInputOptions;+ }+ });+ };+useEffect(() => {try {const inputOption =require("../../../plugins/dscard-input-options/DsSettingsCardInputOptions").inputOption;if (flags.manual_review && inputOption) {- setInputOptions((prevInputOptions) => {- // Check if inputOption already exists in prevInputOptions- if (prevInputOptions.some((opt) => opt.value === inputOption.value)) {- return prevInputOptions; // Return previous state unchanged- } else {- // Create a new array with the existing options and the new option- const updatedInputOptions = [...prevInputOptions, inputOption];- return updatedInputOptions;- }- });+ setUpdatedInputoptions(inputOption);}} catch {// The component will remain null of it is not available@@ -114,9 +119,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {const inputOption =require("../../../plugins/dscard-input-options/AppDeploymentCardInputOptions").appDeploymentInputOption;if (flags.app_deployment && inputOption) {- const updatedInputOptions = inputOptions;- updatedInputOptions.push(inputOption);- setInputOptions(updatedInputOptions);+ setUpdatedInputoptions(inputOption);}} catch {// The component will remain null of it is not available