LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2566 [FIX] Preserve non-Latin characters by adding ensure_ascii=False to repair_json (#1387)
Deepak K7d23f6369388
diff --git a/prompt-service/pyproject.toml b/prompt-service/pyproject.tomlindex c2f216698..8695385dd 100644--- a/prompt-service/pyproject.toml+++ b/prompt-service/pyproject.toml@@ -16,7 +16,7 @@ dependencies = ["json-repair~=0.42.0",# TODO: Temporarily removing the extra dependencies of aws and gcs from unstract-sdk# to resolve lock file. Will have to be re-looked into- "unstract-sdk[azure]~=0.73.1",+ "unstract-sdk[azure]~=0.73.2","gcsfs==2024.10.0","s3fs==2024.10.0","redis>=5.0.3,<5.3",diff --git a/prompt-service/src/unstract/prompt_service/constants.py b/prompt-service/src/unstract/prompt_service/constants.pyindex 2035fb742..d41d246ba 100644--- a/prompt-service/src/unstract/prompt_service/constants.py+++ b/prompt-service/src/unstract/prompt_service/constants.py@@ -94,7 +94,7 @@ class PromptServiceConstants:COMBINED_PROMPT = "combined_prompt"TOOL = "tool"JSON_POSTAMBLE = "JSON_POSTAMBLE"- DEFAULT_JSON_POSTAMBLE = "Wrap the final JSON result inbetween ### like below example:\n###\n<FINAL_JSON_RESULT>\n###"+ DEFAULT_JSON_POSTAMBLE = "Wrap the final JSON result inbetween §§§ like below example:\n§§§\n<FINAL_JSON_RESULT>\n§§§"DOCUMENT_TYPE = "document_type"diff --git a/prompt-service/src/unstract/prompt_service/services/answer_prompt.py b/prompt-service/src/unstract/prompt_service/services/answer_prompt.pyindex 595cdf010..be9b84f9b 100644--- a/prompt-service/src/unstract/prompt_service/services/answer_prompt.py+++ b/prompt-service/src/unstract/prompt_service/services/answer_prompt.py@@ -246,10 +246,12 @@ class AnswerPromptService:structured_output[prompt_key] = Noneelse:# Attempt parsing as-is (could be a valid object, array, or partial JSON)- a = repair_json(json_str=answer, return_objects=True)+ a = repair_json(json_str=answer, return_objects=True, ensure_ascii=False)# Attempt parsing with array wrap (useful for multiple comma-separated objects like {}, {}, {})- b = repair_json(json_str="[" + answer, return_objects=True)+ b = repair_json(+ json_str="[" + answer, return_objects=True, ensure_ascii=False+ )# Heuristic: if wrapping only added '[' and ']', len(b) - len(a) == 2 → original was valid, use 'a'# Otherwise, fallback to 'b' which likely fixed multiple items or invalid top-level structurediff --git a/prompt-service/uv.lock b/prompt-service/uv.lockindex 9155a3db4..bee10d40c 100644--- a/prompt-service/uv.lock+++ b/prompt-service/uv.lock@@ -3143,7 +3143,7 @@ requires-dist = [{ name = "s3fs", specifier = "==2024.10.0" },{ name = "unstract-core", editable = "../unstract/core" },{ name = "unstract-flags", editable = "../unstract/flags" },- { name = "unstract-sdk", extras = ["azure"], specifier = "~=0.73.1" },+ { name = "unstract-sdk", extras = ["azure"], specifier = "~=0.73.2" },][package.metadata.requires-dev]@@ -3166,7 +3166,7 @@ test = [[[package]]name = "unstract-sdk"-version = "0.73.1"+version = "0.73.2"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "filetype" },@@ -3205,9 +3205,9 @@ dependencies = [{ name = "tiktoken" },{ name = "transformers" },]-sdist = { url = "https://files.pythonhosted.org/packages/88/bb/60a8cc6d352ca204e04bf33a85d9b9961415833e0f314b2cbbcd42c5a631/unstract_sdk-0.73.1.tar.gz", hash = "sha256:294d0206bc43c09bc8c884a77ab78794301de0dab3ff6a1fe68f9c5235f72251", size = 2376089 }+sdist = { url = "https://files.pythonhosted.org/packages/ad/07/8bdc82327d6d30ed5c120aa607b044fad88dd50a0e0ac340a0214834d254/unstract_sdk-0.73.2.tar.gz", hash = "sha256:8285e44997b2145ec5920e0cd651312fd0cebf1742bb314f516f59f87f7846f4", size = 2376138 }wheels = [- { url = "https://files.pythonhosted.org/packages/d8/38/4696fe06d95fd62153dd7a270332e3bfc76a4ffc7e57c0b9135459666317/unstract_sdk-0.73.1-py3-none-any.whl", hash = "sha256:527c4904a8a4500db5ebba0ee3db216069d8093d064187c1e09eecde42402ef9", size = 266247 },+ { url = "https://files.pythonhosted.org/packages/e5/45/c90c9d922e62bd3935e3fbb045b3083ae56148aa7bc56ba5ee8c3549ade1/unstract_sdk-0.73.2-py3-none-any.whl", hash = "sha256:7faf204411e5c8b7037fbaae4697e772b22cb4b045e0ec3d961dd1469952e330", size = 266302 },][package.optional-dependencies]