Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2566 [FIX] Preserve non-Latin characters by adding ensure_ascii=False to repair_json (#1387)

Deepak K7d23f6369388
diff --git a/prompt-service/pyproject.toml b/prompt-service/pyproject.toml
index c2f216698..8695385dd 100644
--- a/prompt-service/pyproject.toml
+++ b/prompt-service/pyproject.toml
@@ -16,7 +16,7 @@ dependencies = [
"json-repair~=0.42.0",
# TODO: Temporarily removing the extra dependencies of aws and gcs from unstract-sdk
# to resolve lock file. Will have to be re-looked into
- "unstract-sdk[azure]~=0.73.1",
+ "unstract-sdk[azure]~=0.73.2",
"gcsfs==2024.10.0",
"s3fs==2024.10.0",
"redis>=5.0.3,<5.3",
diff --git a/prompt-service/src/unstract/prompt_service/constants.py b/prompt-service/src/unstract/prompt_service/constants.py
index 2035fb742..d41d246ba 100644
--- a/prompt-service/src/unstract/prompt_service/constants.py
+++ b/prompt-service/src/unstract/prompt_service/constants.py
@@ -94,7 +94,7 @@ class PromptServiceConstants:
COMBINED_PROMPT = "combined_prompt"
TOOL = "tool"
JSON_POSTAMBLE = "JSON_POSTAMBLE"
- DEFAULT_JSON_POSTAMBLE = "Wrap the final JSON result inbetween ### like below example:\n###\n<FINAL_JSON_RESULT>\n###"
+ DEFAULT_JSON_POSTAMBLE = "Wrap the final JSON result inbetween §§§ like below example:\n§§§\n<FINAL_JSON_RESULT>\n§§§"
DOCUMENT_TYPE = "document_type"
diff --git a/prompt-service/src/unstract/prompt_service/services/answer_prompt.py b/prompt-service/src/unstract/prompt_service/services/answer_prompt.py
index 595cdf010..be9b84f9b 100644
--- a/prompt-service/src/unstract/prompt_service/services/answer_prompt.py
+++ b/prompt-service/src/unstract/prompt_service/services/answer_prompt.py
@@ -246,10 +246,12 @@ class AnswerPromptService:
structured_output[prompt_key] = None
else:
# Attempt parsing as-is (could be a valid object, array, or partial JSON)
- a = repair_json(json_str=answer, return_objects=True)
+ a = repair_json(json_str=answer, return_objects=True, ensure_ascii=False)
# Attempt parsing with array wrap (useful for multiple comma-separated objects like {}, {}, {})
- b = repair_json(json_str="[" + answer, return_objects=True)
+ b = repair_json(
+ json_str="[" + answer, return_objects=True, ensure_ascii=False
+ )
# Heuristic: if wrapping only added '[' and ']', len(b) - len(a) == 2 → original was valid, use 'a'
# Otherwise, fallback to 'b' which likely fixed multiple items or invalid top-level structure
diff --git a/prompt-service/uv.lock b/prompt-service/uv.lock
index 9155a3db4..bee10d40c 100644
--- a/prompt-service/uv.lock
+++ b/prompt-service/uv.lock
@@ -3143,7 +3143,7 @@ requires-dist = [
{ name = "s3fs", specifier = "==2024.10.0" },
{ name = "unstract-core", editable = "../unstract/core" },
{ name = "unstract-flags", editable = "../unstract/flags" },
- { name = "unstract-sdk", extras = ["azure"], specifier = "~=0.73.1" },
+ { name = "unstract-sdk", extras = ["azure"], specifier = "~=0.73.2" },
]
[package.metadata.requires-dev]
@@ -3166,7 +3166,7 @@ test = [
[[package]]
name = "unstract-sdk"
-version = "0.73.1"
+version = "0.73.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "filetype" },
@@ -3205,9 +3205,9 @@ dependencies = [
{ name = "tiktoken" },
{ name = "transformers" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/88/bb/60a8cc6d352ca204e04bf33a85d9b9961415833e0f314b2cbbcd42c5a631/unstract_sdk-0.73.1.tar.gz", hash = "sha256:294d0206bc43c09bc8c884a77ab78794301de0dab3ff6a1fe68f9c5235f72251", size = 2376089 }
+sdist = { url = "https://files.pythonhosted.org/packages/ad/07/8bdc82327d6d30ed5c120aa607b044fad88dd50a0e0ac340a0214834d254/unstract_sdk-0.73.2.tar.gz", hash = "sha256:8285e44997b2145ec5920e0cd651312fd0cebf1742bb314f516f59f87f7846f4", size = 2376138 }
wheels = [
- { url = "https://files.pythonhosted.org/packages/d8/38/4696fe06d95fd62153dd7a270332e3bfc76a4ffc7e57c0b9135459666317/unstract_sdk-0.73.1-py3-none-any.whl", hash = "sha256:527c4904a8a4500db5ebba0ee3db216069d8093d064187c1e09eecde42402ef9", size = 266247 },
+ { url = "https://files.pythonhosted.org/packages/e5/45/c90c9d922e62bd3935e3fbb045b3083ae56148aa7bc56ba5ee8c3549ade1/unstract_sdk-0.73.2-py3-none-any.whl", hash = "sha256:7faf204411e5c8b7037fbaae4697e772b22cb4b045e0ec3d961dd1469952e330", size = 266302 },
]
[package.optional-dependencies]