Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

[FIX] Changes to display chunk data properly (#821)

Deepak K20032898d2fa
diff --git a/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py b/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py
index f4d1d6624..c65ea94ea 100644
--- a/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py
+++ b/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py
@@ -152,7 +152,7 @@ class OutputManagerHelper:
output=output,
eval_metrics=eval_metrics,
tool=tool,
- context=context,
+ context=json.dumps(context),
challenge_data=challenge_data,
)
diff --git a/backend/prompt_studio/prompt_studio_output_manager/serializers.py b/backend/prompt_studio/prompt_studio_output_manager/serializers.py
index 0fd7c4060..14840745a 100644
--- a/backend/prompt_studio/prompt_studio_output_manager/serializers.py
+++ b/backend/prompt_studio/prompt_studio_output_manager/serializers.py
@@ -1,3 +1,4 @@
+import json
import logging
from usage.helper import UsageHelper
@@ -25,4 +26,10 @@ class PromptStudioOutputSerializer(AuditSerializer):
)
token_usage = {}
data["token_usage"] = token_usage
+ # Convert string to list
+ try:
+ data["context"] = json.loads(data["context"])
+ except json.JSONDecodeError:
+ # Convert the old value of data["context"] to a list
+ data["context"] = [data["context"]]
return data
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py
index b5ee4dae1..a82ed95bf 100644
--- a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py
@@ -154,7 +154,7 @@ class OutputManagerHelper:
output=output,
eval_metrics=eval_metrics,
tool=tool,
- context=context,
+ context=json.dumps(context),
challenge_data=challenge_data,
)
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py b/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py
index f3b769c1b..0d40826ec 100644
--- a/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py
@@ -1,3 +1,4 @@
+import json
import logging
from usage_v2.helper import UsageHelper
@@ -25,4 +26,10 @@ class PromptStudioOutputSerializer(AuditSerializer):
)
token_usage = {}
data["token_usage"] = token_usage
+ # Convert string to list
+ try:
+ data["context"] = json.loads(data["context"])
+ except json.JSONDecodeError:
+ # Convert the old value of data["context"] to a list
+ data["context"] = [data["context"]]
return data
diff --git a/frontend/src/components/custom-tools/prompt-card/OutputForIndex.jsx b/frontend/src/components/custom-tools/prompt-card/OutputForIndex.jsx
index 506414174..0fa72073a 100644
--- a/frontend/src/components/custom-tools/prompt-card/OutputForIndex.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/OutputForIndex.jsx
@@ -16,19 +16,7 @@ function OutputForIndex({ chunkData, setIsIndexOpen, isIndexOpen }) {
const activeRef = useRef(null);
useEffect(() => {
- if (!chunkData) {
- setChunks([]);
- }
- // Split chunkData into chunks using \f\n delimiter
- const tempChunks = chunkData?.split("\f\n");
- // To remove " at the end
- if (tempChunks?.length > 0) {
- const lastChunk = tempChunks[tempChunks?.length - 1].trim();
- if (lastChunk === '\\n"' || lastChunk === "") {
- tempChunks.pop();
- }
- }
- setChunks(tempChunks);
+ setChunks(chunkData || []);
}, [chunkData]);
// Debounced search handler
diff --git a/prompt-service/src/unstract/prompt_service/helper.py b/prompt-service/src/unstract/prompt_service/helper.py
index ee730173e..0754b74c7 100644
--- a/prompt-service/src/unstract/prompt_service/helper.py
+++ b/prompt-service/src/unstract/prompt_service/helper.py
@@ -83,11 +83,11 @@ def plugin_loader(app: Flask) -> None:
initialize_plugin_endpoints(app=app)
-def get_cleaned_context(context: str) -> str:
+def get_cleaned_context(context: set[str]) -> list[str]:
clean_context_plugin: dict[str, Any] = plugins.get(PSKeys.CLEAN_CONTEXT, {})
if clean_context_plugin:
return clean_context_plugin["entrypoint_cls"].run(context=context)
- return context
+ return list(context)
def initialize_plugin_endpoints(app: Flask) -> None:
diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.py
index 15a46c8fd..8ba54d4ef 100644
--- a/prompt-service/src/unstract/prompt_service/main.py
+++ b/prompt-service/src/unstract/prompt_service/main.py
@@ -251,10 +251,10 @@ def prompt_processor() -> Any:
raise api_error
try:
- context = ""
+ context: set[str] = set()
if output[PSKeys.CHUNK_SIZE] == 0:
# We can do this only for chunkless indexes
- context: Optional[str] = index.query_index(
+ retrieved_context: Optional[str] = index.query_index(
embedding_instance_id=output[PSKeys.EMBEDDING],
vector_db_instance_id=output[PSKeys.VECTOR_DB],
doc_id=doc_id,
@@ -270,13 +270,13 @@ def prompt_processor() -> Any:
# inconsistent, and not reproducible easily,
# this is just a safety net.
time.sleep(2)
- context: Optional[str] = index.query_index(
+ retrieved_context: Optional[str] = index.query_index(
embedding_instance_id=output[PSKeys.EMBEDDING],
vector_db_instance_id=output[PSKeys.VECTOR_DB],
doc_id=doc_id,
usage_kwargs=usage_kwargs,
)
- if context is None:
+ if retrieved_context is None:
# TODO: Obtain user set name for vector DB
msg = NO_CONTEXT_ERROR
app.logger.error(
@@ -294,6 +294,7 @@ def prompt_processor() -> Any:
msg,
)
raise APIError(message=msg)
+ context.add(retrieved_context)
# TODO: Use vectorDB name when available
publish_log(
log_events_id,
@@ -323,7 +324,7 @@ def prompt_processor() -> Any:
tool_settings=tool_settings,
output=output,
llm=llm,
- context=context,
+ context="\n".join(context),
prompt="promptx",
metadata=metadata,
)
@@ -537,7 +538,7 @@ def prompt_processor() -> Any:
llm=llm,
challenge_llm=challenge_llm,
run_id=run_id,
- context=context,
+ context="\n".join(context),
tool_settings=tool_settings,
output=output,
structured_output=structured_output,
@@ -593,7 +594,7 @@ def prompt_processor() -> Any:
try:
evaluator = eval_plugin["entrypoint_cls"](
"",
- context,
+ "\n".join(context),
"",
"",
output,
@@ -680,7 +681,7 @@ def run_retrieval( # type:ignore
retrieval_type: str,
metadata: dict[str, Any],
) -> tuple[str, str]:
- context: str = ""
+ context: set[str] = set()
prompt = output[PSKeys.PROMPTX]
if retrieval_type == PSKeys.SUBQUESTION:
subq_prompt: str = (
@@ -713,19 +714,11 @@ def run_retrieval( # type:ignore
prompt=subq_prompt,
)
subquestion_list = subquestions.split(",")
- raw_retrieved_context = ""
for each_subq in subquestion_list:
retrieved_context = _retrieve_context(
output, doc_id, vector_index, each_subq
)
- # Not adding the potential for pinecode serverless
- # inconsistency issue owing to risk of infinte loop
- # and inablity to diffrentiate genuine cases of
- # empty context.
- raw_retrieved_context = "\f\n".join(
- [raw_retrieved_context, retrieved_context]
- )
- context = _remove_duplicate_nodes(raw_retrieved_context)
+ context.update(retrieved_context)
if retrieval_type == PSKeys.SIMPLE:
@@ -746,7 +739,7 @@ def run_retrieval( # type:ignore
tool_settings=tool_settings,
output=output,
llm=llm,
- context=context,
+ context="\n".join(context),
prompt="promptx",
metadata=metadata,
)
@@ -754,13 +747,7 @@ def run_retrieval( # type:ignore
return (answer, context)
-def _remove_duplicate_nodes(retrieved_context: str) -> str:
- context_set: set[str] = set(retrieved_context.split("\f\n"))
- fomatted_context = "\f\n".join(context_set)
- return fomatted_context
-
-
-def _retrieve_context(output, doc_id, vector_index, answer) -> str:
+def _retrieve_context(output, doc_id, vector_index, answer) -> set[str]:
retriever = vector_index.as_retriever(
similarity_top_k=output[PSKeys.SIMILARITY_TOP_K],
filters=MetadataFilters(
@@ -773,18 +760,18 @@ def _retrieve_context(output, doc_id, vector_index, answer) -> str:
),
)
nodes = retriever.retrieve(answer)
- text = ""
+ context: set[str] = set()
for node in nodes:
# ToDo: May have to fine-tune this value for node score or keep it
# configurable at the adapter level
if node.score > 0:
- text += node.get_content() + "\f\n"
+ context.add(node.get_content())
else:
app.logger.info(
"Node score is less than 0. "
f"Ignored: {node.node_id} with score {node.score}"
)
- return text
+ return context
def log_exceptions(e: HTTPException):